Use Skopeo instead of podman

Dockerfile-locust

@@ -1,11 +1,55 @@
-FROM fedora:32
+ARG PYTHON_VERSION=3.8.6-alpine3.12
 
-RUN dnf -y update && dnf -y install fuse3-devel && yum install python3
-RUN dnf -y install podman
+FROM python:${PYTHON_VERSION} as builder
 
-ADD . /mnt
-WORKDIR /mnt
+ENV PYTHONUNBUFFERED 1
 
-RUN pip install -r /mnt/requirements.txt
+# build time dependencies
+RUN apk update && \
+        apk add --no-cache \
+        gcc \
+        g++ \
+        musl-dev \
+        linux-headers \
+        zeromq-dev \
+        libffi-dev \
+        make
+
+# build wheels instead of installing
+WORKDIR /wheels
+
+COPY requirements.txt .
+
+RUN pip install -U pip && \
+    pip wheel -r requirements.txt
+
+
+FROM python:${PYTHON_VERSION}
+
+# dependencies you need in your final image
+RUN apk update && \
+    apk add --no-cache \
+        bash \
+        libzmq
+
+# copy built previously wheels archives
+COPY --from=builder /wheels /wheels
+
+COPY requirements.txt /wheels/requirements.txt
+
+# use archives from /weels dir
+RUN pip install -U pip \
+       && pip install -r /wheels/requirements.txt -f /wheels \
+       && rm -rf /wheels \
+       && rm -rf /root/.cache/pip/*
+
+# install skopeo
+RUN apk add skopeo
+
+# Add the locustfiles
+COPY locustfiles /quay-performance-scripts/locustfiles
+
+# Expose locust web ui port
 EXPOSE 8089
-CMD ["locust", "-f", "/mnt/testfiles/run.py"]
+
+CMD ["locust", "-f", "/quay-performance-scripts/locustfiles/run.py"]

README.md

@@ -16,7 +16,6 @@ There are a few prerequisites and they will be explained below.
 
 ### Prerequisites
 
-- Create an Elasticsearch instance. The results will be stored here.
 - Deploy a Kubernetes environment. The tests will run within the cluster.
 - Deploy Quay, itself.
 - In Quay, as a superuser (important), create an organization for testing
@@ -73,6 +72,13 @@ file to change the behavior of the tests.
 
 ## Changelog
 
+**v0.1.0**
+changes:
+
+- Tests are run using locust framework
+- Concurrent testing is done using Locust in distributed mode
+- Metrics are now exported as Prometheus metrics
+
 **v0.0.2**
 
 changes:
@@ -116,12 +122,47 @@ The project expects the following environment variables:
 - PODMAN_PASSWORD: Password for the above user
 - PODMAN_HOST: The url of the host registry where images will be pushed
 - QUAY_HOST: The url where Quay is hosted
-- AUTH_TOKEN: The Authorization Token to enable API calls(On Quay: Create an organization followed by creating an application in the organization. Generate token for the application.)
+- OAUTH_TOKEN: The Authorization Token to enable API calls(On Quay: Create an organization followed by creating an application in the organization. Generate token for the application.)
+
+### Building
+
+From the main directory, the docker image can be built using the command: 
+
+```bash
+docker build -t perf-test -f Dockerfile-locust .
+```
 
 ### Running
 
-From the main directory, the docker image can be built using the command: `docker build -t perf-test -f Dockerfile-locust .`
-The built image can be run using the command: `docker run -e PODMAN_USERNAME="username" -e PODMAN_PASSWORD="password" -e PODMAN_HOST="localhost:8080" -e QUAY_HOST="http://www.localhost:8080" -e AUTH_TOKEN="abc" --privileged -v /tmp/csivvv:/var/lib/containers -p 8089:8089 --name quay-test -d perf-test`
-Upon successful starting of the container, the locust dashboard is accessible on port 8089.
+#### Locally for dev
+
+```
+docker run -e PODMAN_USERNAME="username" \
+    -e PODMAN_PASSWORD="password" \
+    -e PODMAN_HOST="localhost:8080" \
+    -e QUAY_HOST="http://www.localhost:8080" \
+    -e AUTH_TOKEN="abc" --privileged \
+    -v /tmp/csivvv:/var/lib/containers \
+    -p 8089:8089 --name quay-test -d perf-test`
+```
+
+Upon successful starting of the container, the locust dashboard is accessible
+on port 8089.
+
+The minimum number of users spawned to run the tests must be at least equal to
+the number of users defined in `testfiles/run.py` to run all user classes.
+
+#### Cluster
+
+The tests are run via locust in distributed mode. There is a single master
+which controls multiple worker pods. The number of replicas for the workers are
+defined in `deploy/locust-distributed.yaml` file. 
+
+Edit the `ConfigMap` `quay-locust-config` in the
+`deploy/locust-distributed.yaml` and set the variables accordingly
+
+Deploy locust on the cluster by running:
 
-The minimum number of users spawned to run the tests must be at least equal to the number of users defined in `testfiles/run.py` to run all user classes.
+```
+kubectl apply -f deploy/locust-distributed.yaml
+```

docker-compose.yaml

@@ -10,7 +10,7 @@ services:
             - ./:/mnt/
         environment:
             PYTHONPATH: '/mnt'
-        command: -f /mnt/testfiles/podman_operations.py
+        command: -f /mnt/locustfiles/skopeo_operations.py
         network_mode: host
 
     locust-metrics-exporter:

testfiles/api_v1.py → locustfiles/api_v1.py

@@ -6,7 +6,7 @@
 class QuayUser(HttpUser):
 
     def on_start(self):
-        self.client.headers = {'Authorization': f'Bearer {os.environ["AUTH_TOKEN"]}'}
+        self.client.headers = {'Authorization': f'Bearer {os.environ["OAUTH_TOKEN"]}'}
         self.name = fetch_random_user()
 
     def on_stop(self):

locustfiles/run.py

@@ -0,0 +1 @@
+from skopeo_operations import SkopeoUser

locustfiles/skopeo_operations.py

@@ -0,0 +1,37 @@
+from locust import User, task, tag
+from subprocess import run
+
+from utils import trigger_event
+import os
+
+class SkopeoUser(User):
+
+    def on_start(self):
+        pass
+
+    @trigger_event(request_type="skopeo", name="Remove Images")
+    def on_stop(self):
+        """
+            Clear all images from local cache
+        """
+        cmd = f"skopeo rmi --all --force"
+        return run(cmd, shell=True, capture_output=True)
+
+    @task
+    @tag('skopeo_push_image')
+    @trigger_event(request_type="skopeo", name="Image push")
+    def push_image(self):
+        """
+            Pushing image via skopeo.
+        """
+        username = os.environ.get('QUAY_USERNAME', 'admin')
+        password = os.environ['QUAY_PASSWORD']
+        host = os.environ['QUAY_HOST']
+
+        cmd = f"skopeo login -u {username} -p {password} --tls-verify=false {host} "
+        run(cmd, shell=True, capture_output=True)
+
+        cmd = ("skopeo copy --tls-verify=false"
+                "docker://quay.io/alecmerdler/bad-image:critical"
+                f"docker://{host}/{username}/bad-image:critial")
+        run(cmd, shell=True, capture_output=True)

utils.py → locustfiles/utils.py

@@ -10,8 +10,8 @@ def wrapper(self, *args):
             start_time = datetime.datetime.utcnow()
             response = func(self)
             end_time = datetime.datetime.utcnow()
-            if (kwargs['request_type'] == 'podman' and response.returncode == 0) or \
-                    (kwargs['request_type'] != 'podman' and response.status_code == 200):
+            if (kwargs['request_type'] == 'skopeo' and response.returncode == 0) or \
+                    (kwargs['request_type'] != 'skopeo' and response.status_code == 200):
                 events.request_success.fire(
                     request_type=kwargs['request_type'],
                     name=kwargs['name'],
@@ -20,7 +20,7 @@ def wrapper(self, *args):
                     response_length=0
                 )
             else:
-                err = response.stderr if kwargs['request_type'] == 'podman' else response.content
+                err = response.stderr if kwargs['request_type'] == 'skopeo' else response.content
                 events.request_failure.fire(
                     request_type=kwargs['request_type'],
                     name=kwargs['name'],

requirements.txt

@@ -1,82 +1,2 @@
-boto3==1.18.8
-botocore==1.21.8
-cachetools==4.2.2
-certifi==2021.5.30
-charset-normalizer==2.0.3
-click==8.0.1
-ConfigArgParse==1.5.1
-configparser==5.0.2
-confluent-kafka==1.7.0
-contextvars==2.4
-dataclasses==0.6
-dateparser==1.0.0
-dictdiffer==0.9.0
-docutils==0.17.1
-elasticsearch==7.13.4
-Flask==1.1.2
-Flask-BasicAuth==0.2.0
-Flask-Cors==3.0.10
-flent==2.0.1
-gevent==21.1.2
-geventhttpclient==1.4.4
-google-api-core==1.31.1
-google-auth==1.34.0
-googleapis-common-protos==1.53.0
-greenlet==1.1.0
-idna==3.2
-immutables==0.15
-importlib-metadata==4.6.1
-itsdangerous==2.0.1
-Jinja2==3.0.1
-jmespath==0.10.0
-kafka-python==2.0.2
-kubernetes==11.0.0
 locust==1.6.0
-locust-plugins==1.1.0
-lxml==4.6.3
-MarkupSafe==2.0.1
-msgpack==1.0.2
-numpy==1.19.5
-oauthlib==3.1.1
-opencensus==0.7.13
-opencensus-context==0.1.2
-opencensus-ext-azure==1.0.8
-openshift==0.11.0
-packaging==21.0
-pandas==1.1.5
-prometheus-api-client==0.4.2
-protobuf==3.17.3
-psutil==5.8.0
-psycogreen==1.0.2
-psycopg2-binary==2.9.1
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pymongo==3.12.0
-pyparsing==2.4.7
-python-dateutil==2.8.2
-python-string-utils==1.0.0
-pytz==2021.1
-PyYAML==5.4.1
-pyzmq==22.1.0
-redis==3.5.3
-regex==2021.7.6
-requests==2.26.0
-requests-oauthlib==1.3.0
-rsa==4.7.2
-ruamel.yaml==0.17.10
-ruamel.yaml.clib==0.2.6
-s3transfer==0.5.0
-scipy==1.5.4
-selenium==3.141.0
-six==1.16.0
-snafu==0.0.1
-statistics==1.0.3.5
-ttp==0.7.2
-typing-extensions==3.10.0.0
-tzlocal==2.1
-urllib3==1.26.6
-websocket-client==1.1.0
-Werkzeug==2.0.1
-zipp==3.5.0
-zope.event==4.5.0
-zope.interface==5.4.0
+