From 317c75504f2ba43ba51e655c4f0f1182f825d821 Mon Sep 17 00:00:00 2001 From: Sean Zhao Date: Sun, 29 Sep 2024 16:21:32 +0800 Subject: [PATCH 1/3] registry stage proxy --- Dockerfile | 13 ++++++-- config.py | 6 ++-- main.py | 96 ++++++++++++++++++++++++++++++++++-------------------- 3 files changed, 75 insertions(+), 40 deletions(-) diff --git a/Dockerfile b/Dockerfile index 54766d2..67a44c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,13 @@ WORKDIR /tmp ARG DEBIAN_FRONTEND=noninteractive # Install necessary libraries for subsequent commands -RUN apt-get update && apt-get install -y podman wget git dumb-init python3.6 python3-distutils python3-pip python3-apt redis-server +RUN apt-get update && \ + apt-get install -y software-properties-common python3.6 python3-venv python3-pip python3-apt wget git dumb-init podman redis-server + +# Create and activate virtual environment +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + # Install vegeta for HTTP benchmarking RUN wget https://github.com/tsenart/vegeta/releases/download/v12.8.3/vegeta-12.8.3-linux-amd64.tar.gz \ && tar -xzf vegeta-12.8.3-linux-amd64.tar.gz \ @@ -17,8 +23,9 @@ RUN wget https://github.com/tsenart/vegeta/releases/download/v12.8.3/vegeta-12.8 RUN mkdir -p /opt/snafu/ \ && wget -O /tmp/benchmark-wrapper.tar.gz https://github.com/cloud-bulldozer/benchmark-wrapper/archive/refs/tags/v1.0.0.tar.gz \ && tar -xzf /tmp/benchmark-wrapper.tar.gz -C /opt/snafu/ --strip-components=1 \ - && pip3 install --upgrade pip \ - && pip3 install -e /opt/snafu/ \ + && pip install --upgrade pip \ + && pip install -e /opt/snafu/ \ + && pip install "numpy<2" \ && rm -rf /tmp/benchmark-wrapper.tar.gz COPY . . diff --git a/config.py b/config.py index 25b987a..fad5fec 100644 --- a/config.py +++ b/config.py @@ -34,7 +34,9 @@ def get_config(self): 'batch_size': int(os.environ.get('TEST_BATCH_SIZE', 400)), 'test_namespace': os.environ.get("TEST_NAMESPACE"), 'base_url': '%s://%s' % ("https", os.environ.get("QUAY_HOST")), - 'test_phases': os.environ.get('TEST_PHASES') + 'test_phases': os.environ.get('TEST_PHASES'), + 'tags': os.environ.get('TAGS'), + 'skip_push': os.environ.get('SKIP_PUSH') } self.validate_config() return self.config @@ -59,4 +61,4 @@ def validate_config(self): assert isinstance(self.config["batch_size"], int), "BATCH_SIZE is not an integer" assert self.config["test_namespace"], "TEST_NAMESPACE is not set" assert self.config["base_url"], "BASE_URL is not set" - assert self.config["test_phases"], "TEST_PHASES are not set. Valid options are LOAD,RUN and DELETE" + assert self.config["test_phases"], "TEST_PHASES are not set. Valid options are LOAD,RUN, PUSH_PULL and DELETE" diff --git a/main.py b/main.py index 29524d5..fe19375 100644 --- a/main.py +++ b/main.py @@ -494,7 +494,7 @@ def create_test_pull_job(namespace, quay_host, username, password, concurrency, ) template = client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels={'quay-perf-test-component-pull': 'executor-'+"-".join(username.split("_"))}), + metadata=client.V1ObjectMeta(labels={'quay-perf-test-component-pull': 'executor-'+"-".join(username.split("_")).replace("|","")}), spec=client.V1PodSpec(restart_policy='Never', containers=[container]) ) @@ -504,7 +504,7 @@ def create_test_pull_job(namespace, quay_host, username, password, concurrency, job = client.V1Job( api_version="batch/v1", kind="Job", - metadata=client.V1ObjectMeta(name="test-registry-pull"+"-".join(username.split("_"))), + metadata=client.V1ObjectMeta(name="test-registry-pull"+"-".join(username.split("_")).replace("|","")), spec=spec ) @@ -534,6 +534,9 @@ def parallel_process(user, **kwargs): """ common_args = kwargs # Container Operations + + logging.info('*** parallel_process user: %s', user) + redis_client.delete('tags_to_push'+"-".join(user.split("_"))) # avoid stale data redis_client.rpush('tags_to_push'+"-".join(user.split("_")), *common_args['tags']) logging.info('Queued %s tags to be created' % len(common_args['tags'])) @@ -543,24 +546,25 @@ def parallel_process(user, **kwargs): logging.info('Queued %s tags to be pulled' % len(common_args['tags'])) # Start the Registry Push Test job - create_test_push_job(common_args['namespace'], common_args['quay_host'], user, - common_args['password'], common_args['concurrency'], common_args['uuid'], common_args['auth_token'], - common_args['batch_size'], len(common_args['tags']), common_args['push_pull_image'], common_args['target_hit_size']) - time.sleep(60) # Give the Job time to start - while True: - # Check Job Status - job_name = 'test-registry-push'+"-".join(user.split("_")) - job_api = client.BatchV1Api() - resp = job_api.read_namespaced_job_status(name=job_name, namespace=common_args['namespace']) - completion_time = resp.status.completion_time - if completion_time: - logging.info("Job %s has been completed." % (job_name)) - break + if common_args['skip_push'] != "true": + create_test_push_job(common_args['namespace'], common_args['quay_host'], user, + common_args['password'], common_args['concurrency'], common_args['uuid'], common_args['auth_token'], + common_args['batch_size'], len(common_args['tags']), common_args['push_pull_image'], common_args['target_hit_size']) + time.sleep(60) # Give the Job time to start + while True: + # Check Job Status + job_name = 'test-registry-push'+"-".join(user.split("_")) + job_api = client.BatchV1Api() + resp = job_api.read_namespaced_job_status(name=job_name, namespace=common_args['namespace']) + completion_time = resp.status.completion_time + if completion_time: + logging.info("Job %s has been completed." % (job_name)) + break - # Log Queue Status - remaining = redis_client.llen('tags_to_push'+"-".join(user.split("_"))) - logging.info('Waiting for %s to finish. Queue: %s/%s' % (job_name, remaining, len(common_args['tags']))) - time.sleep(60 * 1) # 1 minute + # Log Queue Status + remaining = redis_client.llen('tags_to_push'+"-".join(user.split("_"))) + logging.info('Waiting for %s to finish. Queue: %s/%s' % (job_name, remaining, len(common_args['tags']))) + time.sleep(60 * 1) # 1 minute # Start the Registry Pull Test job create_test_pull_job(common_args['namespace'], common_args['quay_host'], user, @@ -570,7 +574,7 @@ def parallel_process(user, **kwargs): while True: # Check Job Status - job_name = 'test-registry-pull'+"-".join(user.split("_")) + job_name = 'test-registry-pull'+"-".join(user.split("_")).replace("|","") job_api = client.BatchV1Api() resp = job_api.read_namespaced_job_status(name=job_name, namespace=common_args['namespace']) completion_time = resp.status.completion_time @@ -657,6 +661,15 @@ def batch_process(users_chunk, batch_args): ] tags.extend(repo_tags) + explicit_tags = env_config["tags"].split(",") + if len(explicit_tags) > 0: + tags = [] + logging.info("explicit tags: %s", explicit_tags) + for i in range(5000): + for tag in explicit_tags: + tags.append(tag) + logging.info("final tags num: %s", len(tags)) + print_header( 'Running Quay Scale & Performance Tests', date=datetime.datetime.utcnow().isoformat(), @@ -671,14 +684,38 @@ def batch_process(users_chunk, batch_args): repos_with_tags_sizes=repo_sizes, total_tags=len(tags), pull_push_batch_size=env_config["batch_size"], - number_of_push_pull_jobs_per_user=len(tags)//env_config["batch_size"], ) namespace = env_config["test_namespace"] - if not ({'load', 'run', 'delete'} & set(phases_list)): - logging.info("No valid phases defined to run the tests. Valid options: LOAD, RUN and DELETE") + if not ({'load', 'run', 'delete', 'push_pull'} & set(phases_list)): + logging.info("No valid phases defined to run the tests. Valid options: LOAD, RUN, PUSH_PULL and DELETE") sys.exit() + + batch_args = { + "namespace": namespace, + "quay_host": env_config["quay_host"], + "concurrency": env_config["concurrency"], + "uuid": env_config["test_uuid"], + "auth_token": env_config["auth_token"], + "batch_size": env_config["batch_size"], + "tags": tags, + "push_pull_image": env_config["push_pull_image"], + "target_hit_size": env_config["target_hit_size"], + "skip_push": env_config["skip_push"] + } + + if ('push_pull' in phases_list): + time.sleep(60) + username = os.environ.get('QUAY_USERNAME') + batch_args['password'] = os.environ.get('QUAY_PASSWORD') + start_time = datetime.datetime.utcnow() + logging.info(f"Starting image push/pulls (UTC): {start_time.strftime('%Y-%m-%d %H:%M:%S.%f')}") + logging.info("^^^PULL user: %s",[username]) + batch_process([username], batch_args) + end_time = datetime.datetime.utcnow() + logging.info(f"Ending image push/pulls (UTC): {end_time.strftime('%Y-%m-%d %H:%M:%S.%f')}") + exit(0) # Load Phase # These tests should run before container images are pushed @@ -697,20 +734,9 @@ def batch_process(users_chunk, batch_args): elapsed_time = end_time - start_time logging.info(f"The load phase took {str(datetime.timedelta(seconds=elapsed_time.total_seconds()))}.") - batch_args = { - "namespace": namespace, - "quay_host": env_config["quay_host"], - "password": password, - "concurrency": env_config["concurrency"], - "uuid": env_config["test_uuid"], - "auth_token": env_config["auth_token"], - "batch_size": env_config["batch_size"], - "tags": tags, - "push_pull_image": env_config["push_pull_image"], - "target_hit_size": env_config["target_hit_size"] - } start_time = datetime.datetime.utcnow() logging.info(f"Starting image push/pulls (UTC): {start_time.strftime('%Y-%m-%d %H:%M:%S.%f')}") + batch_args['password'] = password batch_process([users[0]], batch_args) end_time = datetime.datetime.utcnow() logging.info(f"Ending image push/pulls (UTC): {end_time.strftime('%Y-%m-%d %H:%M:%S.%f')}") From bd80d4a6ad7f29bc9b1f19cda96266ded904e8ba Mon Sep 17 00:00:00 2001 From: Sean Zhao Date: Thu, 31 Jul 2025 11:40:28 +0800 Subject: [PATCH 2/3] increase the created image tag size --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index c09024b..77ba7a2 100644 --- a/main.py +++ b/main.py @@ -70,7 +70,8 @@ def podman_create(tags): unique_id = str(uuid.uuid4()) dockerfile = ( "FROM quay.io/jitesoft/alpine\n" - "RUN echo %s > /tmp/key.txt" + "RUN echo %s > /tmp/key.txt\n" + "RUN dd if=/dev/zero of=/tmp/largefile bs=1M count=10" ) % unique_id # Call Podman to build the Dockerfile @@ -778,4 +779,4 @@ def batch_process(users_chunk, batch_args): end_time = datetime.datetime.utcnow() logging.info(f"Ending cleanup phase (UTC): {end_time.strftime('%Y-%m-%d %H:%M:%S.%f')}") elapsed_time = end_time - start_time - logging.info(f"The cleanup phase took {str(datetime.timedelta(seconds=elapsed_time.total_seconds()))}.") + logging.info(f"The cleanup phase took {str(datetime.timedelta(seconds=elapsed_time.total_seconds()))}.") \ No newline at end of file From dc2b6f0c7f771194ae0e1cda4c7086d7429641e8 Mon Sep 17 00:00:00 2001 From: Sean Zhao Date: Mon, 4 Aug 2025 10:36:33 +0800 Subject: [PATCH 3/3] add ephemeral storage --- deploy/test.job.yaml | 3 +++ main.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/deploy/test.job.yaml b/deploy/test.job.yaml index 730ac5c..ff41b86 100644 --- a/deploy/test.job.yaml +++ b/deploy/test.job.yaml @@ -102,6 +102,9 @@ spec: requests: cpu: "1" memory: "512Mi" + ephemeral-storage: "12Gi" + limits: + ephemeral-storage: "15Gi" imagePullPolicy: Always restartPolicy: Never backoffLimit: 0 diff --git a/main.py b/main.py index 77ba7a2..a7fee89 100644 --- a/main.py +++ b/main.py @@ -71,7 +71,9 @@ def podman_create(tags): dockerfile = ( "FROM quay.io/jitesoft/alpine\n" "RUN echo %s > /tmp/key.txt\n" - "RUN dd if=/dev/zero of=/tmp/largefile bs=1M count=10" + "RUN dd if=/dev/zero of=/tmp/file1 bs=1M count=20\n" + "RUN dd if=/dev/zero of=/tmp/file2 bs=1M count=20\n" + "RUN dd if=/dev/zero of=/tmp/file3 bs=1M count=10" ) % unique_id # Call Podman to build the Dockerfile @@ -410,6 +412,10 @@ def create_test_push_job(namespace, quay_host, username, password, concurrency, requests={ 'cpu': '1', 'memory': '512Mi', + 'ephemeral-storage': '12Gi', + }, + limits={ + 'ephemeral-storage': '15Gi', } ) @@ -483,6 +489,10 @@ def create_test_pull_job(namespace, quay_host, username, password, concurrency, requests={ 'cpu': '1', 'memory': '512Mi', + 'ephemeral-storage': '12Gi', + }, + limits={ + 'ephemeral-storage': '15Gi', } )