From 516109fcb056bf5cf9c97e70c1bc217ee9ffc4f2 Mon Sep 17 00:00:00 2001 From: Andrew Repp Date: Fri, 9 Dec 2022 09:03:12 -0600 Subject: [PATCH 01/44] Refactor gpbackup CI * Rework and rename "master" and "scale-master" pipeline jobs to properly support GPDB7. * Edit lock_tests.go to look for `COPY %s` instead of `COPY %s TO PROGRAM` to get around the fact that we added `(coldefs)` to our copy calls. * Edit end to end test to exclude schemaNameFilter to prevent catalog table stats changes from causing the test to fail sometimes. * Fixed remote_test.go. It was relying on coincidental backupConfig field-setting by other tests, and missing resize-cluster coverage. * Add a regression pipeline to DP concourse. This tests that we can backup and restore the ICW regression database on both GPDB6 and GPDB7. * Decrease end_to_end resize-cluster tests hang timeout to 5 minutes. * Update Makefile invocation of end_to_end tests to a 4 hour timeout. * Remove scale jobs from release pipeline. They were already not being invoked, so they should not be present. * Pend a test of function transforms in GPDB7, due to inability to compile and install the required extension DLL in our CI. --- Makefile | 2 +- ci/gpbackup-generated.yml | 76 ++-- ci/gpbackup-release-generated.yml | 182 ++------ ci/regression/README.md | 13 + ci/regression/regression_pipeline.yml | 453 +++++++++++++++++++ ci/scripts/icw-roundtrip.bash | 69 +++ ci/scripts/test-on-local-cluster.bash | 11 +- ci/tasks/icw-roundtrip.yml | 20 + ci/templates/gpbackup-tpl.yml | 94 ++-- end_to_end/end_to_end_suite_test.go | 12 +- end_to_end/locks_test.go | 12 +- integration/predata_functions_create_test.go | 4 +- integration/predata_types_queries_test.go | 4 +- restore/remote_test.go | 43 +- 14 files changed, 773 insertions(+), 222 deletions(-) create mode 100644 ci/regression/README.md create mode 100644 ci/regression/regression_pipeline.yml create mode 100755 ci/scripts/icw-roundtrip.bash create mode 100644 ci/tasks/icw-roundtrip.yml diff --git a/Makefile b/Makefile index 7f3dcf652..9317f5056 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ integration : $(GINKGO) test : build unit integration end_to_end : $(GINKGO) - ginkgo $(GINKGO_FLAGS) --slow-spec-threshold=10s end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1 + ginkgo $(GINKGO_FLAGS) --timeout=3h --slow-spec-threshold=10s end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1 coverage : @./show_coverage.sh diff --git a/ci/gpbackup-generated.yml b/ci/gpbackup-generated.yml index bda2d796a..347207a0e 100644 --- a/ci/gpbackup-generated.yml +++ b/ci/gpbackup-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2022-11-02 09:43:34.463191 +## Generated by gen_pipeline.py at: 2022-12-19 13:10:04.194093 ## Template file: gpbackup-tpl.yml ## Pipeline Name: gpbackup ## Nightly Trigger: True @@ -30,28 +30,34 @@ groups: - GPDB6-ubuntu - ddboost_plugin_and_boostfs_tests_43 - ddboost_plugin_and_boostfs_tests_5x + - scale-43 + - scale-5x + - scale-6x + - scale-GPDB7 + - s3_plugin_perf + - ddboost_plugin_perf - gpbackup-manager-tests - 5X-head-gpbackup-fixed-test - GPDB6 - GPDB6-7-seg-cluster + - GPDB7 - s3_plugin_tests - backward-compatibility - ddboost_plugin_and_boostfs_tests_6x - - master - final_gate - name: GPDB4.3 jobs: - GPDB4.3 - - scale-43 - ddboost_plugin_and_boostfs_tests_43 + - scale-43 - name: GPDB5 jobs: - GPDB5 - GPDB5-sles11 - - scale-5x - ddboost_plugin_and_boostfs_tests_5x + - scale-5x - 5X-head-gpbackup-fixed-test - s3_plugin_tests - backward-compatibility @@ -66,19 +72,19 @@ groups: - s3_plugin_perf - ddboost_plugin_perf -- name: Master +- name: GPDB7 jobs: - - master - - scale-master + - GPDB7 + - scale-GPDB7 - name: Scale jobs: - scale-43 - scale-5x - scale-6x + - scale-GPDB7 - s3_plugin_perf - ddboost_plugin_perf - - scale-master ##### Anchors ##### @@ -218,6 +224,12 @@ resources: repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test tag: latest +- name: centos7-gpdb7-image + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb7-centos7-test + tag: latest + - name: centos6-image type: registry-image source: @@ -329,14 +341,22 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64((dp/prod/rc-build-type-gcs)).tar.gz -- name: gpdb_master_src +- name: bin_gpdb_rhel8 + type: gcs + icon: google + source: + bucket: ((dp/prod/gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.tar.gz + +- name: gpdb_main_src type: git icon: github-circle source: uri: https://github.com/greenplum-db/gpdb branch: main -- name: bin_gpdb_master_centos7 +- name: bin_gpdb_main_centos7 type: gcs icon: google source: @@ -395,7 +415,7 @@ resources: json_key: ((dp/prod/gcp_svc_acct_key)) regexp: dummy_seclabel_gpdb6-v(.*).so -- name: dummy_seclabel_linux_master +- name: dummy_seclabel_linux_main type: gcs icon: google source: @@ -1071,28 +1091,28 @@ jobs: ensure: <<: *set_failed -- name: master +- name: GPDB7 plan: - in_parallel: - - get: centos7-image + - get: centos7-gpdb7-image - get: gpbackup passed: [build_gppkgs] - - get: bin_gpdb_master_centos7 + - get: bin_gpdb_main_centos7 trigger: true - get: gpdb_src - resource: gpdb_master_src + resource: gpdb_main_src - get: dummy_seclabel - resource: dummy_seclabel_linux_master + resource: dummy_seclabel_linux_main - get: gppkgs trigger: true passed: [build_gppkgs] - task: run-tests-locally-centos7 - image: centos7-image + image: centos7-gpdb7-image file: gpbackup/ci/tasks/test-on-local-cluster.yml params: REQUIRES_DUMMY_SEC: true input_mapping: - bin_gpdb: bin_gpdb_master_centos7 + bin_gpdb: bin_gpdb_main_centos7 on_failure: *slack_alert @@ -1311,17 +1331,17 @@ jobs: params: GPBACKUP_VERSION: "1.12.1" -- name: scale-master +- name: scale-GPDB7 plan: - in_parallel: - - get: centos7-image + - get: centos7-gpdb7-image - get: gpbackup passed: [build_gppkgs] - get: gpdb_binary - resource: bin_gpdb_master_centos7 + resource: bin_gpdb_rhel8 - get: ccp_src - get: gpdb_src - resource: gpdb_master_src + resource: gpdb_main_src - get: scale_schema - get: nightly-trigger trigger: true @@ -1336,19 +1356,21 @@ jobs: terraform_source: ccp_src/google-nvme-block-device/ vars: instance_type: n1-standard-8 - PLATFORM: centos7 + PLATFORM: rhel8-gpdb7 - task: gen_cluster params: <<: *ccp_gen_cluster_default_params - PLATFORM: centos7 + PLATFORM: rhel8-gpdb7 file: ccp_src/ci/tasks/gen_cluster.yml - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup-centos-env - image: centos7-image + image: centos7-gpdb7-image + params: + default_ami_user: rhel file: gpbackup/ci/tasks/setup-centos-env.yml - task: scale-tests - image: centos7-image + image: centos7-gpdb7-image file: gpbackup/ci/tasks/scale-tests.yml on_success: <<: *ccp_destroy_nvme @@ -1502,7 +1524,7 @@ jobs: - GPDB4.3 - GPDB5 - GPDB6 - - master + - GPDB7 - s3_plugin_tests - backward-compatibility - 5X-head-gpbackup-fixed-test diff --git a/ci/gpbackup-release-generated.yml b/ci/gpbackup-release-generated.yml index 327faee3d..a4d30d326 100644 --- a/ci/gpbackup-release-generated.yml +++ b/ci/gpbackup-release-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2022-11-02 09:43:34.479435 +## Generated by gen_pipeline.py at: 2022-12-19 13:10:04.211319 ## Template file: gpbackup-tpl.yml ## Pipeline Name: gpbackup-release ## Nightly Trigger: True @@ -33,6 +33,7 @@ groups: - gpbackup-manager-tests - GPDB6 - GPDB6-7-seg-cluster + - GPDB7 - s3_plugin_tests - backward-compatibility - ddboost_plugin_and_boostfs_tests_6x @@ -43,14 +44,12 @@ groups: - name: GPDB4.3 jobs: - GPDB4.3 - - scale-43 - ddboost_plugin_and_boostfs_tests_43 - name: GPDB5 jobs: - GPDB5 - GPDB5-sles11 - - scale-5x - ddboost_plugin_and_boostfs_tests_5x - s3_plugin_tests - backward-compatibility @@ -60,19 +59,14 @@ groups: - GPDB6 - GPDB6-7-seg-cluster - GPDB6-ubuntu - - scale-6x - ddboost_plugin_and_boostfs_tests_6x - s3_plugin_perf - ddboost_plugin_perf - -- name: Scale +- name: GPDB7 jobs: - - scale-43 - - scale-5x - - scale-6x - - s3_plugin_perf - - ddboost_plugin_perf + - GPDB7 + - name: Packaging and Release jobs: @@ -218,6 +212,12 @@ resources: repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test tag: latest +- name: centos7-gpdb7-image + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb7-centos7-test + tag: latest + - name: centos6-image type: registry-image source: @@ -316,6 +316,14 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64((dp/prod/rc-build-type-gcs)).tar.gz +- name: bin_gpdb_rhel8 + type: gcs + icon: google + source: + bucket: ((dp/prod/gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.tar.gz + # These binaries are used for backwards-compatibility testing only - name: bin_gpbackup_1.0.0_and_1.7.1 @@ -1072,6 +1080,30 @@ jobs: ensure: <<: *set_failed +- name: GPDB7 + plan: + - in_parallel: + - get: centos7-gpdb7-image + - get: gpbackup + passed: [build_gppkgs] + - get: bin_gpdb_main_centos7 + trigger: true + - get: gpdb_src + resource: gpdb_main_src + - get: dummy_seclabel + resource: dummy_seclabel_linux_main + - get: gppkgs + trigger: true + passed: [build_gppkgs] + - task: run-tests-locally-centos7 + image: centos7-gpdb7-image + file: gpbackup/ci/tasks/test-on-local-cluster.yml + params: + REQUIRES_DUMMY_SEC: true + input_mapping: + bin_gpdb: bin_gpdb_main_centos7 + on_failure: + *slack_alert - name: GPDB4.3 plan: @@ -1262,134 +1294,6 @@ jobs: -- name: scale-6x - plan: - - in_parallel: - - get: centos7-image - - get: gpbackup - passed: [build_gppkgs] - - get: gpdb_binary - resource: bin_gpdb_6x_stable_centos7 - - get: ccp_src - - get: gpdb_src - resource: gpdb6_src - - get: scale_schema - - get: gppkgs - passed: [build_gppkgs] - - get: terraform.d - params: - unpack: true - - put: terraform - params: - <<: *ccp_default_params - terraform_source: ccp_src/google-nvme-block-device/ - vars: - instance_type: n1-standard-8 - PLATFORM: centos7 - - task: gen_cluster - params: - <<: *ccp_gen_cluster_default_params - PLATFORM: centos7 - file: ccp_src/ci/tasks/gen_cluster.yml - - task: gpinitsystem - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - image: centos7-image - file: gpbackup/ci/tasks/setup-centos-env.yml - - task: scale-tests - image: centos7-image - file: gpbackup/ci/tasks/scale-tests.yml - on_success: - <<: *ccp_destroy_nvme - on_failure: - *slack_alert - ensure: - <<: *set_failed - -- name: scale-5x - plan: - - in_parallel: - - get: centos6-image - - get: gpbackup - passed: [build_gppkgs] - - get: gpdb_binary - resource: bin_gpdb_5x_stable_centos6 - - get: ccp_src - - get: gpdb_src - resource: gpdb5_src - - get: scale_schema - - get: gppkgs - passed: [build_gppkgs] - - get: terraform.d - params: - unpack: true - - put: terraform - params: - <<: *ccp_default_params - terraform_source: ccp_src/google-nvme-block-device/ - vars: - instance_type: n1-standard-8 - - task: gen_cluster - params: - <<: *ccp_gen_cluster_default_params - file: ccp_src/ci/tasks/gen_cluster.yml - - task: gpinitsystem - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - image: centos6-image - file: gpbackup/ci/tasks/setup-centos-env.yml - - task: scale-tests - image: centos6-image - file: gpbackup/ci/tasks/scale-tests.yml - on_success: - <<: *ccp_destroy_nvme - on_failure: - *slack_alert - ensure: - <<: *set_failed - -- name: scale-43 - plan: - - in_parallel: - - get: centos6-image - - get: gpbackup - passed: [build_gppkgs] - - get: gpdb_binary - resource: bin_gpdb_43_stable - - get: ccp_src - - get: gpdb_src - resource: gpdb5_src - - get: scale_schema - - get: gppkgs - passed: [build_gppkgs] - - get: terraform.d - params: - unpack: true - - put: terraform - params: - <<: *ccp_default_params - terraform_source: ccp_src/google-nvme-block-device/ - vars: - instance_type: n1-standard-8 - - task: gen_cluster - params: - <<: *ccp_gen_cluster_default_params - file: ccp_src/ci/tasks/gen_cluster.yml - - task: gpinitsystem - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - image: centos6-image - file: gpbackup/ci/tasks/setup-centos-env.yml - - task: scale-tests - image: centos6-image - file: gpbackup/ci/tasks/scale-tests.yml - on_success: - <<: *ccp_destroy_nvme - on_failure: - *slack_alert - ensure: - <<: *set_failed - - name: green_gate plan: - in_parallel: diff --git a/ci/regression/README.md b/ci/regression/README.md new file mode 100644 index 000000000..19fe34976 --- /dev/null +++ b/ci/regression/README.md @@ -0,0 +1,13 @@ +# Regression Testing +One of the core tests in gpdb is the `make installcheck-world` invocation in the main `gpdb` repo. This invocation runs a series of tests, including regression tests, that ensure the functionality of gpdb. +These tests leave behind many artifacts in a handful of databases, the most important of which is `regression`. This `regression` database represents a large cross-section of database objects that gpdbd +supports. Testing support of this collection of objects is a standard method used by the gpdb utilities. For `gpbackup` we want to ensure that we can correcly back up and restore a database containing +all of these objects. The `regression` pipeline automates that process. + +## Data Source +We use a dump of the `regression` database generated by the main gpdb pipelines, stored in a GCS bucket. This dump is generated by `pg_dumpall`, and loaded into our test database using `psql` +We use standalone psql commands to clean out a small number of objects with known bugs, linking to the relevant gpdb github issue. + +## Tests +We use a standard gpbackup invocation to save off the database, and an immediate gprestore invocation of that backup to ensure that both utilities correctly complete their functions. +We use a [simple golang utility](https://github.com/AJR-VMware/diffdb) to check the table counts, and the rowcounts of each table, across the backed up and restored databases to ensure that they match. diff --git a/ci/regression/regression_pipeline.yml b/ci/regression/regression_pipeline.yml new file mode 100644 index 000000000..dde0f3d45 --- /dev/null +++ b/ci/regression/regression_pipeline.yml @@ -0,0 +1,453 @@ +# USAGE: fly -t dp set-pipeline -p regression -c ~/workspace/gpbackup/ci/regression/regression_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME +--- +groups: +- name: Regression + jobs: + - build_binaries + - build_gppkgs + - regdb-GPDB6 + - regdb-GPDB7 + +resource_types: +- name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + tag: 0.11.14 + +- name: slack-notification + type: registry-image + source: + repository: cfcommunity/slack-notification-resource + tag: latest + +- name: gcs + type: registry-image + source: + repository: frodenas/gcs-resource + +- name: pivnet + type: registry-image + source: + repository: pivotalcf/pivnet-resource + tag: latest-final + +## ====================================================================== +## _ __ ___ ___ ___ _ _ _ __ ___ ___ ___ +## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __| +## | | | __/\__ \ (_) | |_| | | | (_| __/\__ \ +## |_| \___||___/\___/ \__,_|_| \___\___||___/ +## ====================================================================== + +resources: +##### Docker Images ##### +- name: centos7-image + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test + tag: latest + +- name: rocky8-gpdb7-image + type: registry-image + source: + repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test + tag: latest + +##### Source Code #### +- name: gpbackup + type: git + icon: github-circle + source: + uri: https://github.com/greenplum-db/gpbackup + branch: ((gpbackup-git-branch)) + +- name: diffdb_src + type: git + icon: github-circle + source: + uri: https://github.com/AJR-VMware/diffdb + branch: main + +- name: gpdb6_src + type: git + icon: github-circle + source: + uri: https://github.com/greenplum-db/gpdb + branch: 6X_STABLE + +- name: gpdb_main_src + type: git + icon: github-circle + source: + uri: https://github.com/greenplum-db/gpdb + branch: main + +- name: gpbackup_manager_src + type: git + icon: github-circle + source: + branch: main + private_key: ((dp/dev/gp-backup-manager-remote-deploy-key)) + uri: ((dp/gp-backup-manager-git-remote)) + +- name: gpbackup_s3_plugin + type: git + icon: github-circle + source: + branch: ((dp/dev/gpbackup-s3-plugin-git-branch)) + uri: https://github.com/greenplum-db/gpbackup-s3-plugin + +- name: gpbackup_ddboost_plugin + type: git + icon: github-circle + source: + branch: ((dp/gpbackup-ddboost-plugin-branch)) + private_key: ((dp/gpbackup-ddboost-plugin-remote-key)) + uri: ((dp/gpbackup-ddboost-plugin-git-remote)) + +#### Binaries #### +- name: bin_gpdb_6x_stable_centos7 + type: gcs + icon: google + source: + bucket: ((dp/prod/gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64((dp/dev/rc-build-type-gcs)).tar.gz + +- name: bin_gpdb_rhel8 + type: gcs + icon: google + source: + bucket: ((dp/prod/gcs-bucket)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.tar.gz + +#### Misc Other #### +- name: gpbackup-release-license + type: gcs + icon: google + source: + bucket: gpbackup-release-licenses + json_key: ((dp/dev/gcp_svc_acct_key)) + regexp: open_source_license_VMware_Tanzu_Greenplum_Backup_and_Restore_(.*)_.*.txt + +- name: icw_dump_GPDB6 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-intermediates-prod + json_key: ((concourse-gcs-resources-service-account-key)) + versioned_file: 6X_STABLE_without_asserts/icw_planner_centos6_dump/dump.sql.xz + +- name: icw_dump_GPDB7 + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-intermediates-prod + json_key: ((concourse-gcs-resources-service-account-key)) + versioned_file: gpdb_main/icw_planner_rhel8_dump/dump.sql.xz + +- name: pivnet_release_cache + type: s3 + icon: amazon + source: + access_key_id: ((aws-bucket-access-key-id)) + bucket: ((dp/dev/pivnet_bucket_name)) + region_name: ((dp/aws-region)) + secret_access_key: ((aws-bucket-secret-access-key)) + regexp: pivnet_release_version/v-(.*) + +- name: terraform + type: terraform + source: + env: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) + vars: + project_id: ((dp/dev/google-project-id)) + storage: + access_key_id: ((tf-machine-access-key-id)) + secret_access_key: ((tf-machine-secret-access-key)) + region_name: ((dp/aws-region)) + bucket: ((dp/tf-bucket-name)) + bucket_path: clusters-google/ + +# - name: slack-alert +# type: slack-notification +# source: +# url: ((dp/webhook_url)) +# disable: false + +- name: gpbackup-go-components + type: s3 + icon: amazon + source: + access_key_id: ((aws-bucket-access-key-id)) + bucket: ((dp/dev/gpdb-stable-bucket-name)) + region_name: ((dp/aws-region)) + secret_access_key: ((aws-bucket-secret-access-key)) + versioned_file: gpbackup-go-components/go_components.tar.gz + +- name: gppkgs + type: s3 + icon: amazon + source: + access_key_id: ((aws-bucket-access-key-id)) + bucket: ((dp/dev/gpdb-stable-bucket-name)) + region_name: ((dp/aws-region)) + secret_access_key: ((aws-bucket-secret-access-key)) + versioned_file: gppkgs/intermediates/gpbackup-gppkgs.tar.gz + +- name: ccp_src + type: git + icon: github-circle + source: + branch: ((dp/ccp-git-branch)) + private_key: ((gp-concourse-cluster-provisioner-git-key)) + uri: ((dp/ccp-git-remote)) + +- name: terraform.d + type: s3 + icon: amazon + source: + access_key_id: ((aws-bucket-access-key-id)) + bucket: ccp-terraform-provider-plugins + region_name: ((dp/aws-region)) + secret_access_key: ((aws-bucket-secret-access-key)) + versioned_file: plugin-cache-all.tgz + +## ====================================================================== +## _ +## __ _ _ __ ___| |__ ___ _ __ ___ +## / _` | '_ \ / __| '_ \ / _ \| '__/ __| +## | (_| | | | | (__| | | | (_) | | \__ \ +## \__,_|_| |_|\___|_| |_|\___/|_| |___/ +## ====================================================================== + +anchors: +- &ccp_default_params + action: create + delete_on_failure: true + generate_random_name: true + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 + terraform_source: ccp_src/google/ + +- &ccp_gen_cluster_default_params + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((dp/aws-region)) + BUCKET_PATH: clusters-google/ + BUCKET_NAME: ((dp/tf-bucket-name)) + CLOUD_PROVIDER: google + +- &destroy_params + action: destroy + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 + terraform_source: ccp_src/google/ + env_name_file: terraform/name + vars: + aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. + aws_ebs_volume_type: standard + +- &ccp_destroy + put: terraform + params: + <<: *destroy_params + get_params: + action: destroy + +- &ccp_destroy_nvme + put: terraform + params: + <<: *destroy_params + terraform_source: ccp_src/google-nvme-block-device/ + get_params: + action: destroy + +- &terraform_cluster + type: terraform + source: + env: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) + vars: + project_id: ((dp/dev/google-service-account)) + storage: + access_key_id: ((tf-machine-access-key-id)) + secret_access_key: ((tf-machine-secret-access-key)) + region_name: ((dp/aws-region)) + # This is not parameterized, on purpose. All tfstates will go to this spot, + # and different teams will place there clusters' tfstate files under different paths + bucket: gpdb5-pipeline-dynamic-terraform + bucket_path: clusters-google/ + + +- &slack_alert + put: slack-alert + params: + text: | + [gpbackup/$BUILD_JOB_NAME] failed: + https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME + +## ====================================================================== +## _ _ +## (_) ___ | |__ ___ +## | |/ _ \| '_ \/ __| +## | | (_) | |_) \__ \ +## _/ |\___/|_.__/|___/ +## |__/ +## ====================================================================== + +jobs: +- name: build_binaries + plan: + - in_parallel: + - get: gpbackup_s3_plugin + trigger: true + - get: gpbackup_manager_src + trigger: true + - get: gpbackup + trigger: true + - get: pivnet_release_cache + - task: build-go-binaries + file: gpbackup/ci/tasks/build-go-binaries.yml + - put: gpbackup-go-components + params: + file: go_components/go_components.tar.gz + +- name: build_gppkgs + plan: + - in_parallel: + - get: centos7-image + - get: gpdb_src + resource: gpdb6_src + - get: gpbackup-go-components + trigger: true + passed: [build_binaries] + - get: bin_gpdb_6x_stable_centos7 + - get: gpbackup + passed: [build_binaries] + - get: gpbackup_ddboost_plugin + trigger: true + - get: pivnet_release_cache + - get: gpbackup-release-license + - task: gpbackup-tools-versions + image: centos7-image + file: gpbackup/ci/tasks/gpbackup-tools-versions.yml + - in_parallel: + - do: + - task: build-ddboost-RHEL + image: centos7-image + file: gpbackup/ci/tasks/build-ddboost.yml + input_mapping: + bin_gpdb: bin_gpdb_6x_stable_centos7 + - task: tar-binaries-RHEL + image: centos7-image + file: gpbackup/ci/tasks/build-os-tars.yml + - task: build_gppkgs-RHEL + image: centos7-image + file: gpbackup/ci/tasks/build-gppkg.yml + input_mapping: + bin_gpdb: bin_gpdb_6x_stable_centos7 + output_mapping: + gppkgs: rhel-gppkg + params: + OS: RHEL + - task: tar-gppkgs + image: centos7-image + file: gpbackup/ci/tasks/tar-gppkgs.yml + - put: gppkgs + params: + file: gppkgs/gpbackup-gppkgs.tar.gz + +- name: regdb-GPDB6 + plan: + - in_parallel: + - get: centos7-image + - get: gpbackup + passed: [build_gppkgs] + - get: diffdb_src + - get: gpdb_binary + resource: bin_gpdb_6x_stable_centos7 + - get: ccp_src + - get: gpdb_src + resource: gpdb6_src + - get: gppkgs + trigger: true + passed: [build_gppkgs] + - get: icw_dump + resource: icw_dump_GPDB6 + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos7 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos7 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + image: centos7-image + file: gpbackup/ci/tasks/setup-centos-env.yml + - task: icw-roundtrip + image: centos7-image + file: gpbackup/ci/tasks/icw-roundtrip.yml + on_success: + <<: *ccp_destroy_nvme + on_failure: + *slack_alert + +- name: regdb-GPDB7 + plan: + - in_parallel: + - get: rocky8-gpdb7-image + - get: gpbackup + passed: [build_gppkgs] + - get: gpdb_binary + resource: bin_gpdb_rhel8 + - get: ccp_src + - get: gpdb_src + resource: gpdb_main_src + - get: gppkgs + trigger: true + passed: [build_gppkgs] + - get: diffdb_src + - get: icw_dump + resource: icw_dump_GPDB7 + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: rhel8-gpdb7 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: rhel8-gpdb7 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + image: rocky8-gpdb7-image + file: gpbackup/ci/tasks/setup-centos-env.yml + params: + default_ami_user: rhel + - task: icw-roundtrip + image: rocky8-gpdb7-image + file: gpbackup/ci/tasks/icw-roundtrip.yml + on_success: + <<: *ccp_destroy_nvme + on_failure: + *slack_alert diff --git a/ci/scripts/icw-roundtrip.bash b/ci/scripts/icw-roundtrip.bash new file mode 100755 index 000000000..80f198220 --- /dev/null +++ b/ci/scripts/icw-roundtrip.bash @@ -0,0 +1,69 @@ +#!/bin/bash + +set -ex + +# setup cluster and install gpbackup tools using gppkg +ccp_src/scripts/setup_ssh_to_cluster.sh +out=$(ssh -t mdw 'source env.sh && psql postgres -c "select version();"') +GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p') +mkdir -p /tmp/untarred +tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred +scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg mdw:/home/gpadmin +scp ./icw_dump/dump.sql.xz mdw:/home/gpadmin + +pushd ./diffdb_src + go build + scp ./diffdb mdw:/home/gpadmin/ +popd + +cat <