From f68939e5cea808af11438d1d26a4ebc4685166b4 Mon Sep 17 00:00:00 2001
From: Alex Corvin <acorvin@redhat.com>
Date: Mon, 29 Jan 2024 08:03:32 -0500
Subject: [PATCH] Remove s3 sync pipelines

We are decommissioning this service
---
 s3-sync-pipelines/.sops.yaml                  |   3 -
 s3-sync-pipelines/README.md                   | 116 ------------------
 s3-sync-pipelines/base/kustomization.yaml     |   9 --
 .../s3-sync-eventlistener-rolebinding.yaml    |  11 --
 s3-sync-pipelines/base/s3-sync-pipeline.yaml  |  25 ----
 .../base/s3-sync-serviceaccount.yaml          |   4 -
 s3-sync-pipelines/base/s3-sync-task.yaml      |  38 ------
 .../base/s3-sync-triggertemplate.yaml         |  29 -----
 .../overlays/dev/kustomization.yaml           |   8 --
 ...sync-eventlistener-clusterrolebinding.yaml |  11 --
 .../overlays/prod/kustomization.yaml          |  11 --
 ...sync-eventlistener-clusterrolebinding.yaml |  11 --
 .../secrets/floorist-sync-secret.enc.yaml     |  31 -----
 .../prod/secrets/secret-generator.yaml        |   6 -
 .../prod/sync-instances/floorist.yaml         |  37 ------
 .../prod/sync-instances/kustomization.yaml    |   5 -
 16 files changed, 355 deletions(-)
 delete mode 100644 s3-sync-pipelines/.sops.yaml
 delete mode 100644 s3-sync-pipelines/README.md
 delete mode 100644 s3-sync-pipelines/base/kustomization.yaml
 delete mode 100644 s3-sync-pipelines/base/s3-sync-eventlistener-rolebinding.yaml
 delete mode 100644 s3-sync-pipelines/base/s3-sync-pipeline.yaml
 delete mode 100644 s3-sync-pipelines/base/s3-sync-serviceaccount.yaml
 delete mode 100644 s3-sync-pipelines/base/s3-sync-task.yaml
 delete mode 100644 s3-sync-pipelines/base/s3-sync-triggertemplate.yaml
 delete mode 100644 s3-sync-pipelines/overlays/dev/kustomization.yaml
 delete mode 100644 s3-sync-pipelines/overlays/dev/s3-sync-eventlistener-clusterrolebinding.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/kustomization.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/privileged/s3-sync-eventlistener-clusterrolebinding.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/secrets/floorist-sync-secret.enc.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/secrets/secret-generator.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/sync-instances/floorist.yaml
 delete mode 100644 s3-sync-pipelines/overlays/prod/sync-instances/kustomization.yaml

diff --git a/s3-sync-pipelines/.sops.yaml b/s3-sync-pipelines/.sops.yaml
deleted file mode 100644
index d46207e3..00000000
--- a/s3-sync-pipelines/.sops.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-creation_rules:
-  - encrypted_regex: "^(data|stringData|tls)$"
-    pgp: "EFDB9AFBD18936D9AB6B2EECBD2C73FF891FBC7E"
diff --git a/s3-sync-pipelines/README.md b/s3-sync-pipelines/README.md
deleted file mode 100644
index 3a1e036c..00000000
--- a/s3-sync-pipelines/README.md
+++ /dev/null
@@ -1,116 +0,0 @@
-# S3 Sync Pipelines
-
-Artifacts for Deploying Data Hub s3 sync pipelines.
-
-We use OpenShift Pipelines for orchestrating syncs of data between a source
-and destination bucket. Under the covers, we use the [RClone](https://rclone.org)
-tool to execute the data syncs.
-
-## Transitioning Ownership
-
-As part of the broader Data Hub decommissioning effort, we are working to
-stop maintaining ownership of these sync jobs. Our expectation is that data
-owner teams will take over ownership of their sync jobs. These data owners can
-choose to leverage these existing deployment artifacts for their own use
-and run the sync pipeline in their own cluster/namespace.
-
-The following are instructions for how a data owner can do so. We currently use
-Kustomize with the KSops plugin for managing Kubernetes deployments and
-encrypting secrets in Git. These instructions do not assume use of either
-tool and instead point to specific Kubernetes objects to be deployed. You
-may use the tool of your choice.
-
-1. Make sure you have the OpenShift pipelines operator installed on your
-   OpenShfit cluster.
-2. Copy the Kubernetes objects (everything except for kustomization.yaml) in
-   the [base](base) directory into your own repository.
-2. Find the existing file for your specific sync job in the
-   [overlays/prod/sync-instances](overlays/prod/sync-instances) directory. This
-   file contains the Kubernetes `CronJob` and `Eventlistener` object
-   definitions. Copy these objects into your repository.
-3. Obtain/create the RClone config secret for your sync job. These are stored
-   encrypted in the [overlays/prod/secrets](overlays/prod/secrets) directory.
-   It is probably easiest to work with a member of the Data Hub team to get
-   this secret.
-4. Apply all of the objects into your kubernetes namespace. The sync will
-   execute based on the schedule defined in your `CronJob` object.
-
-
-## Adding a new sync job
-
-Each individual sync job requires the following artifacts:
-
-### RClone Config Secret
-
-A Kubernetes secret containing rclone configuration (including S3 credentials). Example:
-
-```
-apiVersion: v1
-kind: Secret
-metadata:
-    name: CHANGEME
-stringData:
-    rclone.conf: |
-        [source]
-        type = s3
-        provider = AWS
-        env_auth = false
-        access_key_id = CHANGEME
-        secret_access_key = CHANGEME
-        acl = private
-
-        [destination]
-        type = s3
-        provider = Ceph
-        env_auth = false
-        access_key_id = CHANGEME
-        secret_access_key = CHANGEME
-        acl = private
-        endpoint = https://s3.upshift.redhat.com/
-```
-
-### EventListener
-
-A Tekton EventListener to facilitate triggering a sync. Example:
-
-```
-apiVersion: triggers.tekton.dev/v1beta1
-kind: EventListener
-metadata:
-  name: CHANGME
-spec:
-  serviceAccountName: s3-sync
-  triggers:
-    - name: s3-sync
-      bindings:
-        - name: src_bucket
-          value: CHANGEME #The name of the source S3 bucket
-        - name: dest_bucket
-          value: CHANGEME #The name of the destination S3 bucket
-        - name: sync_config_secret_name
-          value: CHANGEME #The name of the rclone config secret
-      template:
-        ref: s3-sync
-```
-
-### Cron Job
-
-A Kubernetes CronJob defining how ofen to run the sync. Example:
-
-```
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: CHANGEME
-spec:
-  schedule: "0 0 * * *"
-  jobTemplate:
-    spec:
-      template:
-        spec:
-          containers:
-          - name: hello
-            image: quay.io/rhn_support_sreber/curl
-            args: ["curl", "-X", "POST", "--data", "{}", "el-CHANGEME-listener:8080"]
-          restartPolicy: Never
-```
diff --git a/s3-sync-pipelines/base/kustomization.yaml b/s3-sync-pipelines/base/kustomization.yaml
deleted file mode 100644
index 5583292c..00000000
--- a/s3-sync-pipelines/base/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - s3-sync-pipeline.yaml
-  - s3-sync-task.yaml
-  - s3-sync-serviceaccount.yaml
-  - s3-sync-eventlistener-rolebinding.yaml
-  - s3-sync-triggertemplate.yaml
diff --git a/s3-sync-pipelines/base/s3-sync-eventlistener-rolebinding.yaml b/s3-sync-pipelines/base/s3-sync-eventlistener-rolebinding.yaml
deleted file mode 100644
index 97e7fdef..00000000
--- a/s3-sync-pipelines/base/s3-sync-eventlistener-rolebinding.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: s3-sync-eventlistener-binding
-subjects:
-  - kind: ServiceAccount
-    name: s3-sync
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: tekton-triggers-eventlistener-roles
diff --git a/s3-sync-pipelines/base/s3-sync-pipeline.yaml b/s3-sync-pipelines/base/s3-sync-pipeline.yaml
deleted file mode 100644
index c26ec34b..00000000
--- a/s3-sync-pipelines/base/s3-sync-pipeline.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Pipeline
-metadata:
-  name: s3-sync
-spec:
-  params:
-    - name: src_bucket
-    - name: dest_bucket
-    - name: rclone_params
-      default: ""
-    - name: sync_config_secret_name
-  tasks:
-    - name: s3-sync
-      timeout: 10h0m0s
-      taskRef:
-        name: s3-sync
-      params:
-        - name: src_bucket
-          value: "$(params.src_bucket)"
-        - name: dest_bucket
-          value: "$(params.dest_bucket)"
-        - name: rclone_params
-          value: "$(params.rclone_params)"
-        - name: sync_config_secret_name
-          value: "$(params.sync_config_secret_name)"
diff --git a/s3-sync-pipelines/base/s3-sync-serviceaccount.yaml b/s3-sync-pipelines/base/s3-sync-serviceaccount.yaml
deleted file mode 100644
index 01a1416a..00000000
--- a/s3-sync-pipelines/base/s3-sync-serviceaccount.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: s3-sync
diff --git a/s3-sync-pipelines/base/s3-sync-task.yaml b/s3-sync-pipelines/base/s3-sync-task.yaml
deleted file mode 100644
index f63b03da..00000000
--- a/s3-sync-pipelines/base/s3-sync-task.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: s3-sync
-spec:
-  params:
-    - name: src_bucket
-      type: string
-    - name: dest_bucket
-      type: string
-    - name: rclone_params
-      type: string
-    - name: sync_config_secret_name
-      type: string
-  steps:
-    - name: s3-sync
-      image: quay.io/internaldatahub/rclone:latest
-      timeout: 10h0m0s
-      resources:
-        limits:
-          memory: 2Gi
-      script: |
-        set -x
-        src_bucket="source:$(params.src_bucket)"
-        dest_bucket="destination:$(params.dest_bucket)"
-        extra_params="$(params.rclone_params)"
-        echo "Creating Bucket $dest_bucket if it doesnt exit yet..."
-        rclone --config /etc/s3-backup-util/rclone.conf mkdir --verbose ${dest_bucket}
-        echo "RClone Copying from $src_bucket to $dest_bucket..."
-        rclone --config /etc/s3-backup-util/rclone.conf copy --verbose ${extra_params} ${src_bucket} ${dest_bucket}
-        echo "Rclone complete."
-      volumeMounts:
-        - mountPath: /etc/s3-backup-util
-          name: sync-config
-  volumes:
-    - name: sync-config
-      secret:
-        secretName: "$(params.sync_config_secret_name)"
diff --git a/s3-sync-pipelines/base/s3-sync-triggertemplate.yaml b/s3-sync-pipelines/base/s3-sync-triggertemplate.yaml
deleted file mode 100644
index a2dffc87..00000000
--- a/s3-sync-pipelines/base/s3-sync-triggertemplate.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-apiVersion: triggers.tekton.dev/v1beta1
-kind: TriggerTemplate
-metadata:
-  name: s3-sync
-spec:
-  params:
-    - name: src_bucket
-    - name: dest_bucket
-    - name: rclone_params
-      default: ""
-    - name: sync_config_secret_name
-  resourcetemplates:
-    - apiVersion: tekton.dev/v1beta1
-      kind: PipelineRun
-      metadata:
-        generateName: "s3-sync-test-sync-"
-      spec:
-        timeout: 10h0m0s
-        pipelineRef:
-          name: s3-sync
-        params:
-          - name: src_bucket
-            value: "$(tt.params.src_bucket)"
-          - name: dest_bucket
-            value: "$(tt.params.dest_bucket)"
-          - name: rclone_params
-            value: "$(tt.params.rclone_params)"
-          - name: sync_config_secret_name
-            value: "$(tt.params.sync_config_secret_name)"
diff --git a/s3-sync-pipelines/overlays/dev/kustomization.yaml b/s3-sync-pipelines/overlays/dev/kustomization.yaml
deleted file mode 100644
index 00bd3968..00000000
--- a/s3-sync-pipelines/overlays/dev/kustomization.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: "dh-dev-s3-sync-pipelines"
-
-resources:
-  - ../../base/
-  - s3-sync-eventlistener-clusterrolebinding.yaml
diff --git a/s3-sync-pipelines/overlays/dev/s3-sync-eventlistener-clusterrolebinding.yaml b/s3-sync-pipelines/overlays/dev/s3-sync-eventlistener-clusterrolebinding.yaml
deleted file mode 100644
index b0a58ec3..00000000
--- a/s3-sync-pipelines/overlays/dev/s3-sync-eventlistener-clusterrolebinding.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: s3-sync-eventlistener-clusterbinding
-subjects:
-  - kind: ServiceAccount
-    name: s3-sync
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: tekton-triggers-eventlistener-clusterroles
diff --git a/s3-sync-pipelines/overlays/prod/kustomization.yaml b/s3-sync-pipelines/overlays/prod/kustomization.yaml
deleted file mode 100644
index 189aa6a0..00000000
--- a/s3-sync-pipelines/overlays/prod/kustomization.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: "dh-prod-s3-sync-pipelines"
-
-resources:
-  - ../../base/
-  - sync-instances/
-
-generators:
-  - secrets/secret-generator.yaml
diff --git a/s3-sync-pipelines/overlays/prod/privileged/s3-sync-eventlistener-clusterrolebinding.yaml b/s3-sync-pipelines/overlays/prod/privileged/s3-sync-eventlistener-clusterrolebinding.yaml
deleted file mode 100644
index b0a58ec3..00000000
--- a/s3-sync-pipelines/overlays/prod/privileged/s3-sync-eventlistener-clusterrolebinding.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: s3-sync-eventlistener-clusterbinding
-subjects:
-  - kind: ServiceAccount
-    name: s3-sync
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: tekton-triggers-eventlistener-clusterroles
diff --git a/s3-sync-pipelines/overlays/prod/secrets/floorist-sync-secret.enc.yaml b/s3-sync-pipelines/overlays/prod/secrets/floorist-sync-secret.enc.yaml
deleted file mode 100644
index 5c59483f..00000000
--- a/s3-sync-pipelines/overlays/prod/secrets/floorist-sync-secret.enc.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: floorist-sync
-stringData:
-    rclone.conf: ENC[AES256_GCM,data:HQXnhEJY2hwGbRryH+kX8CbYeSyebRG8QfCOcUMLq4Po5wPinV7sKyGPB4zRcoHwHXecK9Cs7YsApvcmWduOJItG/hFSr3vus/z6jG6XZoxPQqDvIOF2xP/1z1l62IcDjy+NVCuzhDKXDy1Bbjbl9HvZZrjV4wnts0Wh3ONVvh2Lvx4YgFwJEIvqpQBmibBTFzVI/vSgoyYcLfHd/CdV+WFDZtp8V565sar/6I8vRHMpNjndtt+DUOPNBlH+lV794fYcuR3PndRI0AUmaXwHbgNzLirIrP7KXEOAj61AaL8TKWx5gr7x134CcfSmh6gOkiM370HeiD+cQ1QjDwiMMtWhVf089rXhCrdIMnQ9O9THkwEZwmlVQq1hbwgGi6ggO8eU/ZFV1IblpMF65REOxwe4q377kXmf8yT0IEePOXxn8reg8kWoNhDG7RbV3J9w0stkFZcaOgkZKSmnNa9xJPD9rNehhQX+t+pBIjaHPqbtxG2veip3,iv:/cO9Vn4YIazYG8BhEB1IL74XSEvHJMBS8fCPLeZCrzc=,tag:1J6FKKtEox6oBf2c0sp81A==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    lastmodified: '2022-05-06T18:14:01Z'
-    mac: ENC[AES256_GCM,data:/d+twOfV4ffqBH0zy9klNKSXy2RZVQKL2D19SpMyAXwURYHVQ84A/LTC+nIH50iCh/vTt5aAnygMvgnHg3VDJwN8airHiQt9wJ4B/50AH1g/tPzxQ6DIWqsblnd6nKrrtBmxMKk8CofH9hac5PUGL8IuMhIW8rr5IzH9Grvcp38=,iv:C/9P/+Qf+7xLpbFOHqDHMXmS+zQ4a795306uIZNVNXg=,tag:12+9pg3PYDOAEOn9o1+jgg==,type:str]
-    pgp:
-    -   created_at: '2022-05-06T18:14:00Z'
-        enc: |
-            -----BEGIN PGP MESSAGE-----
-
-            hQEMA/irrHa183bxAQf9E5Y5JbMMShBoid3YnVl1mJA/470HJafs7E3Tjg3oNlMz
-            CqOs4iu/0UZyv901jbZKr4eeRxD1/P3h8Sh8DVYeFY8oZIfy1e4zNI5hvsMszWC0
-            LaGKVk65QNC8tFQC0BgxXEfh/ZNia/IOq2YzEFX7a7LEMWEI9JfN3MbG+mJOp7Ko
-            c1Yc4tI7z4QtOoa9xEIhGL76VpGyHM9Z3TeNnshAnD81cd3dCBg45Bp1hto7XrXY
-            m7OChIv4VyJjcpjOk+sJTp/Z3QdsXNAJ9VGLRKWxLJsIZnHls1jzGipoMVEaVtGJ
-            loWnDgNzziBlorN4vSbkuisNhX/EwtqoSMHX4ntkydJeAR4id9UjcKa6LPwFdz+u
-            YZ4P+bsKzhYDMOITCI/hLiByntY422LFlfmyXhCA9+IIkM5oQd8lbxnUvZzHWysL
-            QW709Y6e4O0DWZElbzUvfhTwjseKYSSnZkzSsdZS1w==
-            =z9bj
-            -----END PGP MESSAGE-----
-        fp: EFDB9AFBD18936D9AB6B2EECBD2C73FF891FBC7E
-    encrypted_regex: ^(data|stringData|tls)$
-    version: 3.6.1
diff --git a/s3-sync-pipelines/overlays/prod/secrets/secret-generator.yaml b/s3-sync-pipelines/overlays/prod/secrets/secret-generator.yaml
deleted file mode 100644
index 613ca757..00000000
--- a/s3-sync-pipelines/overlays/prod/secrets/secret-generator.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: viaduct.ai/v1
-kind: ksops
-metadata:
-  name: secret-generator
-files:
-  - ./secrets/floorist-sync-secret.enc.yaml
diff --git a/s3-sync-pipelines/overlays/prod/sync-instances/floorist.yaml b/s3-sync-pipelines/overlays/prod/sync-instances/floorist.yaml
deleted file mode 100644
index 228b494a..00000000
--- a/s3-sync-pipelines/overlays/prod/sync-instances/floorist.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: floorist-sync
-spec:
-  schedule: "0 5 * * *"
-  jobTemplate:
-    spec:
-      template:
-        spec:
-          containers:
-          - name: trigger
-            image: quay.io/rhn_support_sreber/curl
-            args: ["curl", "-X", "POST", "--data", "{}", "el-floorist-listener:8080"]
-          restartPolicy: Never
-
----
-apiVersion: triggers.tekton.dev/v1beta1
-kind: EventListener
-metadata:
-  name: floorist-listener
-spec:
-  serviceAccountName: s3-sync
-  triggers:
-    - name: s3-sync
-      bindings:
-        - name: src_bucket
-          value: insights-metrics-export-prod
-        - name: dest_bucket
-          value: DH-PROD-RH-INSIGHTS-DATA-EXPORT
-        - name: rclone_params
-          value: ""
-        - name: sync_config_secret_name
-          value: floorist-sync
-      template:
-        ref: s3-sync
diff --git a/s3-sync-pipelines/overlays/prod/sync-instances/kustomization.yaml b/s3-sync-pipelines/overlays/prod/sync-instances/kustomization.yaml
deleted file mode 100644
index 69ede904..00000000
--- a/s3-sync-pipelines/overlays/prod/sync-instances/kustomization.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - floorist.yaml