Local testing setup for debezium

Dockerfile-kafka-connect

@@ -0,0 +1,21 @@
+FROM registry.redhat.io/amq-streams/kafka-36-rhel9:2.7.0-17
+
+ARG CONNECT_PLUGIN_PATH="/opt/kafka/plugins"
+
+USER root:root
+
+RUN mkdir $CONNECT_PLUGIN_PATH
+
+WORKDIR $CONNECT_PLUGIN_PATH
+
+RUN curl -O 'https://maven.repository.redhat.com/ga/io/debezium/debezium-connector-postgres/2.5.4.Final-redhat-00001/debezium-connector-postgres-2.5.4.Final-redhat-00001-plugin.zip'
+RUN curl -O 'https://maven.repository.redhat.com/ga/io/debezium/debezium-scripting/2.5.4.Final-redhat-00001/debezium-scripting-2.5.4.Final-redhat-00001.zip'
+RUN curl -O 'https://repo1.maven.org/maven2/org/apache/groovy/groovy/4.0.22/groovy-4.0.22.jar'
+RUN curl -O 'https://repo1.maven.org/maven2/org/apache/groovy/groovy-jsr223/4.0.22/groovy-jsr223-4.0.22.jar'
+RUN curl -O 'https://repo1.maven.org/maven2/org/apache/groovy/groovy-json/4.0.22/groovy-json-4.0.22.jar'
+
+RUN unzip ./debezium-connector-postgres-2.5.4.Final-redhat-00001-plugin.zip
+RUN unzip ./debezium-scripting-2.5.4.Final-redhat-00001.zip
+
+USER 1001
+WORKDIR /opt/kafka/

docker-compose-kafka.yml

@@ -0,0 +1,187 @@
+services:
+  rbac-server:
+      container_name: rbac_server
+      build:
+          context: .
+          dockerfile: Dockerfile
+      working_dir: /rbac
+      entrypoint:
+        - ./scripts/entrypoint.sh
+      environment:
+        - DATABASE_SERVICE_NAME=POSTGRES_SQL
+        - DATABASE_ENGINE=postgresql
+        - DATABASE_NAME=postgres
+        - POSTGRES_SQL_SERVICE_HOST=db
+        - POSTGRES_SQL_SERVICE_PORT=5432
+        - DATABASE_HOST=db
+        - DATABASE_PORT=5432
+        - API_PATH_PREFIX=/api/rbac
+        - DATABASE_USER=postgres
+        - DATABASE_PASSWORD=postgres
+        - DJANGO_LOG_HANDLERS=console,ecs
+        - DJANGO_READ_DOT_ENV_FILE=True
+        - DEVELOPMENT=${DEVELOPMENT-False}
+        - DJANGO_DEBUG=${DJANGO_DEBUG-True}
+        - REDIS_HOST=${REDIS_HOST-rbac_redis}
+        - PRINCIPAL_PROXY_SERVICE_PROTOCOL=${PRINCIPAL_PROXY_SERVICE_PROTOCOL-https}
+        - PRINCIPAL_PROXY_SERVICE_PORT=${PRINCIPAL_PROXY_SERVICE_PORT-443}
+        - PRINCIPAL_PROXY_SERVICE_HOST=${PRINCIPAL_PROXY_SERVICE_HOST}
+        - PRINCIPAL_PROXY_USER_ENV=${PRINCIPAL_PROXY_USER_ENV-stage}
+        - PRINCIPAL_PROXY_CLIENT_ID=${PRINCIPAL_PROXY_CLIENT_ID-insights-rbac}
+        - PRINCIPAL_PROXY_API_TOKEN=${PRINCIPAL_PROXY_API_TOKEN}
+        - BYPASS_BOP_VERIFICATION=${BYPASS_BOP_VERIFICATION-True}
+        - PRINCIPAL_PROXY_SERVICE_PATH=${PRINCIPAL_PROXY_SERVICE_PATH}
+        - PRINCIPAL_PROXY_SERVICE_SOURCE_CERT=${PRINCIPAL_PROXY_SERVICE_SOURCE_CERT-False}
+        - PRINCIPAL_PROXY_SERVICE_SSL_VERIFY=${PRINCIPAL_PROXY_SERVICE_SSL_VERIFY-False}
+        - RBAC_DESTRUCTIVE_API_ENABLED_UNTIL=${RBAC_DESTRUCTIVE_API_ENABLED_UNTIL}
+        - RBAC_DESTRUCTIVE_SEEDING_ENABLED_UNTIL=${RBAC_DESTRUCTIVE_SEEDING_ENABLED_UNTIL}
+      privileged: true
+      ports:
+          - 9080:8080
+      volumes:
+        - '.:/rbac/'
+      depends_on:
+        db:
+          condition: service_healthy
+        rbac-worker:
+          condition: service_healthy
+        rbac-scheduler:
+          condition: service_healthy
+      healthcheck:
+        test: curl -q http://localhost:8080/metrics
+        interval: 5s
+        timeout: 5s
+        retries: 10
+
+  rbac-worker:
+      container_name: rbac_worker
+      build:
+          context: .
+          dockerfile: Dockerfile
+      working_dir: /opt/rbac/rbac
+      entrypoint: ['celery', '--broker=redis://redis:6379/0', '-A', 'rbac.celery', 'worker', '--loglevel=INFO']
+      privileged: true
+      depends_on:
+        redis:
+          condition: service_healthy
+      healthcheck:
+        test: [ "CMD-SHELL", "celery --broker=redis://redis:6379/0 -A rbac.celery status" ]
+        interval: 30s
+        timeout: 10s
+        retries: 3
+
+  rbac-scheduler:
+      container_name: rbac_scheduler
+      build:
+          context: .
+          dockerfile: Dockerfile
+      working_dir: /opt/rbac/rbac
+      entrypoint: ['celery', '--broker=redis://redis:6379/0', '-A', 'rbac.celery', 'beat', '--loglevel=INFO']
+      privileged: true
+      depends_on:
+        redis:
+          condition: service_healthy
+      healthcheck:
+        test: [ "CMD-SHELL", "celery --broker=redis://redis:6379/0 -A rbac.celery status" ]
+        interval: 30s
+        timeout: 10s
+        retries: 3
+
+  redis:
+    container_name: rbac_redis
+    image: redis:5.0.4
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: [ "CMD-SHELL", "redis-cli ping | grep PONG" ]
+      interval: 1s
+      timeout: 3s
+      retries: 5
+
+  db:
+    container_name: rbac_db
+    image: postgres:14.5
+    command: [
+      "postgres", "-c", "wal_level=logical"
+    ]
+    environment:
+    - POSTGRES_DB=postgres
+    - POSTGRES_USER=postgres
+    - POSTGRES_PASSWORD=postgres
+    - PGDATA=/var/lib/postgresql/data/pgdata
+    ports:
+      - "15432:5432"
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+
+  zookeeper:
+    container_name: rbac_zookeeper
+    image: registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0-6
+    command: [
+      "sh", "-c", "bin/zookeeper-server-start.sh config/zookeeper.properties"
+    ]
+    ports:
+      - "2181:2191"
+    environment:
+      LOG_DIR: /tmp/logs
+
+  kafka:
+    container_name: rbac_kafka
+    image: registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0-6
+    command: [
+      "sh", "-c",
+      "bin/kafka-server-start.sh config/server.properties --override listeners=$${KAFKA_LISTENERS} --override advertised.listeners=$${KAFKA_ADVERTISED_LISTENERS} --override zookeeper.connect=$${KAFKA_ZOOKEEPER_CONNECT} --override inter.broker.listener.name=$${KAFKA_INTER_BROKER_LISTENER_NAME} --override listener.security.protocol.map=$${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP}"
+    ]
+    depends_on:
+      - zookeeper
+    mem_limit: 500m
+    ports:
+      - "9092:9092"
+      - "9094:9094"
+    environment:
+      LOG_DIR: "/tmp/logs"
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092"
+      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"      
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "storage-topic:1:1:compact,global-id-topic:1:1:compact,input-topic:1:1:compact,logx-topic:1:1:compact,dbx-topic:1:1:compact"
+      KAFKA_HEAP_OPTS: "-Xmx500M -Xms500M"
+
+  kafka_connect:
+    build:
+      context: .
+      dockerfile: Dockerfile-kafka-connect
+    container_name: rbac_kafka_connect
+    command: [
+      "sh", "-c", "bin/connect-distributed.sh /opt/kafka/config/rebac_connect.properties"
+    ]
+    depends_on:
+      - kafka
+    mem_limit: 500m
+    ports:
+      - "8083:8083"
+    volumes:
+      - ./scripts/debezium/rebac_connect.properties:/opt/kafka/config/rebac_connect.properties
+
+  wait_for_app:
+    container_name: wait_for_app
+    image: hello-world:latest
+    depends_on:
+      rbac-server:
+        condition: service_healthy
+
+volumes:
+  pg_data:
+
+networks:
+  default:
+    name: rbac-network
+    external: true

scripts/debezium/README.md

@@ -0,0 +1,14 @@
+# Local Debezium Testing
+In the main insights-rbac directory there is now a docker-compose-kafka.yml file that has added a local Kafka cluster and KafkaConnect instance to the standard RBAC setup to allow for testing of Debezium setups for Kessel/Relations.
+
+In this directory are a series of scripts for setting up and testing the results of the debezium connector's use with the RBAC database.
+
+# Setup
+Stand up the testing environment with `docker-compose -f docker-compose-kafka.yml up` to get things up and running. This will require an active login to the cloudservices quay.io repository to pull our KafkaConnect image with the necessary connector plugins already installed.
+
+Once the testing environment is up and running, run the the `setup_local.py` python script in this directory to add the necessary `outbox` table and wal_level configurations to the RBAC postgres DB and create the `debezium-test` connector on the local KafkaConnect instance.
+
+# Testing
+With the above complete, any INSERT into the `outbox` table will result in a corresponding message to the `debezium-testing.public.outbox` topic on Kafka. The messages in this topic can be checked with the `check_events.sh` script, also in this directory.
+
+Eventually, we'll have an output process or Kafka Connector that will grab events from the outbox events topic and replicate them to the relations API or directly into the appropriate DB tables. For now, please use this setup to see how KafkaConnect/Debezium and the RBAC DB work together and what the format of the messages will be. 

scripts/debezium/check_events.sh

@@ -0,0 +1,5 @@
+#/bin/bash
+
+DOCKER_COMMAND=podman
+
+$DOCKER_COMMAND exec -it rbac_kafka /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic outbox.event.one --from-beginning --timeout-ms 600

scripts/debezium/connector.json

@@ -0,0 +1,14 @@
+{
+	"name": "debezium-test",
+	"connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+	"database.hostname": "rbac_db",
+	"database.port": 5432,
+	"database.user": "postgres",
+	"database.password": "postgres",
+	"database.dbname": "postgres",
+	"plugin.name": "pgoutput",
+	"topic.prefix": "debezium-test",
+	"table.include.list": "public.management_outbox",
+	"transforms": "outbox",
+	"transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter"
+}

scripts/debezium/rebac_connect.properties

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These are defaults. This file just demonstrates how to override some settings.
+bootstrap.servers=kafka:9092
+
+# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
+# need to configure these based on the format they want their data in when loaded from or stored into Kafka
+key.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
+# it to
+key.converter.schemas.enable=true
+value.converter.schemas.enable=true
+
+# The internal converter used for offsets and config data is configurable and must be specified, but most users will
+# always want to use the built-in default. Offset and config data is never visible outside of Copcyat in this format.
+internal.key.converter=org.apache.kafka.connect.json.JsonConverter
+internal.value.converter=org.apache.kafka.connect.json.JsonConverter
+internal.key.converter.schemas.enable=false
+internal.value.converter.schemas.enable=false
+
+# Flush much faster than normal, which is useful for testing/debugging
+offset.flush.interval.ms=10000
+
+group.id=rebac-connect
+offset.storage.topic=rebac-connect-cluster-offsets
+offset.storage.replication.factor=1
+config.storage.topic=rebac-connect-cluster-configs
+config.storage.replication.factor=1
+status.storage.topic=rebac-connect-cluster-status
+status.storage.replication.factor=1
+
+plugin.path=/opt/kafka/plugins

scripts/debezium/setup_local.py

@@ -0,0 +1,42 @@
+import psycopg2
+import requests
+
+
+def setup_needs():
+    setup_db_for_debezium()
+    setup_connector()
+
+
+def setup_connector():
+    with open("./connector.json") as cj:
+        d = cj.read()
+
+    print(d)
+    headers = {"Content-type": "application/json"}
+    res = requests.put(
+        url="http://localhost:8083/connectors/debezium-test/config",
+        data=d,
+        headers=headers,
+    )
+    res.raise_for_status()
+
+
+def setup_db_for_debezium():
+    try:
+        conn = psycopg2.connect(
+            "host=localhost port=15432 dbname=postgres user=postgres password=postgres"
+        )
+        conn.autocommit = True
+        cur = conn.cursor()
+        cur.execute("ALTER SYSTEM SET wal_level = logical;")
+        conn.commit()
+        cur.execute(
+            "CREATE TABLE management_outbox (id uuid, aggregatetype varchar(255), aggregateid varchar(255), type varchar(255), payload jsonb);"
+        )
+        conn.commit()
+    except psycopg2.errors.DuplicateTable as e:
+        print(e)
+
+
+if __name__ == "__main__":
+    setup_needs()