From a9a2fea0bcd509d9ee8a0c0d907acae1cb82f7a7 Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 25 Feb 2025 08:15:04 +0100 Subject: [PATCH 01/29] build with gssapi tag --- .goreleaser.yml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index d498f7949..02c76ae49 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -19,6 +19,7 @@ builds: goarch: arm flags: - -v + - -tags gssapi ldflags: - -s -w -X main.version=v{{.Version}} -X main.commit={{.ShortCommit}} -X main.buildDate={{.Date}} archives: diff --git a/Makefile b/Makefile index 3fbaa8841..eaca1dcba 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ init: ## Install linters cd tools && go generate -x -tags=tools build: ## Compile using plain go build - go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter + go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter -tags gssapi release: ## Build the binaries using goreleaser docker run --rm --privileged \ From beb936b0bc27f6ad6db7640d662a6ed07c82f0f0 Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 25 Feb 2025 08:15:19 +0100 Subject: [PATCH 02/29] set up kerberos test env --- .gitignore | 3 +++ docker-compose.yml | 42 +++++++++++++++++++++++++++++ docker/kerberos.dockerfile | 3 +++ docker/scripts/setup-krb5-mongo.sh | 11 ++++++++ docker/scripts/setup-krb5-server.sh | 33 +++++++++++++++++++++++ 5 files changed, 92 insertions(+) create mode 100644 docker/kerberos.dockerfile create mode 100755 docker/scripts/setup-krb5-mongo.sh create mode 100644 docker/scripts/setup-krb5-server.sh diff --git a/.gitignore b/.gitignore index 11a5485e5..3c0eb7b70 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ dist cover.out mongodb_exporter .DS_Store + +docker/kerberos/cache/* +docker/kerberos/conf/* \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9e7cacc79..71c4b755a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -286,6 +286,48 @@ services: - ./docker/scripts:/scripts command: /scripts/run-mongodb-encrypted.sh + kerberos: + build: + dockerfile: ./docker/kerberos.dockerfile + container_name: kerberos + hostname: kerberos + environment: + - KRB5_TRACE=/dev/stderr + - MONGO_HOST=psmdb-kerberos + - KERBEROS_HOST=kerberos + depends_on: + - psmdb-kerberos + volumes: + - ./docker/kerberos/conf:/etc/ + - ./docker/kerberos/cache:/tmp/ + - ./docker/scripts:/scripts + ports: + - "88:88/udp" + entrypoint: [ "sh", "/scripts/setup-krb5-server.sh" ] + healthcheck: + test: [ "CMD", "kadmin.local", "-q", "listprincs" ] + interval: 5s + timeout: 5s + retries: 5 + + psmdb-kerberos: + image: percona/percona-server-mongodb:8.0 + container_name: psmdb-kerberos + hostname: psmdb-kerberos + ports: + - 27017:27017 + environment: + - KRB5_KTNAME=/tmp/mongodb.keytab + - KRB5CCNAME=/tmp/krb5cc_0 + - MONGO_INITDB_ROOT_USERNAME=admin + - MONGO_INITDB_ROOT_PASSWORD=adminpassword + volumes: + - ./docker/kerberos/conf:/etc/ + - ./docker/scripts:/scripts + - ./docker/kerberos/cache:/tmp/ + command: > + mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 + volumes: pbm-backups: diff --git a/docker/kerberos.dockerfile b/docker/kerberos.dockerfile new file mode 100644 index 000000000..990db598a --- /dev/null +++ b/docker/kerberos.dockerfile @@ -0,0 +1,3 @@ +FROM alpine +RUN apk add --no-cache bash krb5 krb5-server krb5-pkinit +EXPOSE 88/udp diff --git a/docker/scripts/setup-krb5-mongo.sh b/docker/scripts/setup-krb5-mongo.sh new file mode 100755 index 000000000..8edc2d454 --- /dev/null +++ b/docker/scripts/setup-krb5-mongo.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +username=${MONGO_INITDB_ROOT_USERNAME} +password=${MONGO_INITDB_ROOT_PASSWORD} +port=${PORT:-27017} + +docker exec ${KERBEROS_HOST} bash -c "kinit pmm-test@PERCONATEST.COM -kt /tmp/mongodb.keytab" + +#docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/krb5cc_0" +docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/mongodb.keytab" +docker exec ${MONGO_HOST} mongosh "${MONGO_HOST}:${port}" -u ${username} -p ${password} --eval 'db.getSiblingDB("$external").createUser({user: "pmm-test@PERCONATEST.COM",roles: [{role: "read", db: "admin"}]});' diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh new file mode 100644 index 000000000..928c14836 --- /dev/null +++ b/docker/scripts/setup-krb5-server.sh @@ -0,0 +1,33 @@ +#! /env/sh + +mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` +kerberos_host=`getent hosts ${KERBEROS_HOST} | awk '{ print $1 }'` + +cat > /etc/krb5.conf < Date: Tue, 4 Mar 2025 00:16:00 +0100 Subject: [PATCH 03/29] improve setup --- docker-compose.yml | 23 +++++++++++++++++++++-- docker/kerberos-exporter.dockerfile | 5 +++++ docker/scripts/setup-krb5-exporter.sh | 10 ++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docker/kerberos-exporter.dockerfile create mode 100755 docker/scripts/setup-krb5-exporter.sh diff --git a/docker-compose.yml b/docker-compose.yml index 71c4b755a..0c6a5af80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -295,10 +295,12 @@ services: - KRB5_TRACE=/dev/stderr - MONGO_HOST=psmdb-kerberos - KERBEROS_HOST=kerberos + - MONGO_KERBEROS_USERNAME=pmm-test + - MONGO_KERBEROS_PASSWORD=password1 depends_on: - psmdb-kerberos volumes: - - ./docker/kerberos/conf:/etc/ + - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - ./docker/kerberos/cache:/tmp/ - ./docker/scripts:/scripts ports: @@ -322,12 +324,29 @@ services: - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword volumes: - - ./docker/kerberos/conf:/etc/ + - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - ./docker/scripts:/scripts - ./docker/kerberos/cache:/tmp/ command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 + + exporter-kerberos: + build: + dockerfile: ./docker/kerberos-exporter.dockerfile + container_name: exporter-kerberos + hostname: exporter-kerberos + ports: + - "9217:9216" + environment: + - KRB5_KTNAME=/tmp/mongodb.keytab + - KRB5CCNAME=/tmp/krb5cc_0 + - MONGO + volumes: + - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf + - ./docker/scripts/setup-krb5-exporter.sh:/scripts/setup-krb5-exporter.sh + - ./docker/kerberos/cache:/tmp/ + entrypoint: [ "sh", "/scripts/setup-krb5-exporter.sh" ] volumes: pbm-backups: diff --git a/docker/kerberos-exporter.dockerfile b/docker/kerberos-exporter.dockerfile new file mode 100644 index 000000000..06ca13571 --- /dev/null +++ b/docker/kerberos-exporter.dockerfile @@ -0,0 +1,5 @@ +FROM alpine +RUN apk add --no-cache ca-certificates +USER 65535:65535 +COPY ./mongodb_exporter / +EXPOSE 9216 diff --git a/docker/scripts/setup-krb5-exporter.sh b/docker/scripts/setup-krb5-exporter.sh new file mode 100755 index 000000000..0b8d9d22b --- /dev/null +++ b/docker/scripts/setup-krb5-exporter.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` + +username=${MONGO_KERBEROS_USERNAME} +password=${MONGO_KERBEROS_PASSWORD} + +export MONGODB_URI="mongodb://${username}:${password}@${mongohost}:27017/?directConnection=true&authSource=%24external&authMechanism=GSSAPI" + +/mongodb_exporter "$@" \ No newline at end of file From 382894518ffdcdcbe6e8b93653c159d01d8c7d26 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 07:08:57 +0100 Subject: [PATCH 04/29] create separate exporter keytab --- docker-compose.yml | 18 ------------------ docker/scripts/setup-krb5-exporter.sh | 10 ---------- docker/scripts/setup-krb5-mongo.sh | 3 ++- docker/scripts/setup-krb5-server.sh | 10 ++++++++-- 4 files changed, 10 insertions(+), 31 deletions(-) delete mode 100755 docker/scripts/setup-krb5-exporter.sh diff --git a/docker-compose.yml b/docker-compose.yml index 0c6a5af80..3dbcada83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -329,24 +329,6 @@ services: - ./docker/kerberos/cache:/tmp/ command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 - - - exporter-kerberos: - build: - dockerfile: ./docker/kerberos-exporter.dockerfile - container_name: exporter-kerberos - hostname: exporter-kerberos - ports: - - "9217:9216" - environment: - - KRB5_KTNAME=/tmp/mongodb.keytab - - KRB5CCNAME=/tmp/krb5cc_0 - - MONGO - volumes: - - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - - ./docker/scripts/setup-krb5-exporter.sh:/scripts/setup-krb5-exporter.sh - - ./docker/kerberos/cache:/tmp/ - entrypoint: [ "sh", "/scripts/setup-krb5-exporter.sh" ] volumes: pbm-backups: diff --git a/docker/scripts/setup-krb5-exporter.sh b/docker/scripts/setup-krb5-exporter.sh deleted file mode 100755 index 0b8d9d22b..000000000 --- a/docker/scripts/setup-krb5-exporter.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` - -username=${MONGO_KERBEROS_USERNAME} -password=${MONGO_KERBEROS_PASSWORD} - -export MONGODB_URI="mongodb://${username}:${password}@${mongohost}:27017/?directConnection=true&authSource=%24external&authMechanism=GSSAPI" - -/mongodb_exporter "$@" \ No newline at end of file diff --git a/docker/scripts/setup-krb5-mongo.sh b/docker/scripts/setup-krb5-mongo.sh index 8edc2d454..471742d6a 100755 --- a/docker/scripts/setup-krb5-mongo.sh +++ b/docker/scripts/setup-krb5-mongo.sh @@ -8,4 +8,5 @@ docker exec ${KERBEROS_HOST} bash -c "kinit pmm-test@PERCONATEST.COM -kt /tmp/mo #docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/krb5cc_0" docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/mongodb.keytab" -docker exec ${MONGO_HOST} mongosh "${MONGO_HOST}:${port}" -u ${username} -p ${password} --eval 'db.getSiblingDB("$external").createUser({user: "pmm-test@PERCONATEST.COM",roles: [{role: "read", db: "admin"}]});' +docker exec ${MONGO_HOST} mongosh "${MONGO_HOST}:${port}" -u ${username} -p ${password} --eval '' +db.getSiblingDB("$external").createUser({user: "pmm-test@PERCONATEST.COM",roles: [{role: "read", db: "admin"}]}); \ No newline at end of file diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh index 928c14836..62b785e2e 100644 --- a/docker/scripts/setup-krb5-server.sh +++ b/docker/scripts/setup-krb5-server.sh @@ -2,6 +2,7 @@ mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` kerberos_host=`getent hosts ${KERBEROS_HOST} | awk '{ print $1 }'` +gateway_ip=`ip route | grep default | awk '{print $3}'` cat > /etc/krb5.conf < /etc/krb5.conf < Date: Wed, 5 Mar 2025 13:42:11 +0100 Subject: [PATCH 05/29] prepare test environment --- Makefile | 4 +- docker-compose.yml | 5 +- docker/scripts/{setup-pbm.sh => init-pbm.sh} | 0 docker/scripts/init-psmdb-kerberos.sh | 3 + docker/scripts/setup-krb5-mongo.sh | 25 +++++-- docker/scripts/setup-krb5-server.sh | 8 --- .../scripts/{init-shard.sh => setup-shard.sh} | 2 +- exporter/exporter_test.go | 71 +++++++++++++++++++ internal/tu/testutils.go | 13 ++++ 9 files changed, 113 insertions(+), 18 deletions(-) rename docker/scripts/{setup-pbm.sh => init-pbm.sh} (100%) create mode 100755 docker/scripts/init-psmdb-kerberos.sh rename docker/scripts/{init-shard.sh => setup-shard.sh} (97%) diff --git a/Makefile b/Makefile index eaca1dcba..e73cc617d 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,9 @@ test-cover: env ## Run tests and collect cross-package coverage inf test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster docker compose up --build -d - ./docker/scripts/setup-pbm.sh + ./docker/scripts/init-psmdb-kerberos.sh + ./docker/scripts/init-pbm.sh test-cluster-clean: env ## Stops MongoDB test cluster. docker compose down --remove-orphans --volumes + rm -f ./docker/kerberos/cache/* diff --git a/docker-compose.yml b/docker-compose.yml index 3dbcada83..ea0d4e7e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -265,7 +265,7 @@ services: - PORT2=27017 - PORT3=27017 - VERSION=${TEST_MONGODB_IMAGE} - entrypoint: [ "/scripts/init-shard.sh" ] + entrypoint: [ "/scripts/setup-shard.sh" ] restart: on-failure:20 standalone: @@ -317,12 +317,13 @@ services: container_name: psmdb-kerberos hostname: psmdb-kerberos ports: - - 27017:27017 + - 29017:27017 environment: - KRB5_KTNAME=/tmp/mongodb.keytab - KRB5CCNAME=/tmp/krb5cc_0 - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword + - MONGODB_HOST=psmdb-kerberos volumes: - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - ./docker/scripts:/scripts diff --git a/docker/scripts/setup-pbm.sh b/docker/scripts/init-pbm.sh similarity index 100% rename from docker/scripts/setup-pbm.sh rename to docker/scripts/init-pbm.sh diff --git a/docker/scripts/init-psmdb-kerberos.sh b/docker/scripts/init-psmdb-kerberos.sh new file mode 100755 index 000000000..fc9f29335 --- /dev/null +++ b/docker/scripts/init-psmdb-kerberos.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it psmdb-kerberos bash -c '/scripts/setup-krb5-mongo.sh' diff --git a/docker/scripts/setup-krb5-mongo.sh b/docker/scripts/setup-krb5-mongo.sh index 471742d6a..1c962eadb 100755 --- a/docker/scripts/setup-krb5-mongo.sh +++ b/docker/scripts/setup-krb5-mongo.sh @@ -2,11 +2,24 @@ username=${MONGO_INITDB_ROOT_USERNAME} password=${MONGO_INITDB_ROOT_PASSWORD} -port=${PORT:-27017} -docker exec ${KERBEROS_HOST} bash -c "kinit pmm-test@PERCONATEST.COM -kt /tmp/mongodb.keytab" +echo "Waiting for startup.." +until mongosh --host 127.0.0.1:27017 -u ${username} -p ${password} --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do + printf '.' + sleep 1 +done -#docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/krb5cc_0" -docker exec --user root ${MONGO_HOST} bash -c "chown -R mongodb:root /tmp/mongodb.keytab" -docker exec ${MONGO_HOST} mongosh "${MONGO_HOST}:${port}" -u ${username} -p ${password} --eval '' -db.getSiblingDB("$external").createUser({user: "pmm-test@PERCONATEST.COM",roles: [{role: "read", db: "admin"}]}); \ No newline at end of file +echo "Started.." + +# create role with anyAction on all resources (needed to allow exporter run execute commands) +# create mongodb user using the same username as the kerberos principal +mongosh --host 127.0.0.1:27017 -u ${username} -p ${password} < /etc/krb5.conf < Date: Wed, 5 Mar 2025 13:48:38 +0100 Subject: [PATCH 06/29] include test tags --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e73cc617d..6574c29b5 100644 --- a/Makefile +++ b/Makefile @@ -99,13 +99,13 @@ help: ## Display this help message awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' test: env ## Run all tests - go test -v -count 1 -timeout 30s ./... + go test -tags gssapi -v -count 1 -timeout 30s ./... test-race: env ## Run all tests with race flag - go test -race -v -timeout 30s ./... + go test -tags gssapi -race -v -timeout 30s ./... test-cover: env ## Run tests and collect cross-package coverage information - go test -race -timeout 30s -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./... + go test -tags gssapi -race -timeout 30s -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./... test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster docker compose up --build -d From 260cbd0de5ef3cff38f100ed27e224cac9d562da Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 13:54:59 +0100 Subject: [PATCH 07/29] create conf file before starting containers --- .gitignore | 2 +- Makefile | 1 + docker/scripts/setup-krb5-server.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3c0eb7b70..f1b2d21df 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ mongodb_exporter .DS_Store docker/kerberos/cache/* -docker/kerberos/conf/* \ No newline at end of file +docker/kerberos/conf/* diff --git a/Makefile b/Makefile index 6574c29b5..73f5de7b8 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ test-cover: env ## Run tests and collect cross-package coverage inf go test -tags gssapi -race -timeout 30s -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./... test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster + touch ./docker/kerberos/cache/krb5.conf docker compose up --build -d ./docker/scripts/init-psmdb-kerberos.sh ./docker/scripts/init-pbm.sh diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh index 722192c4f..d26f3d55e 100644 --- a/docker/scripts/setup-krb5-server.sh +++ b/docker/scripts/setup-krb5-server.sh @@ -28,4 +28,4 @@ kadmin.local -q "addprinc -pw password root/admin" kadmin.local -q "addprinc -pw mongodb mongodb/${mongohost}" kadmin.local -q "addprinc -pw password1 pmm-test" kadmin.local -q "ktadd -k /tmp/mongodb.keytab mongodb/${mongohost}@PERCONATEST.COM" -krb5kdc -n \ No newline at end of file +krb5kdc -n From 49cd21c0fa5d1d2a4dd690371a66536c82055650 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 14:00:39 +0100 Subject: [PATCH 08/29] use tmp dir for config --- .gitignore | 2 +- Makefile | 1 - docker-compose.yml | 4 ++-- docker/scripts/setup-krb5-server.sh | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f1b2d21df..c7baff44d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ mongodb_exporter .DS_Store docker/kerberos/cache/* -docker/kerberos/conf/* + diff --git a/Makefile b/Makefile index 73f5de7b8..6574c29b5 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,6 @@ test-cover: env ## Run tests and collect cross-package coverage inf go test -tags gssapi -race -timeout 30s -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./... test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster - touch ./docker/kerberos/cache/krb5.conf docker compose up --build -d ./docker/scripts/init-psmdb-kerberos.sh ./docker/scripts/init-pbm.sh diff --git a/docker-compose.yml b/docker-compose.yml index ea0d4e7e9..179ed4165 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -295,12 +295,12 @@ services: - KRB5_TRACE=/dev/stderr - MONGO_HOST=psmdb-kerberos - KERBEROS_HOST=kerberos + - KRB5_CONFIG=/tmp/krb5.conf - MONGO_KERBEROS_USERNAME=pmm-test - MONGO_KERBEROS_PASSWORD=password1 depends_on: - psmdb-kerberos volumes: - - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - ./docker/kerberos/cache:/tmp/ - ./docker/scripts:/scripts ports: @@ -321,11 +321,11 @@ services: environment: - KRB5_KTNAME=/tmp/mongodb.keytab - KRB5CCNAME=/tmp/krb5cc_0 + - KRB5_CONFIG=/tmp/krb5.conf - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword - MONGODB_HOST=psmdb-kerberos volumes: - - ./docker/kerberos/conf/krb5.conf:/etc/krb5.conf - ./docker/scripts:/scripts - ./docker/kerberos/cache:/tmp/ command: > diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh index d26f3d55e..1499f7b07 100644 --- a/docker/scripts/setup-krb5-server.sh +++ b/docker/scripts/setup-krb5-server.sh @@ -4,7 +4,7 @@ mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` kerberos_host=`getent hosts ${KERBEROS_HOST} | awk '{ print $1 }'` gateway_ip=`ip route | grep default | awk '{print $3}'` -cat > /etc/krb5.conf < /tmp/krb5.conf < Date: Wed, 5 Mar 2025 14:04:09 +0100 Subject: [PATCH 09/29] re-run script --- docker/scripts/init-psmdb-kerberos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/init-psmdb-kerberos.sh b/docker/scripts/init-psmdb-kerberos.sh index fc9f29335..3bd9c56d2 100755 --- a/docker/scripts/init-psmdb-kerberos.sh +++ b/docker/scripts/init-psmdb-kerberos.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker exec -it psmdb-kerberos bash -c '/scripts/setup-krb5-mongo.sh' +docker exec psmdb-kerberos bash -c '/scripts/setup-krb5-mongo.sh' From 7de17fb4486e1f6e0548427ae91fac5d899ba5b3 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 14:21:05 +0100 Subject: [PATCH 10/29] add logrus reporter --- exporter/exporter_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 4f1d2689d..ddaa25620 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -201,6 +201,7 @@ func TestMongoS(t *testing.T) { func TestMongoWithGSSAPI(t *testing.T) { logger := logrus.New() + logger.SetReportCaller(true) kerberosHost, err := tu.IpForContainer("kerberos") require.NoError(t, err) From 78546b8ce41363793804e903e0ee451b3ff3f657 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 14:27:41 +0100 Subject: [PATCH 11/29] install libkrb5-dev in runner --- .github/workflows/go.yml | 3 +++ docker-compose.yml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 62337481a..f1a568194 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,6 +46,9 @@ jobs: with: go-version-file: ${{ github.workspace }}/go.mod + - name: Install krb5 libraries + run: sudo apt-get install -y libkrb5-dev + - name: Run tests with code coverage run: | TEST_MONGODB_IMAGE=${{ matrix.image }} make test-cluster diff --git a/docker-compose.yml b/docker-compose.yml index 179ed4165..fe0c0c1f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -324,7 +324,6 @@ services: - KRB5_CONFIG=/tmp/krb5.conf - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword - - MONGODB_HOST=psmdb-kerberos volumes: - ./docker/scripts:/scripts - ./docker/kerberos/cache:/tmp/ From c43f06f33d436028c3ab5e8ad5fcabdcf416e525 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 14:31:04 +0100 Subject: [PATCH 12/29] drop installation --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f1a568194..2993c1811 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,8 +46,8 @@ jobs: with: go-version-file: ${{ github.workspace }}/go.mod - - name: Install krb5 libraries - run: sudo apt-get install -y libkrb5-dev + #- name: Install krb5 libraries + # run: sudo apt-get install -y libkrb5-dev - name: Run tests with code coverage run: | From df39b915e2d1e571292bd45fe740258df8889219 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 14:45:00 +0100 Subject: [PATCH 13/29] do not mount tmp directly --- docker-compose.yml | 12 ++++++------ docker/scripts/setup-krb5-server.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fe0c0c1f6..1d3c3c324 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -295,13 +295,13 @@ services: - KRB5_TRACE=/dev/stderr - MONGO_HOST=psmdb-kerberos - KERBEROS_HOST=kerberos - - KRB5_CONFIG=/tmp/krb5.conf + - KRB5_CONFIG=/krb5/krb5.conf - MONGO_KERBEROS_USERNAME=pmm-test - MONGO_KERBEROS_PASSWORD=password1 depends_on: - psmdb-kerberos volumes: - - ./docker/kerberos/cache:/tmp/ + - ./docker/kerberos/cache:/krb5/ - ./docker/scripts:/scripts ports: - "88:88/udp" @@ -319,14 +319,14 @@ services: ports: - 29017:27017 environment: - - KRB5_KTNAME=/tmp/mongodb.keytab - - KRB5CCNAME=/tmp/krb5cc_0 - - KRB5_CONFIG=/tmp/krb5.conf + - KRB5_KTNAME=/krb5/mongodb.keytab + - KRB5CCNAME=/krb5/krb5cc_0 + - KRB5_CONFIG=/krb5/krb5.conf - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword volumes: - ./docker/scripts:/scripts - - ./docker/kerberos/cache:/tmp/ + - ./docker/kerberos/cache:/krb5/ command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 volumes: diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh index 1499f7b07..960aaea86 100644 --- a/docker/scripts/setup-krb5-server.sh +++ b/docker/scripts/setup-krb5-server.sh @@ -4,7 +4,7 @@ mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` kerberos_host=`getent hosts ${KERBEROS_HOST} | awk '{ print $1 }'` gateway_ip=`ip route | grep default | awk '{print $3}'` -cat > /tmp/krb5.conf < /krb5/krb5.conf < Date: Wed, 5 Mar 2025 14:48:56 +0100 Subject: [PATCH 14/29] install krb5 libraries --- .github/workflows/go.yml | 6 ++++-- exporter/exporter_test.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2993c1811..ac6028a10 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,8 +46,10 @@ jobs: with: go-version-file: ${{ github.workspace }}/go.mod - #- name: Install krb5 libraries - # run: sudo apt-get install -y libkrb5-dev + - name: Install jq tool + run: | + sudo apt-get update + sudo apt-get install libkrb5-dev - name: Run tests with code coverage run: | diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index ddaa25620..e076a21f7 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -18,7 +18,6 @@ package exporter import ( "context" "fmt" - "github.com/stretchr/testify/require" "io" "net" "net/http" @@ -32,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/percona/mongodb_exporter/internal/tu" ) From 2e5902f0aba640f694cd0bb1c27255e6edf9bd07 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 21:43:37 +0100 Subject: [PATCH 15/29] use one liners for mongo commands --- docker/scripts/setup-krb5-mongo.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/scripts/setup-krb5-mongo.sh b/docker/scripts/setup-krb5-mongo.sh index 1c962eadb..11e913581 100755 --- a/docker/scripts/setup-krb5-mongo.sh +++ b/docker/scripts/setup-krb5-mongo.sh @@ -13,13 +13,5 @@ echo "Started.." # create role with anyAction on all resources (needed to allow exporter run execute commands) # create mongodb user using the same username as the kerberos principal -mongosh --host 127.0.0.1:27017 -u ${username} -p ${password} < Date: Wed, 5 Mar 2025 22:28:27 +0100 Subject: [PATCH 16/29] fix permissions --- docker/scripts/init-psmdb-kerberos.sh | 1 + exporter/exporter_test.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/scripts/init-psmdb-kerberos.sh b/docker/scripts/init-psmdb-kerberos.sh index 3bd9c56d2..3ae827c3f 100755 --- a/docker/scripts/init-psmdb-kerberos.sh +++ b/docker/scripts/init-psmdb-kerberos.sh @@ -1,3 +1,4 @@ #!/bin/bash +docker exec --user root psmdb-kerberos bash -c 'chown mongodb:root /krb5/mongodb.keytab' docker exec psmdb-kerberos bash -c '/scripts/setup-krb5-mongo.sh' diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index e076a21f7..c75954186 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -227,8 +227,12 @@ func TestMongoWithGSSAPI(t *testing.T) { %s = PERCONATEST.COM `, kerberosHost, kerberosHost) - configFile, err := os.Create(t.TempDir() + "krb5.conf") + configFile, err := os.Create(t.TempDir() + "/krb5.conf") require.NoError(t, err) + defer func() { + _ = configFile.Close() + _ = os.Setenv("KRB5_CONFIG", "") + }() _, err = configFile.WriteString(config) require.NoError(t, err) From 4168b9b6d40add4c511822842798e6a969881f86 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 22:36:24 +0100 Subject: [PATCH 17/29] fix up linter --- Makefile | 1 - exporter/exporter_test.go | 14 +++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6574c29b5..33e0dce88 100644 --- a/Makefile +++ b/Makefile @@ -114,4 +114,3 @@ test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MON test-cluster-clean: env ## Stops MongoDB test cluster. docker compose down --remove-orphans --volumes - rm -f ./docker/kerberos/cache/* diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index c75954186..3456b9f8d 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -199,7 +199,7 @@ func TestMongoS(t *testing.T) { } } -func TestMongoWithGSSAPI(t *testing.T) { +func TestGSSAPIAuth(t *testing.T) { logger := logrus.New() logger.SetReportCaller(true) @@ -241,7 +241,11 @@ func TestMongoWithGSSAPI(t *testing.T) { username := "pmm-test%40PERCONATEST.COM" password := "password1" - uri := fmt.Sprintf("mongodb://%s:%s@%s:27017/?authSource=$external&authMechanism=GSSAPI", username, password, mongoHost) + uri := fmt.Sprintf("mongodb://%s:%s@%s/?authSource=$external&authMechanism=GSSAPI", + username, + password, + net.JoinHostPort(mongoHost, "27017"), + ) exporterOpts := &Opts{ URI: uri, Logger: logger, @@ -258,10 +262,10 @@ func TestMongoWithGSSAPI(t *testing.T) { gc := newGeneralCollector(ctx, client, nodeType, e.opts.Logger) r := e.makeRegistry(ctx, client, new(labelsGetterMock), *e.opts) - expected := strings.NewReader(fmt.Sprintf(` + expected := strings.NewReader(` # HELP mongodb_up Whether MongoDB is up. # TYPE mongodb_up gauge - mongodb_up {cluster_role="mongod"} 1`) + "\n") + mongodb_up {cluster_role="mongod"} 1` + "\n") filter := []string{ "mongodb_up", @@ -270,7 +274,7 @@ func TestMongoWithGSSAPI(t *testing.T) { assert.NoError(t, err, "mongodb_up metric should be 1") res := r.Unregister(gc) - assert.Equal(t, true, res) + assert.True(t, res) } func TestMongoUpMetric(t *testing.T) { From b4fffedbf1c07396320680cfee0b6b4cd84d02da Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 22:47:03 +0100 Subject: [PATCH 18/29] split test function to satisfy linter --- exporter/exporter_test.go | 25 ++++++++++++++++--------- internal/tu/testutils.go | 1 + 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 3456b9f8d..ed31d262f 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -199,14 +199,9 @@ func TestMongoS(t *testing.T) { } } -func TestGSSAPIAuth(t *testing.T) { - logger := logrus.New() - logger.SetReportCaller(true) - +func generateKerberosConfigFile(t *testing.T) (*os.File, error) { kerberosHost, err := tu.IpForContainer("kerberos") require.NoError(t, err) - mongoHost, err := tu.IpForContainer("psmdb-kerberos") - require.NoError(t, err) config := fmt.Sprintf(` [libdefaults] @@ -226,15 +221,27 @@ func TestGSSAPIAuth(t *testing.T) { perconatest.com = PERCONATEST.COM %s = PERCONATEST.COM `, kerberosHost, kerberosHost) - configFile, err := os.Create(t.TempDir() + "/krb5.conf") require.NoError(t, err) + + _, err = configFile.WriteString(config) + require.NoError(t, err) + + return configFile, nil +} +func TestGSSAPIAuth(t *testing.T) { + logger := logrus.New() + logger.SetReportCaller(true) + + mongoHost, err := tu.IpForContainer("psmdb-kerberos") + require.NoError(t, err) + + configFile, err := generateKerberosConfigFile(t) + require.NoError(t, err) defer func() { _ = configFile.Close() _ = os.Setenv("KRB5_CONFIG", "") }() - _, err = configFile.WriteString(config) - require.NoError(t, err) t.Setenv("KRB5_CONFIG", configFile.Name()) ctx := context.Background() diff --git a/internal/tu/testutils.go b/internal/tu/testutils.go index 9cfebcaa8..fa3af43c9 100644 --- a/internal/tu/testutils.go +++ b/internal/tu/testutils.go @@ -199,6 +199,7 @@ func PortForContainer(name string) (string, error) { return ports[0].HostPort, nil } +// IpForContainer returns the IP address of a running container. func IpForContainer(name string) (string, error) { di, err := InspectContainer(name) if err != nil { From 5f131b2f8e46128f2cdb11cb1ac66d5fc7dd1a93 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 22:48:31 +0100 Subject: [PATCH 19/29] fix formatting --- exporter/exporter_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index ed31d262f..64f403b47 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -229,6 +229,7 @@ func generateKerberosConfigFile(t *testing.T) (*os.File, error) { return configFile, nil } + func TestGSSAPIAuth(t *testing.T) { logger := logrus.New() logger.SetReportCaller(true) From 436e06841170803e0a508d809821959c46634de6 Mon Sep 17 00:00:00 2001 From: idoko Date: Wed, 5 Mar 2025 22:55:29 +0100 Subject: [PATCH 20/29] remove unused changes --- .github/workflows/go.yml | 2 +- docker/kerberos-exporter.dockerfile | 5 ----- docker/scripts/setup-krb5-server.sh | 1 - exporter/exporter_test.go | 11 ++++++----- internal/tu/testutils.go | 4 ++-- 5 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 docker/kerberos-exporter.dockerfile diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ac6028a10..a04e17d3c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,7 +46,7 @@ jobs: with: go-version-file: ${{ github.workspace }}/go.mod - - name: Install jq tool + - name: Install kerberos development libraries run: | sudo apt-get update sudo apt-get install libkrb5-dev diff --git a/docker/kerberos-exporter.dockerfile b/docker/kerberos-exporter.dockerfile deleted file mode 100644 index 06ca13571..000000000 --- a/docker/kerberos-exporter.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM alpine -RUN apk add --no-cache ca-certificates -USER 65535:65535 -COPY ./mongodb_exporter / -EXPOSE 9216 diff --git a/docker/scripts/setup-krb5-server.sh b/docker/scripts/setup-krb5-server.sh index 960aaea86..5098b312a 100644 --- a/docker/scripts/setup-krb5-server.sh +++ b/docker/scripts/setup-krb5-server.sh @@ -2,7 +2,6 @@ mongohost=`getent hosts ${MONGO_HOST} | awk '{ print $1 }'` kerberos_host=`getent hosts ${KERBEROS_HOST} | awk '{ print $1 }'` -gateway_ip=`ip route | grep default | awk '{print $3}'` cat > /krb5/krb5.conf < Date: Mon, 10 Mar 2025 14:38:36 +0100 Subject: [PATCH 21/29] use prebuilt docker image and volume --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1d3c3c324..11faee0f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -287,8 +287,7 @@ services: command: /scripts/run-mongodb-encrypted.sh kerberos: - build: - dockerfile: ./docker/kerberos.dockerfile + image: ${TEST_KERBEROS_IMAGE:-ghcr.io/idoqo/kerberos:0.1.0} # todo: change after publishing to perconalab registry. container_name: kerberos hostname: kerberos environment: @@ -301,7 +300,7 @@ services: depends_on: - psmdb-kerberos volumes: - - ./docker/kerberos/cache:/krb5/ + - kerberos-config:/krb5/ - ./docker/scripts:/scripts ports: - "88:88/udp" @@ -326,11 +325,12 @@ services: - MONGO_INITDB_ROOT_PASSWORD=adminpassword volumes: - ./docker/scripts:/scripts - - ./docker/kerberos/cache:/krb5/ + - kerberos-config:/krb5/ command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 volumes: pbm-backups: + kerberos-config: networks: rs1: From 87b120675669f7c8d2d30bb0601a0dde5a77d9f3 Mon Sep 17 00:00:00 2001 From: idoko Date: Mon, 10 Mar 2025 14:42:44 +0100 Subject: [PATCH 22/29] rename 'docker' folder to 'test-setup' --- .gitignore | 2 -- Makefile | 4 +-- docker-compose.yml | 36 +++++++++---------- {docker => test-setup}/kerberos.dockerfile | 0 .../mongodb-auth.dockerfile | 2 +- {docker => test-setup}/pbm/config/pbm.yaml | 0 {docker => test-setup}/scripts/init-pbm.sh | 0 .../scripts/init-psmdb-kerberos.sh | 0 .../scripts/run-mongodb-encrypted.sh | 0 .../scripts/setup-krb5-mongo.sh | 0 .../scripts/setup-krb5-server.sh | 0 {docker => test-setup}/scripts/setup-shard.sh | 0 {docker => test-setup}/scripts/setup.sh | 0 {docker => test-setup}/secret/keyfile | 0 .../secret/mongodb_secrets.txt | 0 15 files changed, 21 insertions(+), 23 deletions(-) rename {docker => test-setup}/kerberos.dockerfile (100%) rename {docker => test-setup}/mongodb-auth.dockerfile (86%) rename {docker => test-setup}/pbm/config/pbm.yaml (100%) rename {docker => test-setup}/scripts/init-pbm.sh (100%) rename {docker => test-setup}/scripts/init-psmdb-kerberos.sh (100%) rename {docker => test-setup}/scripts/run-mongodb-encrypted.sh (100%) rename {docker => test-setup}/scripts/setup-krb5-mongo.sh (100%) rename {docker => test-setup}/scripts/setup-krb5-server.sh (100%) rename {docker => test-setup}/scripts/setup-shard.sh (100%) rename {docker => test-setup}/scripts/setup.sh (100%) rename {docker => test-setup}/secret/keyfile (100%) rename {docker => test-setup}/secret/mongodb_secrets.txt (100%) diff --git a/.gitignore b/.gitignore index c7baff44d..b49f65a8a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,3 @@ cover.out mongodb_exporter .DS_Store -docker/kerberos/cache/* - diff --git a/Makefile b/Makefile index 33e0dce88..12d1ac521 100644 --- a/Makefile +++ b/Makefile @@ -109,8 +109,8 @@ test-cover: env ## Run tests and collect cross-package coverage inf test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster docker compose up --build -d - ./docker/scripts/init-psmdb-kerberos.sh - ./docker/scripts/init-pbm.sh + ./test-setup/scripts/init-psmdb-kerberos.sh + ./test-setup/scripts/init-pbm.sh test-cluster-clean: env ## Stops MongoDB test cluster. docker compose down --remove-orphans --volumes diff --git a/docker-compose.yml b/docker-compose.yml index 11faee0f8..2bdc5e64b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: - "mongo-1-3" - "mongo-1-arbiter" volumes: - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts environment: - MONGO1=mongo-1-1 - MONGO2=mongo-1-2 @@ -63,7 +63,7 @@ services: mongo-2-2: container_name: "mongo-2-2" build: - dockerfile: ./docker/mongodb-auth.dockerfile + dockerfile: test-setup/mongodb-auth.dockerfile environment: - MONGO_INITDB_ROOT_USERNAME=${TEST_MONGODB_USERNAME:-admin} - MONGO_INITDB_ROOT_PASSWORD=${TEST_MONGODB_PASSWORD:-admin} @@ -76,7 +76,7 @@ services: mongo-2-3: container_name: "mongo-2-3" build: - dockerfile: ./docker/mongodb-auth.dockerfile + dockerfile: test-setup/mongodb-auth.dockerfile ports: - "${TEST_MONGODB_S2_SECONDARY1_PORT:-17005}:27017" command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile @@ -86,7 +86,7 @@ services: mongo-2-1: container_name: "mongo-2-1" build: - dockerfile: ./docker/mongodb-auth.dockerfile + dockerfile: test-setup/mongodb-auth.dockerfile ports: - "${TEST_MONGODB_S2_SECONDARY2_PORT:-17006}:27017" command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile @@ -96,7 +96,7 @@ services: mongo-2-arbiter: container_name: "mongo-2-arbiter" build: - dockerfile: ./docker/mongodb-auth.dockerfile + dockerfile: test-setup/mongodb-auth.dockerfile ports: - "${TEST_MONGODB_S2_ARBITER:-17012}:27017" command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile @@ -112,8 +112,8 @@ services: environment: - PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-1:27017 volumes: - - ./docker/pbm/config:/etc/config - - ./docker/scripts:/scripts + - ./test-setup/pbm/config:/etc/config + - ./test-setup/scripts:/scripts - pbm-backups:/opt/backups networks: - rs2 @@ -127,8 +127,8 @@ services: environment: - PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-2:27017 volumes: - - ./docker/pbm/config:/etc/config - - ./docker/scripts:/scripts + - ./test-setup/pbm/config:/etc/config + - ./test-setup/scripts:/scripts - pbm-backups:/opt/backups networks: - rs2 @@ -142,8 +142,8 @@ services: environment: - PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-3:27017 volumes: - - ./docker/pbm/config:/etc/config - - ./docker/scripts:/scripts + - ./test-setup/pbm/config:/etc/config + - ./test-setup/scripts:/scripts - pbm-backups:/opt/backups networks: - rs2 @@ -157,7 +157,7 @@ services: - "mongo-2-3" - "mongo-2-arbiter" volumes: - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts environment: - MONGO1=mongo-2-2 - MONGO2=mongo-2-1 @@ -210,7 +210,7 @@ services: - "mongo-cnf-2" - "mongo-cnf-3" volumes: - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts environment: - MONGO1=mongo-cnf-1 - MONGO2=mongo-cnf-2 @@ -249,7 +249,7 @@ services: - rs2 - cnf-serv volumes: - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts environment: - MONGOS=mongos - MONGO11=mongo-1-1 @@ -282,8 +282,8 @@ services: ports: - "${TEST_MONGODB_STANDALONE_ENCRYPTED_PORT:-27027}:27017" volumes: - - ./docker/secret/mongodb_secrets.txt:/secret/mongodb_secrets.txt - - ./docker/scripts:/scripts + - ./test-setup/secret/mongodb_secrets.txt:/secret/mongodb_secrets.txt + - ./test-setup/scripts:/scripts command: /scripts/run-mongodb-encrypted.sh kerberos: @@ -301,7 +301,7 @@ services: - psmdb-kerberos volumes: - kerberos-config:/krb5/ - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts ports: - "88:88/udp" entrypoint: [ "sh", "/scripts/setup-krb5-server.sh" ] @@ -324,7 +324,7 @@ services: - MONGO_INITDB_ROOT_USERNAME=admin - MONGO_INITDB_ROOT_PASSWORD=adminpassword volumes: - - ./docker/scripts:/scripts + - ./test-setup/scripts:/scripts - kerberos-config:/krb5/ command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 diff --git a/docker/kerberos.dockerfile b/test-setup/kerberos.dockerfile similarity index 100% rename from docker/kerberos.dockerfile rename to test-setup/kerberos.dockerfile diff --git a/docker/mongodb-auth.dockerfile b/test-setup/mongodb-auth.dockerfile similarity index 86% rename from docker/mongodb-auth.dockerfile rename to test-setup/mongodb-auth.dockerfile index 6933a8b44..905eadfda 100644 --- a/docker/mongodb-auth.dockerfile +++ b/test-setup/mongodb-auth.dockerfile @@ -1,7 +1,7 @@ ARG TEST_MONGODB_IMAGE=mongo:4.4 FROM ${TEST_MONGODB_IMAGE} USER root -COPY docker/secret/keyfile /opt/keyfile +COPY test-setup/secret/keyfile /opt/keyfile RUN chown mongodb /opt/keyfile && chmod 400 /opt/keyfile && mkdir -p /home/mongodb/ && chown mongodb /home/mongodb RUN mkdir /opt/backups && touch /opt/backups/.gitkeep && chown mongodb /opt/backups USER mongodb diff --git a/docker/pbm/config/pbm.yaml b/test-setup/pbm/config/pbm.yaml similarity index 100% rename from docker/pbm/config/pbm.yaml rename to test-setup/pbm/config/pbm.yaml diff --git a/docker/scripts/init-pbm.sh b/test-setup/scripts/init-pbm.sh similarity index 100% rename from docker/scripts/init-pbm.sh rename to test-setup/scripts/init-pbm.sh diff --git a/docker/scripts/init-psmdb-kerberos.sh b/test-setup/scripts/init-psmdb-kerberos.sh similarity index 100% rename from docker/scripts/init-psmdb-kerberos.sh rename to test-setup/scripts/init-psmdb-kerberos.sh diff --git a/docker/scripts/run-mongodb-encrypted.sh b/test-setup/scripts/run-mongodb-encrypted.sh similarity index 100% rename from docker/scripts/run-mongodb-encrypted.sh rename to test-setup/scripts/run-mongodb-encrypted.sh diff --git a/docker/scripts/setup-krb5-mongo.sh b/test-setup/scripts/setup-krb5-mongo.sh similarity index 100% rename from docker/scripts/setup-krb5-mongo.sh rename to test-setup/scripts/setup-krb5-mongo.sh diff --git a/docker/scripts/setup-krb5-server.sh b/test-setup/scripts/setup-krb5-server.sh similarity index 100% rename from docker/scripts/setup-krb5-server.sh rename to test-setup/scripts/setup-krb5-server.sh diff --git a/docker/scripts/setup-shard.sh b/test-setup/scripts/setup-shard.sh similarity index 100% rename from docker/scripts/setup-shard.sh rename to test-setup/scripts/setup-shard.sh diff --git a/docker/scripts/setup.sh b/test-setup/scripts/setup.sh similarity index 100% rename from docker/scripts/setup.sh rename to test-setup/scripts/setup.sh diff --git a/docker/secret/keyfile b/test-setup/secret/keyfile similarity index 100% rename from docker/secret/keyfile rename to test-setup/secret/keyfile diff --git a/docker/secret/mongodb_secrets.txt b/test-setup/secret/mongodb_secrets.txt similarity index 100% rename from docker/secret/mongodb_secrets.txt rename to test-setup/secret/mongodb_secrets.txt From e59c50ab4244ee5c8da17d63cfcc2a3ceee0e758 Mon Sep 17 00:00:00 2001 From: idoko Date: Mon, 10 Mar 2025 14:48:47 +0100 Subject: [PATCH 23/29] fix image in use --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2bdc5e64b..800c92e0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -287,7 +287,7 @@ services: command: /scripts/run-mongodb-encrypted.sh kerberos: - image: ${TEST_KERBEROS_IMAGE:-ghcr.io/idoqo/kerberos:0.1.0} # todo: change after publishing to perconalab registry. + image: ${TEST_KERBEROS_IMAGE:-idoko/kerberos:0.1.0} # todo: change after publishing to perconalab registry. container_name: kerberos hostname: kerberos environment: From 8568537e211ff35d60ff05df25ea9cc1b1656de9 Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 11 Mar 2025 05:39:02 +0100 Subject: [PATCH 24/29] build docker image --- docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 800c92e0e..48a06239b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -287,7 +287,8 @@ services: command: /scripts/run-mongodb-encrypted.sh kerberos: - image: ${TEST_KERBEROS_IMAGE:-idoko/kerberos:0.1.0} # todo: change after publishing to perconalab registry. + build: + dockerfile: test-setup/kerberos.dockerfile container_name: kerberos hostname: kerberos environment: @@ -300,7 +301,7 @@ services: depends_on: - psmdb-kerberos volumes: - - kerberos-config:/krb5/ + - kerberos-config:/krb5 - ./test-setup/scripts:/scripts ports: - "88:88/udp" @@ -325,7 +326,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=adminpassword volumes: - ./test-setup/scripts:/scripts - - kerberos-config:/krb5/ + - kerberos-config:/krb5 command: > mongod --bind_ip_all --auth --setParameter authenticationMechanisms=GSSAPI,SCRAM-SHA-256 volumes: From 5da2a9d5b69ef9325b9c334706aa040001e21d0d Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 11 Mar 2025 09:17:57 +0100 Subject: [PATCH 25/29] enable CGO in goreleaser --- .goreleaser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 02c76ae49..9d91ac305 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -3,7 +3,7 @@ version: 2 builds: - binary: mongodb_exporter env: - - CGO_ENABLED=0 + - CGO_ENABLED=1 goos: - linux - darwin From 813c7ba0e187856aaec3813a7b29dfc69c77f2cc Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 11 Mar 2025 10:00:00 +0100 Subject: [PATCH 26/29] fix releaser tags --- .goreleaser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 9d91ac305..04a4e3230 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -19,7 +19,7 @@ builds: goarch: arm flags: - -v - - -tags gssapi + - -tags=gssapi ldflags: - -s -w -X main.version=v{{.Version}} -X main.commit={{.ShortCommit}} -X main.buildDate={{.Date}} archives: From 859c765b0e36117ecb2d67a0034e033853bb5577 Mon Sep 17 00:00:00 2001 From: idoko Date: Tue, 11 Mar 2025 11:07:00 +0100 Subject: [PATCH 27/29] force cgo during build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12d1ac521..226504168 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ init: ## Install linters cd tools && go generate -x -tags=tools build: ## Compile using plain go build - go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter -tags gssapi + env CGO_ENABLED=1 go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter -tags gssapi release: ## Build the binaries using goreleaser docker run --rm --privileged \ From b44322b74fa2105e8b931f1b79823c1f08aa8539 Mon Sep 17 00:00:00 2001 From: idoko Date: Thu, 13 Mar 2025 02:37:34 +0100 Subject: [PATCH 28/29] add timeout to script --- Makefile | 2 +- exporter/exporter_test.go | 2 +- test-setup/scripts/setup-krb5-mongo.sh | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 226504168..80e016119 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ init: ## Install linters cd tools && go generate -x -tags=tools build: ## Compile using plain go build - env CGO_ENABLED=1 go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter -tags gssapi + CGO_ENABLED=1 go build -ldflags="$(GO_BUILD_LDFLAGS)" -o $(PMM_RELEASE_PATH)/mongodb_exporter -tags gssapi release: ## Build the binaries using goreleaser docker run --rm --privileged \ diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 421c01af1..fa5ff2c8e 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -280,7 +280,7 @@ func TestGSSAPIAuth(t *testing.T) { "mongodb_up", } err = testutil.CollectAndCompare(gc, expected, filter...) - assert.NoError(t, err, "mongodb_up metric should be 1") + require.NoError(t, err, "mongodb_up metric should be 1") res := r.Unregister(gc) assert.True(t, res) diff --git a/test-setup/scripts/setup-krb5-mongo.sh b/test-setup/scripts/setup-krb5-mongo.sh index 11e913581..acc5876d8 100755 --- a/test-setup/scripts/setup-krb5-mongo.sh +++ b/test-setup/scripts/setup-krb5-mongo.sh @@ -3,10 +3,17 @@ username=${MONGO_INITDB_ROOT_USERNAME} password=${MONGO_INITDB_ROOT_PASSWORD} -echo "Waiting for startup.." +echo "Waiting for startup..." +max_attempts=20 +attempts=0 until mongosh --host 127.0.0.1:27017 -u ${username} -p ${password} --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do + if [ $attempts -eq $max_attempts ]; then + echo "Failed to check MongoDB status after $max_attempts attempts" + exit 1 + fi printf '.' sleep 1 + attempts=$((attempts+1)) done echo "Started.." From 811436a446feead2a69f36713a703e78aa425130 Mon Sep 17 00:00:00 2001 From: idoko Date: Thu, 24 Apr 2025 14:55:09 +0100 Subject: [PATCH 29/29] remove logrus --- exporter/exporter_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 33d164baa..428b335f3 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -232,8 +232,7 @@ func generateKerberosConfigFile(t *testing.T) *os.File { } func TestGSSAPIAuth(t *testing.T) { - logger := logrus.New() - logger.SetReportCaller(true) + logger := promslog.New(&promslog.Config{}) mongoHost, err := tu.IPForContainer("psmdb-kerberos") require.NoError(t, err)