From c580ba8a11e76a52a67c2ed8dd6b3948f3d14b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Casta=C3=B1o=20Arteaga?= Date: Thu, 25 May 2023 14:14:49 +0200 Subject: [PATCH] Automate annual review notifications (#1102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergio Castaño Arteaga --- .github/dependabot.yml | 5 + .github/workflows/ci.yml | 27 + Cargo.lock | 467 ++++++++++++++++++ Cargo.toml | 2 + chart/templates/notifier_cronjob.yaml | 41 ++ chart/templates/notifier_secret.yaml | 21 + chart/values-staging.yaml | 7 + chart/values.yaml | 10 + clomonitor-apiserver/Dockerfile | 1 + clomonitor-archiver/Cargo.toml | 2 +- clomonitor-archiver/Dockerfile | 1 + clomonitor-linter/Dockerfile | 1 + clomonitor-notifier/Cargo.toml | 31 ++ clomonitor-notifier/Dockerfile | 21 + clomonitor-notifier/src/db.rs | 117 +++++ clomonitor-notifier/src/github.rs | 120 +++++ clomonitor-notifier/src/main.rs | 65 +++ clomonitor-notifier/src/notifier.rs | 323 ++++++++++++ clomonitor-notifier/src/tmpl.rs | 11 + .../templates/annual-review-due-reminder.md | 5 + .../templates/annual-review-due.md | 7 + clomonitor-registrar/Dockerfile | 1 + clomonitor-tracker/Dockerfile | 1 + .../functions/001_load_functions.sql | 1 + ...et_pending_annual_review_notifications.sql | 50 ++ .../009_annual_review_notifications.sql | 12 + ...et_pending_annual_review_notifications.sql | 329 ++++++++++++ database/tests/schema/schema.sql | 4 +- web/src/data.tsx | 2 +- 29 files changed, 1682 insertions(+), 3 deletions(-) create mode 100644 chart/templates/notifier_cronjob.yaml create mode 100644 chart/templates/notifier_secret.yaml create mode 100644 clomonitor-notifier/Cargo.toml create mode 100644 clomonitor-notifier/Dockerfile create mode 100644 clomonitor-notifier/src/db.rs create mode 100644 clomonitor-notifier/src/github.rs create mode 100644 clomonitor-notifier/src/main.rs create mode 100644 clomonitor-notifier/src/notifier.rs create mode 100644 clomonitor-notifier/src/tmpl.rs create mode 100644 clomonitor-notifier/templates/annual-review-due-reminder.md create mode 100644 clomonitor-notifier/templates/annual-review-due.md create mode 100644 database/migrations/functions/notifications/get_pending_annual_review_notifications.sql create mode 100644 database/migrations/schema/009_annual_review_notifications.sql create mode 100644 database/tests/functions/notifications/get_pending_annual_review_notifications.sql diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f8b17e90..b43a93c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -37,6 +37,11 @@ updates: schedule: interval: "weekly" day: "wednesday" + - package-ecosystem: "docker" + directory: "/clomonitor-notifier" + schedule: + interval: "weekly" + day: "wednesday" - package-ecosystem: "docker" directory: "/clomonitor-registrar" schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 518672ba..e3cc0b90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,33 @@ jobs: docker build -f clomonitor-linter/Dockerfile -t public.ecr.aws/g6m3a0y9/linter:latest . docker push public.ecr.aws/g6m3a0y9/linter:latest + build-notifier-image: + if: github.ref == 'refs/heads/main' + needs: + - linter-backend + - tests-database + - tests-backend + - tests-frontend + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Build and push image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + docker build -f clomonitor-notifier/Dockerfile -t $ECR_REGISTRY/notifier:$GITHUB_SHA . + docker push $ECR_REGISTRY/notifier:$GITHUB_SHA + build-registrar-image: if: github.ref == 'refs/heads/main' needs: diff --git a/Cargo.lock b/Cargo.lock index eb9e5203..809b5870 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,6 +48,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.3.0" @@ -207,6 +216,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-recursion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.13", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -441,6 +461,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + [[package]] name = "clap" version = "4.2.7" @@ -587,6 +620,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "clomonitor-notifier" +version = "1.0.0" +dependencies = [ + "anyhow", + "askama", + "askama_axum", + "async-trait", + "clap", + "config", + "deadpool", + "deadpool-postgres", + "futures", + "lazy_static", + "mockall", + "octorust", + "openssl", + "postgres-openssl", + "regex", + "tokio", + "tokio-postgres", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "clomonitor-registrar" version = "1.0.0" @@ -938,6 +997,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "dyn-clone" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" + [[package]] name = "either" version = "1.8.1" @@ -1503,6 +1568,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1516,6 +1594,29 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1659,6 +1760,20 @@ dependencies = [ "serde", ] +[[package]] +name = "jsonwebtoken" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +dependencies = [ + "base64 0.21.0", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "kurbo" version = "0.8.3" @@ -1755,6 +1870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", + "serde", ] [[package]] @@ -2012,6 +2128,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -2031,6 +2168,36 @@ dependencies = [ "libc", ] +[[package]] +name = "octorust" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fac860084f1858d4207a3242df22e2a60dbae86d3df6878ce8eac2a51eb5c9" +dependencies = [ + "anyhow", + "async-recursion", + "chrono", + "http", + "jsonwebtoken", + "log", + "mime", + "parse_link_header", + "pem", + "percent-encoding", + "reqwest", + "reqwest-conditional-middleware", + "reqwest-middleware", + "reqwest-retry", + "reqwest-tracing", + "ring", + "schemars", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "url", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -2091,6 +2258,25 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "thiserror", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2136,6 +2322,17 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "parse_link_header" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3687fe9debbbf2a019f381a8bc6b42049b22647449b39af54b3013985c0cf6de" +dependencies = [ + "http", + "lazy_static", + "regex", +] + [[package]] name = "paste" version = "1.0.9" @@ -2148,6 +2345,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2596,28 +2802,98 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "reqwest-conditional-middleware" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bce134f515eb4c2748bbd928086e7b0aae0d1568daf6c63b51e829aa6f2cf464" +dependencies = [ + "async-trait", + "reqwest", + "reqwest-middleware", + "task-local-extensions", +] + +[[package]] +name = "reqwest-middleware" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69539cea4148dce683bec9dc95be3f0397a9bb2c248a49c8296a9d21659a8cdd" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + +[[package]] +name = "reqwest-retry" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce246a729eaa6aff5e215aee42845bf5fed9893cc6cd51aeeb712f34e04dd9f3" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "http", + "hyper", + "reqwest", + "reqwest-middleware", + "retry-policies", + "task-local-extensions", + "tokio", + "tracing", +] + +[[package]] +name = "reqwest-tracing" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64977f9a47fa7768cc88751e29026e569730ac1667c2eaeaac04b32624849fbe" +dependencies = [ + "async-trait", + "opentelemetry", + "reqwest", + "reqwest-middleware", + "task-local-extensions", + "tokio", + "tracing", + "tracing-opentelemetry", +] + [[package]] name = "resvg" version = "0.27.0" @@ -2642,6 +2918,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" +[[package]] +name = "retry-policies" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e09bbcb5003282bcb688f0bae741b278e9c7e8f378f561522c9806c58e075d9b" +dependencies = [ + "anyhow", + "chrono", + "rand 0.8.5", +] + [[package]] name = "rgb" version = "0.8.34" @@ -2651,6 +2938,21 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rmp" version = "0.8.11" @@ -2726,6 +3028,37 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.0", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2773,12 +3106,50 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "schemars" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +dependencies = [ + "bytes", + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.8.2" @@ -2822,6 +3193,17 @@ dependencies = [ "syn 2.0.13", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_json" version = "1.0.96" @@ -2945,6 +3327,18 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "simplecss" version = "0.2.1" @@ -3001,6 +3395,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strict-num" version = "0.1.0" @@ -3098,6 +3498,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "task-local-extensions" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8" +dependencies = [ + "pin-utils", +] + [[package]] name = "tempfile" version = "3.5.0" @@ -3306,6 +3715,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.7" @@ -3428,6 +3847,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -3614,6 +4047,12 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -3807,6 +4246,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "weezl" version = "0.1.7" @@ -3855,6 +4313,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.36.1" diff --git a/Cargo.toml b/Cargo.toml index 0f1c22e6..caf57981 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "clomonitor-archiver", "clomonitor-core", "clomonitor-linter", + "clomonitor-notifier", "clomonitor-registrar", "clomonitor-tracker", ] @@ -42,6 +43,7 @@ metrics-exporter-prometheus = "0.12.1" mime = "0.3.17" mockall = "0.11.4" mockito = "1.0.2" +octorust = "0.3.2" openssl = { version = "0.10.52", features = ["vendored"] } postgres-openssl = "0.5.0" postgres-types = { version = "0.2.5", features = ["derive"] } diff --git a/chart/templates/notifier_cronjob.yaml b/chart/templates/notifier_cronjob.yaml new file mode 100644 index 00000000..097caa67 --- /dev/null +++ b/chart/templates/notifier_cronjob.yaml @@ -0,0 +1,41 @@ +{{- if .Values.notifier.enabled -}} +{{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }} +apiVersion: batch/v1 +{{- else }} +apiVersion: batch/v1beta1 +{{- end }} +kind: CronJob +metadata: + name: {{ include "chart.resourceNamePrefix" . }}notifier +spec: + schedule: "0 * * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + restartPolicy: Never + initContainers: + - {{- include "chart.checkDbIsReadyInitContainer" . | nindent 14 }} + containers: + - name: notifier + image: {{ .Values.notifier.cronjob.image.repository }}:{{ .Values.imageTag | default (printf "v%s" .Chart.AppVersion) }} + imagePullPolicy: {{ .Values.pullPolicy }} + resources: + {{- toYaml .Values.notifier.cronjob.resources | nindent 16 }} + volumeMounts: + - name: notifier-config + mountPath: {{ .Values.configDir | quote }} + readOnly: true + command: ['clomonitor-notifier', '-c', '{{ .Values.configDir }}/notifier.yaml'] + volumes: + - name: notifier-config + secret: + secretName: {{ include "chart.resourceNamePrefix" . }}notifier-config +{{- end }} diff --git a/chart/templates/notifier_secret.yaml b/chart/templates/notifier_secret.yaml new file mode 100644 index 00000000..462e7059 --- /dev/null +++ b/chart/templates/notifier_secret.yaml @@ -0,0 +1,21 @@ +{{- if .Values.notifier.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "chart.resourceNamePrefix" . }}notifier-config +type: Opaque +stringData: + notifier.yaml: |- + db: + host: {{ default (printf "%s-postgresql.%s" .Release.Name .Release.Namespace) .Values.db.host }} + port: {{ .Values.db.port }} + dbname: {{ .Values.db.dbname }} + user: {{ .Values.db.user }} + password: {{ .Values.db.password }} + creds: + githubToken: {{ .Values.creds.notifierGithubToken }} + log: + format: {{ .Values.log.format }} + notifier: + enabled: {{ .Values.notifier.enabled }} +{{- end }} diff --git a/chart/values-staging.yaml b/chart/values-staging.yaml index a80f269d..9ece9032 100644 --- a/chart/values-staging.yaml +++ b/chart/values-staging.yaml @@ -38,6 +38,13 @@ archiver: cpu: 1 memory: 100Mi +notifier: + cronjob: + resources: + requests: + cpu: 1 + memory: 100Mi + registrar: cronjob: resources: diff --git a/chart/values.yaml b/chart/values.yaml index 8f30a886..af9f7ff3 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -29,6 +29,7 @@ db: # Credentials creds: githubTokens: [] + notifierGithubToken: null # Log configuration log: @@ -81,6 +82,15 @@ archiver: repository: clomonitor/archiver resources: {} +# Notifier configuration +notifier: + enabled: false + cronjob: + image: + # Notifier image repository (without the tag) + repository: clomonitor/notifier + resources: {} + # Registrar configuration registrar: cronjob: diff --git a/clomonitor-apiserver/Dockerfile b/clomonitor-apiserver/Dockerfile index cc5e44b5..4b661c22 100644 --- a/clomonitor-apiserver/Dockerfile +++ b/clomonitor-apiserver/Dockerfile @@ -7,6 +7,7 @@ COPY clomonitor-apiserver clomonitor-apiserver COPY clomonitor-archiver/Cargo.* clomonitor-archiver COPY clomonitor-core clomonitor-core COPY clomonitor-linter/Cargo.* clomonitor-linter +COPY clomonitor-notifier/Cargo.* clomonitor-notifier COPY clomonitor-registrar/Cargo.* clomonitor-registrar COPY clomonitor-tracker/Cargo.* clomonitor-tracker WORKDIR /clomonitor/clomonitor-apiserver diff --git a/clomonitor-archiver/Cargo.toml b/clomonitor-archiver/Cargo.toml index f15dba82..9de4d347 100644 --- a/clomonitor-archiver/Cargo.toml +++ b/clomonitor-archiver/Cargo.toml @@ -23,6 +23,6 @@ tracing-subscriber = { workspace = true } uuid = { workspace = true } [dev-dependencies] +futures = { workspace = true } lazy_static = { workspace = true } mockall = { workspace = true } -futures = { workspace = true } diff --git a/clomonitor-archiver/Dockerfile b/clomonitor-archiver/Dockerfile index 257f7cb5..90cad8a1 100644 --- a/clomonitor-archiver/Dockerfile +++ b/clomonitor-archiver/Dockerfile @@ -7,6 +7,7 @@ COPY clomonitor-apiserver/Cargo.* clomonitor-apiserver COPY clomonitor-archiver clomonitor-archiver COPY clomonitor-core/Cargo.* clomonitor-core COPY clomonitor-linter/Cargo.* clomonitor-linter +COPY clomonitor-notifier/Cargo.* clomonitor-notifier COPY clomonitor-registrar/Cargo.* clomonitor-registrar COPY clomonitor-tracker/Cargo.* clomonitor-tracker WORKDIR /clomonitor/clomonitor-archiver diff --git a/clomonitor-linter/Dockerfile b/clomonitor-linter/Dockerfile index d97a2eed..4dc06955 100644 --- a/clomonitor-linter/Dockerfile +++ b/clomonitor-linter/Dockerfile @@ -7,6 +7,7 @@ COPY clomonitor-apiserver/Cargo.* clomonitor-apiserver COPY clomonitor-archiver/Cargo.* clomonitor-archiver COPY clomonitor-core clomonitor-core COPY clomonitor-linter clomonitor-linter +COPY clomonitor-notifier/Cargo.* clomonitor-notifier COPY clomonitor-registrar/Cargo.* clomonitor-registrar COPY clomonitor-tracker/Cargo.* clomonitor-tracker WORKDIR /clomonitor/clomonitor-linter diff --git a/clomonitor-notifier/Cargo.toml b/clomonitor-notifier/Cargo.toml new file mode 100644 index 00000000..a5c4021b --- /dev/null +++ b/clomonitor-notifier/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "clomonitor-notifier" +description = "A tool that notifies projects about certain events" +version.workspace = true +license.workspace = true +edition.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +askama = { workspace = true } +askama_axum = { workspace = true } +async-trait = { workspace = true } +clap = { workspace = true } +config = { workspace = true } +deadpool = { workspace = true } +deadpool-postgres = { workspace = true } +lazy_static = { workspace = true } +octorust = { workspace = true } +openssl = { workspace = true } +postgres-openssl = { workspace = true } +regex = { workspace = true } +tokio = { workspace = true } +tokio-postgres = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +mockall = { workspace = true } diff --git a/clomonitor-notifier/Dockerfile b/clomonitor-notifier/Dockerfile new file mode 100644 index 00000000..e3ae6a1b --- /dev/null +++ b/clomonitor-notifier/Dockerfile @@ -0,0 +1,21 @@ +# Build notifier +FROM rust:1-alpine3.17 as builder +RUN apk --no-cache add musl-dev perl make +WORKDIR /clomonitor +COPY Cargo.* ./ +COPY clomonitor-apiserver/Cargo.* clomonitor-apiserver +COPY clomonitor-archiver/Cargo.* clomonitor-archiver +COPY clomonitor-core/Cargo.* clomonitor-core +COPY clomonitor-linter/Cargo.* clomonitor-linter +COPY clomonitor-notifier clomonitor-notifier +COPY clomonitor-registrar/Cargo.* clomonitor-registrar +COPY clomonitor-tracker/Cargo.* clomonitor-tracker +WORKDIR /clomonitor/clomonitor-notifier +RUN cargo build --release + +# Final stage +FROM alpine:3.17.3 +RUN apk --no-cache add ca-certificates git && addgroup -S clomonitor && adduser -S clomonitor -G clomonitor +USER clomonitor +WORKDIR /home/clomonitor +COPY --from=builder /clomonitor/target/release/clomonitor-notifier /usr/local/bin diff --git a/clomonitor-notifier/src/db.rs b/clomonitor-notifier/src/db.rs new file mode 100644 index 00000000..5db56c7d --- /dev/null +++ b/clomonitor-notifier/src/db.rs @@ -0,0 +1,117 @@ +use crate::notifier::AnnualReviewNotification; +use anyhow::Result; +use async_trait::async_trait; +use deadpool_postgres::Pool; +#[cfg(test)] +use mockall::automock; +use uuid::Uuid; + +/// Trait that defines some operations a DB implementation must support. +#[async_trait] +#[cfg_attr(test, automock)] +pub(crate) trait DB { + /// Returns the pending annual review notifications. + async fn get_pending_annual_review_notifications( + &self, + ) -> Result>; + + /// Pre-register annual review notification. + async fn pre_register_annual_review_notification( + &self, + project_id: &Uuid, + repository_url: &str, + ) -> Result; + + /// Update annual review notification details. + async fn update_annual_review_notification( + &self, + notification_id: &Uuid, + issue_number: Option, + comment_id: Option, + ) -> Result<()>; +} + +/// Type alias to represent a DB trait object. +pub(crate) type DynDB = Box; + +/// DB implementation backed by PostgreSQL. +pub(crate) struct PgDB { + pool: Pool, +} + +impl PgDB { + /// Create a new PgDB instance. + pub(crate) fn new(pool: Pool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl DB for PgDB { + /// [DB::get_pending_annual_review_notifications] + async fn get_pending_annual_review_notifications( + &self, + ) -> Result> { + let db = self.pool.get().await?; + let pending_notifications = db + .query( + "select * from get_pending_annual_review_notifications()", + &[], + ) + .await? + .iter() + .map(|row| AnnualReviewNotification { + project_id: row.get("project_id"), + community_repo_url: row.get("community_repo_url"), + issue_number: row.get("issue_number"), + }) + .collect(); + Ok(pending_notifications) + } + + /// [DB::pre_register_annual_review_notification] + async fn pre_register_annual_review_notification( + &self, + project_id: &Uuid, + repository_url: &str, + ) -> Result { + let db = self.pool.get().await?; + let notification_id = db + .query_one( + " + insert into annual_review_notification ( + project_id, + repository_url + ) values ( + $1::uuid, + $2::text + ) returning annual_review_notification_id; + ", + &[&project_id, &repository_url], + ) + .await? + .get("annual_review_notification_id"); + Ok(notification_id) + } + + /// [DB::update_annual_review_notification] + async fn update_annual_review_notification( + &self, + notification_id: &Uuid, + issue_number: Option, + comment_id: Option, + ) -> Result<()> { + let db = self.pool.get().await?; + db.execute( + " + update annual_review_notification set + issue_number = $1::bigint, + comment_id = $2::bigint + where annual_review_notification_id = $3::uuid; + ", + &[&issue_number, &comment_id, ¬ification_id], + ) + .await?; + Ok(()) + } +} diff --git a/clomonitor-notifier/src/github.rs b/clomonitor-notifier/src/github.rs new file mode 100644 index 00000000..c40f84ba --- /dev/null +++ b/clomonitor-notifier/src/github.rs @@ -0,0 +1,120 @@ +use anyhow::Result; +use async_trait::async_trait; +use config::Config; +#[cfg(test)] +use mockall::automock; +use octorust::{ + auth::Credentials, + types::{IssuesCreateRequest, PullsUpdateReviewRequest, TitleOneOf}, + Client, +}; + +/// Trait that defines some operations a GH implementation must support. +#[async_trait] +#[cfg_attr(test, automock)] +pub(crate) trait GH { + /// Create an issue comment. + async fn create_comment( + &self, + owner: &str, + repo: &str, + issue_number: IssueNumber, + body: &str, + ) -> Result; + + /// Create an issue. + async fn create_issue( + &self, + owner: &str, + repo: &str, + title: &str, + body: &str, + ) -> Result; + + /// Check if the issue provided is closed. + async fn is_issue_closed( + &self, + owner: &str, + repo: &str, + issue_number: IssueNumber, + ) -> Result; +} + +/// Type alias to represent a GH trait object. +pub(crate) type DynGH = Box; + +/// Type alias to represent an issue number. +type IssueNumber = i64; + +/// Type alias to represent a comment id. +type CommentId = i64; + +/// GH implementation backed by the GitHub API. +pub(crate) struct GHApi { + client: Client, +} + +impl GHApi { + /// Create a new GHApi instance. + pub(crate) fn new(cfg: &Config) -> Result { + let token = cfg.get_string("creds.githubToken")?; + let client = Client::new( + format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), + Credentials::Token(token), + )?; + Ok(Self { client }) + } +} + +#[async_trait] +impl GH for GHApi { + /// [GH::create_comment] + async fn create_comment( + &self, + owner: &str, + repo: &str, + issue_number: i64, + body: &str, + ) -> Result { + let body = &PullsUpdateReviewRequest { + body: body.to_string(), + }; + let comment = self + .client + .issues() + .create_comment(owner, repo, issue_number, body) + .await?; + Ok(comment.id) + } + + /// [GH::create_issue] + async fn create_issue( + &self, + owner: &str, + repo: &str, + title: &str, + body: &str, + ) -> Result { + let body = IssuesCreateRequest { + assignee: "".to_string(), + assignees: vec![], + body: body.to_string(), + labels: vec![], + milestone: None, + title: TitleOneOf::String(title.to_string()), + }; + let issue = self.client.issues().create(owner, repo, &body).await?; + Ok(issue.number) + } + + /// [GH::is_issue_closed] + async fn is_issue_closed( + &self, + owner: &str, + repo: &str, + issue_number: IssueNumber, + ) -> Result { + let issue = self.client.issues().get(owner, repo, issue_number).await?; + Ok(issue.closed_at.is_some()) + } +} diff --git a/clomonitor-notifier/src/main.rs b/clomonitor-notifier/src/main.rs new file mode 100644 index 00000000..927273c2 --- /dev/null +++ b/clomonitor-notifier/src/main.rs @@ -0,0 +1,65 @@ +use crate::{db::PgDB, github::GHApi}; +use anyhow::{Context, Result}; +use clap::Parser; +use config::{Config, File}; +use deadpool_postgres::{Config as DbConfig, Runtime}; +use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode}; +use postgres_openssl::MakeTlsConnector; +use std::path::PathBuf; +use tracing::{debug, info}; +use tracing_subscriber::EnvFilter; + +mod db; +mod github; +mod notifier; +mod tmpl; + +#[derive(Debug, Parser)] +#[clap(author, version, about)] +struct Args { + /// Config file path + #[clap(short, long)] + config: PathBuf, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + + // Setup configuration + let cfg = Config::builder() + .add_source(File::from(args.config)) + .build() + .context("error setting up configuration")?; + + // Setup logging + if std::env::var_os("RUST_LOG").is_none() { + std::env::set_var("RUST_LOG", "clomonitor_notifier=debug") + } + let s = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()); + match cfg.get_string("log.format").as_deref() { + Ok("json") => s.json().init(), + _ => s.init(), + }; + + // Setup database + debug!("setting up database"); + let mut builder = SslConnector::builder(SslMethod::tls())?; + builder.set_verify(SslVerifyMode::NONE); + let connector = MakeTlsConnector::new(builder.build()); + let db_cfg: DbConfig = cfg.get("db")?; + let pool = db_cfg.create_pool(Some(Runtime::Tokio1), connector)?; + let db = Box::new(PgDB::new(pool)); + + // Setup GitHub client + let gh = Box::new(GHApi::new(&cfg).context("error setting up github client")?); + + // Run notifier + if cfg.get_bool("notifier.enabled").unwrap_or(false) { + notifier::run(&cfg, db, gh).await?; + } else { + info!("notifier not enabled, exiting..."); + } + + Ok(()) +} diff --git a/clomonitor-notifier/src/notifier.rs b/clomonitor-notifier/src/notifier.rs new file mode 100644 index 00000000..0b0b9d11 --- /dev/null +++ b/clomonitor-notifier/src/notifier.rs @@ -0,0 +1,323 @@ +use crate::{db::DynDB, github::DynGH, tmpl}; +use anyhow::{format_err, Result}; +use askama::Template; +use config::Config; +use lazy_static::lazy_static; +use regex::Regex; +use std::time::Duration; +use tokio::time::sleep; +use tracing::{info, instrument}; +use uuid::Uuid; + +/// Process pending notifications. +#[instrument(skip_all, err)] +pub(crate) async fn run(cfg: &Config, db: DynDB, gh: DynGH) -> Result<()> { + info!("started"); + + process_annual_review_notifications(cfg, db, gh).await?; + + info!("finished"); + Ok(()) +} + +/// Title used in the issues created to notify that the annual review is due. +const ANNUAL_REVIEW_DUE_TITLE: &str = "CNCF TOC annual review due"; + +/// Information needed to send an annual review notification. +pub(crate) struct AnnualReviewNotification { + pub project_id: Uuid, + pub community_repo_url: String, + pub issue_number: Option, +} + +/// Process annual review notifications. +#[instrument(skip_all, err)] +async fn process_annual_review_notifications(cfg: &Config, db: DynDB, gh: DynGH) -> Result<()> { + match db.get_pending_annual_review_notifications().await { + Ok(mut notifications) => { + // If a list of allowed repositories is provided, filter out + // notifications whose repository isn't listed on it + if let Ok(allowed_repos) = cfg.get::>("notifier.allowedRepositories") { + notifications.retain(|n| allowed_repos.contains(&n.community_repo_url)) + }; + + // Process pending notifications + for (i, n) in notifications.iter().enumerate() { + info!(?n.project_id, ?n.community_repo_url, "processing pending annual review notification"); + + // Extract owner and repo from url + let Ok((owner, repo)) = get_owner_and_repo(&n.community_repo_url) else { continue }; + + // Pre-register notification in database + // (to avoid sending multiple notifications if registration failed after sending) + let notification_id = db + .pre_register_annual_review_notification(&n.project_id, &n.community_repo_url) + .await?; + + // If the notification contains an issue number, check if it's closed + let mut is_issue_closed = false; + if let Some(issue_number) = n.issue_number { + is_issue_closed = gh.is_issue_closed(&owner, &repo, issue_number).await?; + } + + // Send notification + let issue_number; + let mut comment_id = None; + if n.issue_number.is_none() || is_issue_closed { + // Create new issue + let body = tmpl::AnnualReviewDue {}.render().unwrap(); + issue_number = Some( + gh.create_issue(&owner, &repo, ANNUAL_REVIEW_DUE_TITLE, &body) + .await?, + ); + info!( + ?owner, + ?repo, + ?issue_number, + "annual review due notification sent" + ); + } else { + // Post comment in existing issue + issue_number = Some(n.issue_number.unwrap()); + let body = tmpl::AnnualReviewDueReminder {}.render().unwrap(); + comment_id = Some( + gh.create_comment(&owner, &repo, issue_number.unwrap(), &body) + .await?, + ); + info!( + ?owner, + ?repo, + ?issue_number, + ?comment_id, + "annual review due reminder notification sent" + ); + } + + // Update notification details in database + db.update_annual_review_notification(¬ification_id, issue_number, comment_id) + .await?; + + // If there are more notifications to process, pause before the + // next one to avoid hitting GitHub secondary rate limits + // https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits + if i < notifications.len() - 1 { + sleep(Duration::from_secs(10)).await; + } + } + + Ok(()) + } + Err(err) => Err(err), + } +} + +lazy_static! { + static ref GITHUB_REPO_URL: Regex = + Regex::new("^https://github.com/(?P[^/]+)/(?P[^/]+)/?$") + .expect("exprs in GITHUB_REPO_URL to be valid"); +} + +/// Extract the owner and repository from the repository url provided. +fn get_owner_and_repo(repo_url: &str) -> Result<(String, String)> { + let c = GITHUB_REPO_URL + .captures(repo_url) + .ok_or_else(|| format_err!("invalid repository url"))?; + Ok((c["owner"].to_string(), c["repo"].to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{db::MockDB, github::MockGH}; + use futures::future; + use mockall::predicate::eq; + + const FAKE_ERROR: &str = "fake error"; + const REPO1_URL: &str = "https://github.com/owner/repo1"; + const REPO2_URL: &str = "https://github.com/owner/repo2"; + const ISSUE_NUMBER: i64 = 1; + const COMMENT_ID: i64 = 1234; + + lazy_static! { + static ref PROJECT_ID: Uuid = + Uuid::parse_str("00000000-0001-0000-0000-000000000000").unwrap(); + static ref NOTIFICATION_ID: Uuid = + Uuid::parse_str("00000000-0001-0000-0000-000000000000").unwrap(); + } + + #[tokio::test] + async fn error_getting_pending_annual_review_notifications() { + let cfg = Config::builder().build().unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| Box::pin(future::ready(Err(format_err!(FAKE_ERROR))))); + + let gh = MockGH::new(); + + let result = run(&cfg, Box::new(db), Box::new(gh)).await; + assert_eq!(result.unwrap_err().root_cause().to_string(), FAKE_ERROR); + } + + #[tokio::test] + async fn no_pending_annual_review_notifications() { + let cfg = Config::builder().build().unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| Box::pin(future::ready(Ok(vec![])))); + + let gh = MockGH::new(); + + run(&cfg, Box::new(db), Box::new(gh)).await.unwrap(); + } + + #[tokio::test] + async fn filter_out_repo_not_in_allowed_repositories() { + let cfg = Config::builder() + .set_default("notifier.allowedRepositories", vec![REPO2_URL]) + .unwrap() + .build() + .unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| { + Box::pin(future::ready(Ok(vec![AnnualReviewNotification { + project_id: *PROJECT_ID, + community_repo_url: REPO1_URL.to_string(), + issue_number: None, + }]))) + }); + + let gh = MockGH::new(); + + run(&cfg, Box::new(db), Box::new(gh)).await.unwrap(); + } + + #[tokio::test] + async fn create_new_issue_because_none_was_provided() { + let cfg = Config::builder().build().unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| { + Box::pin(future::ready(Ok(vec![AnnualReviewNotification { + project_id: *PROJECT_ID, + community_repo_url: REPO1_URL.to_string(), + issue_number: None, + }]))) + }); + db.expect_pre_register_annual_review_notification() + .with(eq(*PROJECT_ID), eq(REPO1_URL)) + .times(1) + .returning(|_, _| Box::pin(future::ready(Ok(*NOTIFICATION_ID)))); + db.expect_update_annual_review_notification() + .with(eq(*NOTIFICATION_ID), eq(Some(ISSUE_NUMBER)), eq(None)) + .times(1) + .returning(|_, _, _| Box::pin(future::ready(Ok(())))); + + let mut gh = MockGH::new(); + gh.expect_create_issue() + .with( + eq("owner"), + eq("repo1"), + eq(ANNUAL_REVIEW_DUE_TITLE), + eq(tmpl::AnnualReviewDue {}.render().unwrap()), + ) + .times(1) + .returning(|_, _, _, _| Box::pin(future::ready(Ok(ISSUE_NUMBER)))); + + run(&cfg, Box::new(db), Box::new(gh)).await.unwrap(); + } + + #[tokio::test] + async fn create_new_issue_because_existing_one_was_closed() { + let cfg = Config::builder().build().unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| { + Box::pin(future::ready(Ok(vec![AnnualReviewNotification { + project_id: *PROJECT_ID, + community_repo_url: REPO1_URL.to_string(), + issue_number: Some(ISSUE_NUMBER), + }]))) + }); + db.expect_pre_register_annual_review_notification() + .with(eq(*PROJECT_ID), eq(REPO1_URL)) + .times(1) + .returning(|_, _| Box::pin(future::ready(Ok(*NOTIFICATION_ID)))); + db.expect_update_annual_review_notification() + .with(eq(*NOTIFICATION_ID), eq(Some(ISSUE_NUMBER)), eq(None)) + .times(1) + .returning(|_, _, _| Box::pin(future::ready(Ok(())))); + + let mut gh = MockGH::new(); + gh.expect_is_issue_closed() + .with(eq("owner"), eq("repo1"), eq(ISSUE_NUMBER)) + .times(1) + .returning(|_, _, _| Box::pin(future::ready(Ok(true)))); + gh.expect_create_issue() + .with( + eq("owner"), + eq("repo1"), + eq(ANNUAL_REVIEW_DUE_TITLE), + eq(tmpl::AnnualReviewDue {}.render().unwrap()), + ) + .times(1) + .returning(|_, _, _, _| Box::pin(future::ready(Ok(ISSUE_NUMBER)))); + + run(&cfg, Box::new(db), Box::new(gh)).await.unwrap(); + } + + #[tokio::test] + async fn create_new_comment_because_existing_one_was_open() { + let cfg = Config::builder().build().unwrap(); + + let mut db = MockDB::new(); + db.expect_get_pending_annual_review_notifications() + .times(1) + .returning(|| { + Box::pin(future::ready(Ok(vec![AnnualReviewNotification { + project_id: *PROJECT_ID, + community_repo_url: REPO1_URL.to_string(), + issue_number: Some(ISSUE_NUMBER), + }]))) + }); + db.expect_pre_register_annual_review_notification() + .with(eq(*PROJECT_ID), eq(REPO1_URL)) + .times(1) + .returning(|_, _| Box::pin(future::ready(Ok(*NOTIFICATION_ID)))); + db.expect_update_annual_review_notification() + .with( + eq(*NOTIFICATION_ID), + eq(Some(ISSUE_NUMBER)), + eq(Some(COMMENT_ID)), + ) + .times(1) + .returning(|_, _, _| Box::pin(future::ready(Ok(())))); + + let mut gh = MockGH::new(); + gh.expect_is_issue_closed() + .with(eq("owner"), eq("repo1"), eq(ISSUE_NUMBER)) + .times(1) + .returning(|_, _, _| Box::pin(future::ready(Ok(false)))); + gh.expect_create_comment() + .with( + eq("owner"), + eq("repo1"), + eq(ISSUE_NUMBER), + eq(tmpl::AnnualReviewDueReminder {}.render().unwrap()), + ) + .times(1) + .returning(|_, _, _, _| Box::pin(future::ready(Ok(COMMENT_ID)))); + + run(&cfg, Box::new(db), Box::new(gh)).await.unwrap(); + } +} diff --git a/clomonitor-notifier/src/tmpl.rs b/clomonitor-notifier/src/tmpl.rs new file mode 100644 index 00000000..5ec79707 --- /dev/null +++ b/clomonitor-notifier/src/tmpl.rs @@ -0,0 +1,11 @@ +use askama::Template; + +/// Template for the annual review due comment. +#[derive(Template)] +#[template(path = "annual-review-due.md")] +pub(crate) struct AnnualReviewDue {} + +/// Template for the annual review due reminder comment. +#[derive(Template)] +#[template(path = "annual-review-due-reminder.md")] +pub(crate) struct AnnualReviewDueReminder {} diff --git a/clomonitor-notifier/templates/annual-review-due-reminder.md b/clomonitor-notifier/templates/annual-review-due-reminder.md new file mode 100644 index 00000000..4c559878 --- /dev/null +++ b/clomonitor-notifier/templates/annual-review-due-reminder.md @@ -0,0 +1,5 @@ +## Annual review due reminder + +This is a friendly reminder to let you know that your annual review is still due. + +For more information about how to file your annual review please see the [Sandbox annual review documentation](https://github.com/cncf/toc/blob/main/process/sandbox-annual-review.md#how-to-file-your-annual-review). diff --git a/clomonitor-notifier/templates/annual-review-due.md b/clomonitor-notifier/templates/annual-review-due.md new file mode 100644 index 00000000..a52a5b0f --- /dev/null +++ b/clomonitor-notifier/templates/annual-review-due.md @@ -0,0 +1,7 @@ +## Annual review due + +**Sandbox** projects are subject to an *annual review* by the **TOC**. This is intended to be a lightweight process to ensure that projects are on track, and getting the support they need. + +[CLOMonitor](https://clomonitor.io) has detected that the annual review for this project has not been filed yet. CLOMonitor relies on the information in the `annual_review_url` and `annual_review_date` fields in the [CNCF Landscape configuration file](https://github.com/cncf/landscape/blob/master/landscape.yml) for this check. If your annual review has already been presented, please make sure this information has been correctly added. + +For more information about how to file your annual review please see the [Sandbox annual review documentation](https://github.com/cncf/toc/blob/main/process/sandbox-annual-review.md#how-to-file-your-annual-review). diff --git a/clomonitor-registrar/Dockerfile b/clomonitor-registrar/Dockerfile index e6fc463b..b02f0173 100644 --- a/clomonitor-registrar/Dockerfile +++ b/clomonitor-registrar/Dockerfile @@ -7,6 +7,7 @@ COPY clomonitor-apiserver/Cargo.* clomonitor-apiserver COPY clomonitor-archiver/Cargo.* clomonitor-archiver COPY clomonitor-core/Cargo.* clomonitor-core COPY clomonitor-linter/Cargo.* clomonitor-linter +COPY clomonitor-notifier/Cargo.* clomonitor-notifier COPY clomonitor-tracker/Cargo.* clomonitor-tracker COPY clomonitor-registrar clomonitor-registrar WORKDIR /clomonitor/clomonitor-registrar diff --git a/clomonitor-tracker/Dockerfile b/clomonitor-tracker/Dockerfile index 61245b72..17093eac 100644 --- a/clomonitor-tracker/Dockerfile +++ b/clomonitor-tracker/Dockerfile @@ -7,6 +7,7 @@ COPY clomonitor-apiserver/Cargo.* clomonitor-apiserver COPY clomonitor-archiver/Cargo.* clomonitor-archiver COPY clomonitor-core clomonitor-core COPY clomonitor-linter clomonitor-linter +COPY clomonitor-notifier/Cargo.* clomonitor-notifier COPY clomonitor-registrar/Cargo.* clomonitor-registrar COPY clomonitor-tracker clomonitor-tracker WORKDIR /clomonitor/clomonitor-tracker diff --git a/database/migrations/functions/001_load_functions.sql b/database/migrations/functions/001_load_functions.sql index 88358158..dcc3b931 100644 --- a/database/migrations/functions/001_load_functions.sql +++ b/database/migrations/functions/001_load_functions.sql @@ -1,3 +1,4 @@ +{{ template "notifications/get_pending_annual_review_notifications.sql" }} {{ template "projects/get_project_by_id.sql" }} {{ template "projects/get_project_by_name.sql" }} {{ template "projects/get_project_checks.sql" }} diff --git a/database/migrations/functions/notifications/get_pending_annual_review_notifications.sql b/database/migrations/functions/notifications/get_pending_annual_review_notifications.sql new file mode 100644 index 00000000..765d7d2f --- /dev/null +++ b/database/migrations/functions/notifications/get_pending_annual_review_notifications.sql @@ -0,0 +1,50 @@ +-- Returns some information about the pending annual review notifications. +create or replace function get_pending_annual_review_notifications() +returns table( + project_id uuid, + community_repo_url text, + issue_number bigint +) as $$ + select distinct on (project_id) + project_id, + r.url as community_repo_url, + last_initial_notification.issue_number + from repository r + join report rp using (repository_id) + join project p using (project_id) + left join ( + select + distinct on (project_id) + project_id, + created_at + from annual_review_notification + order by project_id, created_at desc + ) last_notification using (project_id) + left join ( + select + distinct on (project_id) + project_id, + issue_number + from annual_review_notification + where issue_number is not null + and comment_id is null + and current_timestamp - created_at < '1 year'::interval + order by project_id, created_at desc + ) last_initial_notification using (project_id) + where + 'community' = any(r.check_sets) + and rp.data @> + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }' + and ( + last_notification.created_at is null + or current_timestamp - last_notification.created_at > '1 month'::interval + ); +$$ language sql; diff --git a/database/migrations/schema/009_annual_review_notifications.sql b/database/migrations/schema/009_annual_review_notifications.sql new file mode 100644 index 00000000..0eb1a05c --- /dev/null +++ b/database/migrations/schema/009_annual_review_notifications.sql @@ -0,0 +1,12 @@ +create table if not exists annual_review_notification ( + annual_review_notification_id uuid primary key default gen_random_uuid(), + repository_url text not null, + issue_number bigint, + comment_id bigint, + created_at timestamptz default current_timestamp not null, + project_id uuid references project on delete cascade +); + +---- create above / drop below ---- + +drop table if exists annual_review_notification; diff --git a/database/tests/functions/notifications/get_pending_annual_review_notifications.sql b/database/tests/functions/notifications/get_pending_annual_review_notifications.sql new file mode 100644 index 00000000..e73df6c5 --- /dev/null +++ b/database/tests/functions/notifications/get_pending_annual_review_notifications.sql @@ -0,0 +1,329 @@ +-- Start transaction and plan tests +begin; +select plan(1); + +-- Seed some data +insert into foundation values ('cncf', 'CNCF', 'http://127.0.0.1:8080/cncf.yaml'); + +-- Project 1 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0001-0000-0000-000000000000', + 'project1', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0001-0000-000000000000', + 'repository1', + 'https://repo1.url', + '{code,community}', + '00000000-0001-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0001-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }', + '00000000-0000-0001-0000-000000000000' +); + +-- Project 2 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0002-0000-0000-000000000000', + 'project2', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0002-0000-000000000000', + 'repository2', + 'https://repo2.url', + '{code,community}', + '00000000-0002-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0002-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": true + } + } + }', + '00000000-0000-0002-0000-000000000000' +); + +-- Project 3 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0003-0000-0000-000000000000', + 'project3', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0003-0000-000000000000', + 'repository3', + 'https://repo3.url', + '{code,community}', + '00000000-0003-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0003-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }', + '00000000-0000-0003-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + created_at, + project_id +) values ( + 'https://repo3.url', + current_timestamp - '1 day'::interval, + '00000000-0003-0000-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + created_at, + project_id +) values ( + 'https://repo3.url', + current_timestamp - '7 day'::interval, + '00000000-0003-0000-0000-000000000000' +); + +-- Project 4 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0004-0000-0000-000000000000', + 'project4', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0004-0000-000000000000', + 'repository4', + 'https://repo4.url', + '{code,community}', + '00000000-0004-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0004-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }', + '00000000-0000-0004-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + created_at, + project_id +) values ( + 'https://repo4.url', + current_timestamp - '6 months'::interval, + '00000000-0004-0000-0000-000000000000' +); + +-- Project 5 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0005-0000-0000-000000000000', + 'project5', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0005-0000-000000000000', + 'repository5', + 'https://repo5.url', + '{code,community}', + '00000000-0005-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0005-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }', + '00000000-0000-0005-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + issue_number, + created_at, + project_id +) values ( + 'https://repo5.url', + 1, + current_timestamp - '6 months'::interval, + '00000000-0005-0000-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + issue_number, + created_at, + project_id +) values ( + 'https://repo5.url', + 2, + current_timestamp - '3 months'::interval, + '00000000-0005-0000-0000-000000000000' +); + +-- Project 6 +insert into project ( + project_id, + name, + foundation_id +) values ( + '00000000-0006-0000-0000-000000000000', + 'project6', + 'cncf' +); +insert into repository ( + repository_id, + name, + url, + check_sets, + project_id +) values ( + '00000000-0000-0006-0000-000000000000', + 'repository6', + 'https://repo6.url', + '{code,community}', + '00000000-0006-0000-0000-000000000000' +); +insert into report ( + report_id, + data, + repository_id +) values ( + '00000000-0000-0000-0006-000000000000', + '{ + "documentation": { + "annual_review": { + "passed": false, + "exempt": false, + "failed": false + } + } + }', + '00000000-0000-0006-0000-000000000000' +); +insert into annual_review_notification ( + repository_url, + issue_number, + created_at, + project_id +) values ( + 'https://repo5.url', + 1, + current_timestamp - '2 years'::interval, + '00000000-0006-0000-0000-000000000000' +); + +-- Run some tests +select results_eq( + $$ + select * from get_pending_annual_review_notifications() + $$, + $$ + values + ('00000000-0001-0000-0000-000000000000'::uuid, 'https://repo1.url', null::bigint), + ('00000000-0004-0000-0000-000000000000'::uuid, 'https://repo4.url', null::bigint), + ('00000000-0005-0000-0000-000000000000'::uuid, 'https://repo5.url', 2), + ('00000000-0006-0000-0000-000000000000'::uuid, 'https://repo6.url', null::bigint) + $$, + 'Return pending annual review notifications' +); + +-- Finish tests and rollback transaction +select * from finish(); +rollback; diff --git a/database/tests/schema/schema.sql b/database/tests/schema/schema.sql index fd0d8df9..ad63cf70 100644 --- a/database/tests/schema/schema.sql +++ b/database/tests/schema/schema.sql @@ -1,6 +1,6 @@ -- Start transaction and plan tests begin; -select plan(31); +select plan(32); -- Check expected extension exist select has_extension('pgcrypto'); @@ -98,6 +98,8 @@ select indexes_are('repository', array[ ]); -- Check expected functions exist +-- Notifications +select has_function('get_pending_annual_review_notifications'); -- Projects select has_function('get_project_by_id'); select has_function('get_project_by_name'); diff --git a/web/src/data.tsx b/web/src/data.tsx index 12465d34..e71474ce 100644 --- a/web/src/data.tsx +++ b/web/src/data.tsx @@ -220,7 +220,7 @@ export const REPORT_OPTIONS: ReportOptionInfo = { [ReportOption.AnnualReview]: { icon: , name: 'Annual review', - legend: Sandbox projects are subject to an annual review by the TOC, + legend: CNCF Sandbox projects are subject to an annual review by the TOC, reference: '/docs/topics/checks/#annual-review', }, [ReportOption.ApprovedLicense]: {