diff --git a/.env.example b/.env.example index 8ded1a3..508ffc0 100644 --- a/.env.example +++ b/.env.example @@ -48,6 +48,15 @@ THROTTLE_LIMIT=100 # Event polling interval (ms) EVENT_POLLING_INTERVAL_MS=5000 +# Nightly database backups (02:00 UTC) +BACKUP_ENABLED=false +BACKUP_S3_BUCKET=your-backup-bucket +BACKUP_S3_PREFIX=database-backups +BACKUP_RETENTION_DAYS=30 +AWS_REGION=us-east-1 +# Optional endpoint for S3-compatible offsite storage +# BACKUP_S3_ENDPOINT=https://s3.example.com + # CORS — frontend origin CORS_ORIGIN=http://localhost:3001 diff --git a/package-lock.json b/package-lock.json index 07f61eb..dc0a7b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "@aws-sdk/client-s3": "^3.1098.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.0.0", "@nestjs/core": "^10.0.0", @@ -240,6 +241,314 @@ "tslib": "^2.1.0" } }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.23", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.23.tgz", + "integrity": "sha512-Hv4VX5+IdDYmAbOdiwZguz8fLh3WnE4VtyJwVNEHLulA+OYy7Z5L0ETGUlX2T4g8DLO8XxCV8CyWrtVL4uwqkg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1098.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1098.0.tgz", + "integrity": "sha512-6Z53QG2jdujsCt3eHO2cjCpdfvz2Pgs+hP/NINuGONZUJmdrHLLDvWM9F114Yq0NPIGtk+yA/GCjG3XiUhw+gA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.23", + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/credential-provider-node": "^3.972.75", + "@aws-sdk/middleware-sdk-s3": "^3.972.69", + "@aws-sdk/signature-v4-multi-region": "^3.996.43", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.3.tgz", + "integrity": "sha512-6YZTpF5Zzl0KdSrztM0l6ErKdnXrnnodIDYfayHE9SFfZU8Ubxls7H2XnfWT121jgwiG+WP1z5kyfVDsH5V4qA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.37", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.31.1", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.64", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.64.tgz", + "integrity": "sha512-14kkR5aj1c7D+cYCrEcG2W3xw87wMYAEUeB/tMiQ2j0RVKzzdewd4mJw1+Q0JVLr4IFU1JHGDCyj0WqLrtIixg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.66.tgz", + "integrity": "sha512-BrZxoXScBZ+nTOYy388zHkz1n5cS8C+uGFMozD4w9OKEWMq39Cu/9l/k385Hb54dtv0VIeh12f8h67o3xDNH9g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.9.tgz", + "integrity": "sha512-d/mMvS+sQjSWNA6+JCldK1YB/jQmtoWM8Izj10mrOZdRCeWNVb8IYQuuyHHKKKXcpGo5Pd0LIK6onHbLvlhuVQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/credential-provider-env": "^3.972.64", + "@aws-sdk/credential-provider-http": "^3.972.66", + "@aws-sdk/credential-provider-login": "^3.972.71", + "@aws-sdk/credential-provider-process": "^3.972.64", + "@aws-sdk/credential-provider-sso": "^3.973.8", + "@aws-sdk/credential-provider-web-identity": "^3.972.70", + "@aws-sdk/nested-clients": "^3.997.38", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.71", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.71.tgz", + "integrity": "sha512-l93922lXnTs5RjM3QrMCXSmXiEgLFLaQ0Lco9F3tJ08o0mzGdAj6m2nAXDuTMHoUuL0u7RKmrhm+Z8/7GCilyg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/nested-clients": "^3.997.38", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.75", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.75.tgz", + "integrity": "sha512-sdFhR4E3HWZefGL6OsBhxqXdqtFvxGOh8VLfiu9yttPgVNcpEozEDPmrTYmXA3CIf6npUETVLveQPGQ7GBzVhg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.64", + "@aws-sdk/credential-provider-http": "^3.972.66", + "@aws-sdk/credential-provider-ini": "^3.973.9", + "@aws-sdk/credential-provider-process": "^3.972.64", + "@aws-sdk/credential-provider-sso": "^3.973.8", + "@aws-sdk/credential-provider-web-identity": "^3.972.70", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.64", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.64.tgz", + "integrity": "sha512-Q5noG5vfFo++bUaLGcjj8OHX5tmwa1O/5nDVMeaSSZgjzgvWbW6tgu/gSa+aENwcJXjoXDQh+8TRZR2zHds/aw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.8.tgz", + "integrity": "sha512-37okeZyRdVkGrU6nyKBPGqQvyl/7CEiMpUVOb0+BCNCIR0VAuBxNgjMb0mnsR1EXm8dqQFfQZOJ5YW/0sBfa1w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/nested-clients": "^3.997.38", + "@aws-sdk/token-providers": "3.1098.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.70.tgz", + "integrity": "sha512-Ps/f9USafScb6CSQTRPNMzdKL5x5XRwDIRgFvnuFWfClGZe7/fI7iCFqXdxKjc9LBxP28E7iUgNloGgfgw406w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/nested-clients": "^3.997.38", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.69", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.69.tgz", + "integrity": "sha512-ziSQA8AkB+u4K/5t/SStFbeoiNTG04reUeddCLNCAgmdRJGQ3+MJcpZOXXLdD6uyfCqSK1/R6004gRdXvHbYQA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.43", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.38", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.38.tgz", + "integrity": "sha512-jQDs+nxnmgo4+WPOmY373HIC8jx4KyYYMYDI74FWrzX3Ba9fYsV0dSA+7PNYNkhRbbRCRLre83tQ31F1ACCHkA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.43", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.43", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.43.tgz", + "integrity": "sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1098.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1098.0.tgz", + "integrity": "sha512-6cFAviffeqYGjrXn4FqGbWcTxB3nbpBpQXUm6MvnrWZThJaVBTHHnespHgSUh4yVLfzhipSh9H/XjzkHQd9P4g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.3", + "@aws-sdk/nested-clients": "^3.997.38", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.37.tgz", + "integrity": "sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -2351,6 +2660,87 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@smithy/core": { + "version": "3.31.1", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.31.1.tgz", + "integrity": "sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.16", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.16.tgz", + "integrity": "sha512-QfuLWAkLzptffFW980AFeHZFdqds2B64rpEd3uJ6lgs3xVn9QegGMUgUcj+4d7dRrAsya3r58ZKpku97WcFb4w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.13", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.13.tgz", + "integrity": "sha512-4fW86pEUOMbrD5nkbyl/tTvPHHWJFbuB2odl6ps9lWfHoXf9HWh3Q/Smh59qH1g7+c/BSZghX6bbUk4gsiMs8A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.13", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.13.tgz", + "integrity": "sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.12", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.12.tgz", + "integrity": "sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@stellar/js-xdr": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@stellar/js-xdr/-/js-xdr-3.1.2.tgz", @@ -3855,6 +4245,12 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/boxen": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/boxen/-/boxen-5.1.2.tgz", diff --git a/package.json b/package.json index 4814d03..59ea418 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "format": "prettier --write \"src/**/*.ts\"" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1098.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.0.0", "@nestjs/core": "^10.0.0", diff --git a/src/app.module.ts b/src/app.module.ts index 7cdade5..366015f 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,7 @@ import { ModerationModule } from './modules/moderation/moderation.module'; import { GamificationModule } from './modules/gamification/gamification.module'; import { ReferralsModule } from './modules/referrals/referrals.module'; import { AuditLogModule } from './modules/audit-log/audit-log.module'; +import { BackupsModule } from './modules/backups/backups.module'; @Module({ imports: [ @@ -72,6 +73,7 @@ import { AuditLogModule } from './modules/audit-log/audit-log.module'; GamificationModule, ReferralsModule, AuditLogModule, + BackupsModule, ], }) export class AppModule {} diff --git a/src/modules/backups/backups.constants.ts b/src/modules/backups/backups.constants.ts new file mode 100644 index 0000000..ca888f0 --- /dev/null +++ b/src/modules/backups/backups.constants.ts @@ -0,0 +1 @@ +export const DB_BACKUP_S3_CLIENT = Symbol('DB_BACKUP_S3_CLIENT'); diff --git a/src/modules/backups/backups.module.ts b/src/modules/backups/backups.module.ts new file mode 100644 index 0000000..ccc694d --- /dev/null +++ b/src/modules/backups/backups.module.ts @@ -0,0 +1,31 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { S3Client } from '@aws-sdk/client-s3'; +import { DB_BACKUP_S3_CLIENT } from './backups.constants'; +import { DbBackupService } from './db-backup.service'; + +@Module({ + imports: [ConfigModule], + providers: [ + { + provide: DB_BACKUP_S3_CLIENT, + inject: [ConfigService], + useFactory: (config: ConfigService) => { + const endpoint = config.get('BACKUP_S3_ENDPOINT'); + + return new S3Client({ + region: config.get('AWS_REGION', 'us-east-1'), + ...(endpoint + ? { + endpoint, + forcePathStyle: true, + } + : {}), + }); + }, + }, + DbBackupService, + ], + exports: [DbBackupService], +}) +export class BackupsModule {} diff --git a/src/modules/backups/db-backup.service.spec.ts b/src/modules/backups/db-backup.service.spec.ts new file mode 100644 index 0000000..59ca7a8 --- /dev/null +++ b/src/modules/backups/db-backup.service.spec.ts @@ -0,0 +1,208 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { writeFileSync } from 'node:fs'; +import { ConfigService } from '@nestjs/config'; +import { + DeleteObjectsCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { DbBackupService } from './db-backup.service'; + +jest.mock('node:child_process', () => ({ + execFile: jest.fn(), +})); + +describe('DbBackupService', () => { + let service: DbBackupService; + let configValues: Record; + let send: jest.Mock; + const mockedExecFile = execFile as unknown as jest.Mock; + + beforeEach(() => { + configValues = { + BACKUP_ENABLED: true, + BACKUP_S3_BUCKET: 'backup-bucket', + BACKUP_S3_PREFIX: '/nightly/', + BACKUP_RETENTION_DAYS: 7, + DATABASE_URL: + 'postgresql://backup-user:secret@database.internal:5433/hamplard?sslmode=require', + }; + const config = { + get: jest.fn((key: string, fallback?: unknown) => + key in configValues ? configValues[key] : fallback, + ), + } as unknown as ConfigService; + send = jest.fn(); + service = new DbBackupService(config, { send } as unknown as S3Client); + mockedExecFile.mockReset(); + }); + + it('creates, validates, uploads, and rotates a backup with status metrics', async () => { + mockedExecFile.mockImplementation( + ( + command: string, + args: string[], + _options: unknown, + callback: (error: Error | null, stdout: string, stderr: string) => void, + ) => { + if (command === 'pg_dump') { + const dumpPath = args[args.indexOf('--file') + 1]; + writeFileSync(dumpPath, 'valid-dump'); + } + callback(null, command === 'pg_restore' ? 'archive contents' : '', ''); + }, + ); + + let listPage = 0; + send.mockImplementation(async (command: unknown) => { + if (command instanceof PutObjectCommand) { + for await (const _chunk of command.input.Body as Readable) { + // Consume the stream before the temporary file is removed. + } + return {}; + } + + if (command instanceof ListObjectsV2Command) { + listPage += 1; + return listPage === 1 + ? { + Contents: [ + { + Key: 'nightly/expired.dump', + LastModified: new Date('2000-01-01T00:00:00.000Z'), + }, + ], + IsTruncated: true, + NextContinuationToken: 'next-page', + } + : { + Contents: [ + { + Key: 'nightly/current.dump', + LastModified: new Date(Date.now() + 60_000), + }, + ], + IsTruncated: false, + }; + } + + return {}; + }); + + const result = await service.runBackup(); + + expect(result).toEqual( + expect.objectContaining({ + state: 'SUCCESS', + dumpDurationMs: expect.any(Number), + durationMs: expect.any(Number), + sizeBytes: 10, + objectKey: expect.stringMatching(/^nightly\/hamplard-.*\.dump$/), + }), + ); + expect(service.getLastRunStatus()).toEqual(result); + + const dumpCall = mockedExecFile.mock.calls[0]; + expect(dumpCall[0]).toBe('pg_dump'); + expect(dumpCall[1]).toEqual([ + '--format=custom', + '--no-password', + '--file', + expect.any(String), + ]); + expect(dumpCall[1].join(' ')).not.toContain('secret'); + expect(dumpCall[2].env).toEqual( + expect.objectContaining({ + PGHOST: 'database.internal', + PGPORT: '5433', + PGUSER: 'backup-user', + PGPASSWORD: 'secret', + PGDATABASE: 'hamplard', + PGSSLMODE: 'require', + }), + ); + expect(mockedExecFile.mock.calls[1][0]).toBe('pg_restore'); + expect(mockedExecFile.mock.calls[1][1][0]).toBe('--list'); + + const putCommand = send.mock.calls + .map(([command]) => command) + .find((command) => command instanceof PutObjectCommand); + expect(putCommand.input).toEqual( + expect.objectContaining({ + Bucket: 'backup-bucket', + ContentLength: 10, + ChecksumSHA256: createHash('sha256') + .update('valid-dump') + .digest('base64'), + }), + ); + + const listCommands = send.mock.calls + .map(([command]) => command) + .filter((command) => command instanceof ListObjectsV2Command); + expect(listCommands).toHaveLength(2); + expect(listCommands[1].input.ContinuationToken).toBe('next-page'); + + const deleteCommand = send.mock.calls + .map(([command]) => command) + .find((command) => command instanceof DeleteObjectsCommand); + expect(deleteCommand.input.Delete.Objects).toEqual([ + { Key: 'nightly/expired.dump' }, + ]); + }); + + it('does not upload a dump that fails integrity validation', async () => { + mockedExecFile.mockImplementation( + ( + command: string, + args: string[], + _options: unknown, + callback: (error: Error | null, stdout: string, stderr: string) => void, + ) => { + if (command === 'pg_dump') { + writeFileSync(args[args.indexOf('--file') + 1], 'invalid-dump'); + callback(null, '', ''); + return; + } + callback(new Error('invalid archive'), '', 'invalid archive'); + }, + ); + + const result = await service.runBackup(); + + expect(result).toEqual( + expect.objectContaining({ + state: 'FAILED', + message: 'pg_restore failed: invalid archive', + }), + ); + expect(send).not.toHaveBeenCalled(); + }); + + it('skips execution when backups are disabled', async () => { + configValues.BACKUP_ENABLED = false; + + const result = await service.runBackup(); + + expect(result.state).toBe('SKIPPED'); + expect(result.message).toBe('Database backups are disabled'); + expect(mockedExecFile).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('delegates scheduled runs to the backup runner', async () => { + const expected = { + state: 'SKIPPED' as const, + startedAt: '2026-01-01T00:00:00.000Z', + completedAt: '2026-01-01T00:00:00.000Z', + durationMs: 0, + }; + jest.spyOn(service, 'runBackup').mockResolvedValue(expected); + + await expect(service.handleScheduledBackup()).resolves.toEqual(expected); + expect(service.runBackup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/modules/backups/db-backup.service.ts b/src/modules/backups/db-backup.service.ts new file mode 100644 index 0000000..b2949e1 --- /dev/null +++ b/src/modules/backups/db-backup.service.ts @@ -0,0 +1,372 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cron } from '@nestjs/schedule'; +import { + DeleteObjectsCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { DB_BACKUP_S3_CLIENT } from './backups.constants'; + +export type BackupRunState = 'SUCCESS' | 'FAILED' | 'SKIPPED'; + +export interface BackupRunStatus { + state: BackupRunState; + startedAt: string; + completedAt: string; + durationMs: number; + dumpDurationMs?: number; + sizeBytes?: number; + objectKey?: string; + message?: string; +} + +interface DatabaseConnection { + hostname: string; + port: string; + username: string; + password: string; + database: string; + sslMode?: string; +} + +@Injectable() +export class DbBackupService { + private readonly logger = new Logger(DbBackupService.name); + private backupInProgress = false; + private lastRunStatus: BackupRunStatus | null = null; + + constructor( + private readonly config: ConfigService, + @Inject(DB_BACKUP_S3_CLIENT) private readonly s3Client: S3Client, + ) {} + + @Cron('0 0 2 * * *', { + name: 'nightly-database-backup', + timeZone: 'UTC', + }) + async handleScheduledBackup(): Promise { + return this.runBackup(); + } + + async runBackup(): Promise { + const startedAt = new Date(); + + if (!this.isBackupEnabled()) { + return this.recordSkippedRun(startedAt, 'Database backups are disabled'); + } + + if (this.backupInProgress) { + return this.recordSkippedRun( + startedAt, + 'A database backup is already in progress', + ); + } + + this.backupInProgress = true; + let temporaryDirectory: string | undefined; + let dumpDurationMs: number | undefined; + let sizeBytes: number | undefined; + let objectKey: string | undefined; + + this.logger.log(`Database backup started at ${startedAt.toISOString()}`); + + try { + const databaseUrl = this.getRequiredConfig('DATABASE_URL'); + const bucket = this.getRequiredConfig('BACKUP_S3_BUCKET'); + const prefix = this.getObjectPrefix(); + const connection = this.parseDatabaseUrl(databaseUrl); + + temporaryDirectory = await mkdtemp(join(tmpdir(), 'hamplard-db-backup-')); + const fileName = `hamplard-${this.toFileTimestamp(startedAt)}.dump`; + const dumpPath = join(temporaryDirectory, fileName); + objectKey = `${prefix}/${fileName}`; + + const dumpStartedAt = Date.now(); + await this.createDump(connection, dumpPath); + dumpDurationMs = Date.now() - dumpStartedAt; + + const dumpStats = await stat(dumpPath); + if (!dumpStats.isFile() || dumpStats.size === 0) { + throw new Error('pg_dump created an empty backup archive'); + } + sizeBytes = dumpStats.size; + + await this.validateDump(dumpPath); + const checksum = await this.calculateChecksum(dumpPath); + + await this.s3Client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: objectKey, + Body: createReadStream(dumpPath), + ContentLength: sizeBytes, + ContentType: 'application/octet-stream', + ChecksumSHA256: checksum, + Metadata: { + 'dump-duration-ms': String(dumpDurationMs), + }, + }), + ); + + await this.rotateExpiredBackups(bucket, prefix); + + const status = this.createStatus(startedAt, 'SUCCESS', { + dumpDurationMs, + sizeBytes, + objectKey, + }); + this.lastRunStatus = status; + this.logger.log(`Database backup succeeded ${JSON.stringify(status)}`); + return status; + } catch (error) { + const message = this.getErrorMessage(error); + const status = this.createStatus(startedAt, 'FAILED', { + dumpDurationMs, + sizeBytes, + objectKey, + message, + }); + this.lastRunStatus = status; + this.logger.error(`Database backup failed ${JSON.stringify(status)}`); + return status; + } finally { + if (temporaryDirectory) { + await rm(temporaryDirectory, { recursive: true, force: true }).catch( + (error) => + this.logger.warn( + `Failed to remove temporary backup files: ${this.getErrorMessage(error)}`, + ), + ); + } + this.backupInProgress = false; + } + } + + getLastRunStatus(): BackupRunStatus | null { + return this.lastRunStatus ? { ...this.lastRunStatus } : null; + } + + private async createDump( + connection: DatabaseConnection, + dumpPath: string, + ): Promise { + await this.runDatabaseCommand( + 'pg_dump', + ['--format=custom', '--no-password', '--file', dumpPath], + connection, + ); + } + + private async validateDump(dumpPath: string): Promise { + await this.runDatabaseCommand('pg_restore', ['--list', dumpPath]); + } + + private runDatabaseCommand( + command: string, + args: string[], + connection?: DatabaseConnection, + ): Promise { + const commandEnvironment = connection + ? { + ...process.env, + PGHOST: connection.hostname, + PGPORT: connection.port, + PGUSER: connection.username, + PGPASSWORD: connection.password, + PGDATABASE: connection.database, + ...(connection.sslMode ? { PGSSLMODE: connection.sslMode } : {}), + } + : process.env; + + return new Promise((resolve, reject) => { + execFile( + command, + args, + { + env: commandEnvironment, + maxBuffer: 10 * 1024 * 1024, + }, + (error, _stdout, stderr) => { + if (!error) { + resolve(); + return; + } + + const detail = stderr.trim() || error.message; + reject(new Error(`${command} failed: ${detail}`)); + }, + ); + }); + } + + private async calculateChecksum(dumpPath: string): Promise { + const hash = createHash('sha256'); + + for await (const chunk of createReadStream(dumpPath)) { + hash.update(chunk); + } + + return hash.digest('base64'); + } + + private async rotateExpiredBackups( + bucket: string, + prefix: string, + ): Promise { + const retentionDays = this.getPositiveInteger('BACKUP_RETENTION_DAYS', 30); + const expiresBefore = new Date( + Date.now() - retentionDays * 24 * 60 * 60 * 1000, + ); + let continuationToken: string | undefined; + let deletedCount = 0; + + do { + const response = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `${prefix}/`, + ContinuationToken: continuationToken, + }), + ); + const expiredObjects = (response.Contents ?? []) + .filter( + (object) => + object.Key && + object.LastModified && + object.LastModified < expiresBefore, + ) + .map((object) => ({ Key: object.Key as string })); + + if (expiredObjects.length > 0) { + const deleteResponse = await this.s3Client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { + Objects: expiredObjects, + Quiet: true, + }, + }), + ); + if (deleteResponse.Errors?.length) { + throw new Error( + `Failed to delete ${deleteResponse.Errors.length} expired backup object(s)`, + ); + } + deletedCount += expiredObjects.length; + } + + continuationToken = response.IsTruncated + ? response.NextContinuationToken + : undefined; + } while (continuationToken); + + this.logger.log( + `Database backup retention completed: deleted ${deletedCount} object(s) older than ${retentionDays} day(s)`, + ); + } + + private parseDatabaseUrl(databaseUrl: string): DatabaseConnection { + let parsedUrl: URL; + + try { + parsedUrl = new URL(databaseUrl); + } catch { + throw new Error('DATABASE_URL must be a valid PostgreSQL URL'); + } + + if ( + parsedUrl.protocol !== 'postgresql:' && + parsedUrl.protocol !== 'postgres:' + ) { + throw new Error('DATABASE_URL must use the PostgreSQL protocol'); + } + + const database = decodeURIComponent(parsedUrl.pathname.replace(/^\//, '')); + if (!parsedUrl.hostname || !database) { + throw new Error('DATABASE_URL must include a host and database name'); + } + + return { + hostname: parsedUrl.hostname, + port: parsedUrl.port || '5432', + username: decodeURIComponent(parsedUrl.username), + password: decodeURIComponent(parsedUrl.password), + database, + sslMode: parsedUrl.searchParams.get('sslmode') ?? undefined, + }; + } + + private getRequiredConfig(key: string): string { + const value = this.config.get(key)?.trim(); + if (!value) { + throw new Error(`${key} is required when database backups are enabled`); + } + return value; + } + + private getObjectPrefix(): string { + const configuredPrefix = this.config.get( + 'BACKUP_S3_PREFIX', + 'database-backups', + ); + const prefix = configuredPrefix.replace(/^\/+|\/+$/g, ''); + return prefix || 'database-backups'; + } + + private getPositiveInteger(key: string, fallback: number): number { + const configuredValue = this.config.get(key, fallback); + const parsedValue = Number(configuredValue); + + if (!Number.isInteger(parsedValue) || parsedValue <= 0) { + throw new Error(`${key} must be a positive integer`); + } + + return parsedValue; + } + + private isBackupEnabled(): boolean { + return ( + String(this.config.get('BACKUP_ENABLED', false)) + .trim() + .toLowerCase() === 'true' + ); + } + + private recordSkippedRun(startedAt: Date, message: string): BackupRunStatus { + const status = this.createStatus(startedAt, 'SKIPPED', { message }); + this.lastRunStatus = status; + this.logger.warn(`Database backup skipped ${JSON.stringify(status)}`); + return status; + } + + private createStatus( + startedAt: Date, + state: BackupRunState, + details: Partial, + ): BackupRunStatus { + const completedAt = new Date(); + return { + state, + startedAt: startedAt.toISOString(), + completedAt: completedAt.toISOString(), + durationMs: completedAt.getTime() - startedAt.getTime(), + ...details, + }; + } + + private toFileTimestamp(date: Date): string { + return date.toISOString().replace(/[:.]/g, '-'); + } + + private getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : 'Unknown error'; + } +}