From 4881bc44dec3dbc9cc48032fb6b3b2f5f3d955ba Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 7 Apr 2025 13:56:04 -0700 Subject: [PATCH 01/98] [TM-1861] WIP status state machine support. --- .../src/lib/sequelize-config.service.ts | 20 ++- .../lib/util/model-column-state-machine.ts | 132 ++++++++++++++++++ 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 libs/database/src/lib/util/model-column-state-machine.ts diff --git a/libs/database/src/lib/sequelize-config.service.ts b/libs/database/src/lib/sequelize-config.service.ts index 543e1004..2d773ca5 100644 --- a/libs/database/src/lib/sequelize-config.service.ts +++ b/libs/database/src/lib/sequelize-config.service.ts @@ -2,6 +2,8 @@ import { Injectable, Logger } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import * as Entities from "./entities"; import { SequelizeModuleOptions, SequelizeOptionsFactory } from "@nestjs/sequelize"; +import { Model } from "sequelize-typescript"; +import { StateMachineModel } from "./util/model-column-state-machine"; @Injectable() export class SequelizeConfigService implements SequelizeOptionsFactory { @@ -18,7 +20,23 @@ export class SequelizeConfigService implements SequelizeOptionsFactory { database: this.configService.get("DB_DATABASE"), synchronize: false, models: Object.values(Entities), - logging: sql => logger.log(sql) + logging: sql => logger.log(sql), + hooks: { + afterSave: function (model: Model) { + // After any model saves, check if we have a state machine defined on one or more of its + // columns, and if so, call afterSave on the state machine instance for the possible + // processing of afterTransitionHooks. See model-column-state-machine.ts + const stateMachineMetadataKeys = Reflect.getMetadataKeys(model).filter(key => + key.startsWith("model-column-state-machine:") + ); + for (const key of stateMachineMetadataKeys) { + const propertyName = key.split(":").pop(); + if (!Reflect.getMetadata(key, model)) continue; + + (model as StateMachineModel)._stateMachines?.[propertyName]?.afterSave(); + } + } + } }; } } diff --git a/libs/database/src/lib/util/model-column-state-machine.ts b/libs/database/src/lib/util/model-column-state-machine.ts new file mode 100644 index 00000000..4c910da2 --- /dev/null +++ b/libs/database/src/lib/util/model-column-state-machine.ts @@ -0,0 +1,132 @@ +import { Column, Model } from "sequelize-typescript"; +import { Attributes, STRING } from "sequelize"; +import { applyDecorators, CustomDecorator, SetMetadata } from "@nestjs/common"; + +export type States, S extends string> = { + default: S; + transitions: Partial>; + + /** + * If specified, this function can be used to perform extra validations for a transition within + * the context of the model it's attached to. If the function is defined and returns false, the + * transition will be rejected with a StateMachineError. + * + * @param from The "from" state for the transition + * @param to The "to" state for the transition + * @param model The model the transition will be applied to. + */ + transitionValidForModel?: (from: S, to: S, model: M) => boolean; + + /** + * Specify hooks that should be fired when a given state is transitioned to. The callback + * will be executed after the model has been saved. + * + * Note: This means that if multiple transitions are executed on a given model column without + * a save to the database in between, this hook will only fire after the _last_ transition. To + * ensure the hook fires for every transition, make sure to save the model state between + * successive transitions. + */ + afterTransitionHooks?: Partial void>>; +}; + +export class StateMachineError extends Error {} + +export type StateMachineModel, S extends string> = M & { + _stateMachines?: Record>; +}; + +function ensureStateMachine, S extends string>( + model: StateMachineModel, + propertyName: keyof Attributes, + states: States +) { + if (model._stateMachines == null) model._stateMachines = {}; + if (model._stateMachines[propertyName as string] == null) { + model._stateMachines[propertyName as string] = new ModelColumnStateMachine(model, propertyName, states); + } + return model._stateMachines[propertyName as string]; +} + +export const StateMachineColumn = + , S extends string>(states: States) => + (target: object, propertyName: string, propertyDescriptor?: PropertyDescriptor) => { + applyDecorators( + // Will cause the `afterSave` method of the state machine to be called when the model is + // saved. See sequelize-config.service.ts + SetMetadata(`model-column-state-machine:${propertyName}`, true), + Column({ + type: STRING, + defaultValue: states.default, + + get(this: StateMachineModel) { + return ensureStateMachine(this, propertyName as keyof Attributes, states); + }, + + set(this: StateMachineModel, value: S) { + ensureStateMachine(this, propertyName as keyof Attributes, states).transitionTo(value); + } + }) as CustomDecorator + )(target, propertyName, propertyDescriptor); + }; + +export class ModelColumnStateMachine, S extends string> { + constructor( + protected readonly model: M, + protected readonly column: keyof Attributes, + protected readonly states: States + ) {} + + protected fromState?: S; + + get current(): S { + return this.model.getDataValue(this.column) as S; + } + + /** + * Allows this object to serialize to the string value of its DB column when included in an API DTO. + */ + toJSON() { + return this.current; + } + + canBe(from: S, to: S) { + if (!Object.keys(this.states.transitions).includes(from)) { + throw new StateMachineError(`Current state is not defined [${from}]`); + } + + return from === to || this.states.transitions[from]?.includes(to) === true; + } + + transitionTo(to: S) { + this.validateTransition(to); + + this.fromState = this.current; + // @ts-expect-error force a string into the column this is applied to. + this.model.setDataValue(this.column, to); + } + + validateTransition(to: S) { + if (this.model.isNewRecord) return; + + if (!this.canBe(this.current, to)) { + throw new StateMachineError(`Transition not valid [from=${this.current}, to=${to}]`); + } + + if ( + this.states.transitionValidForModel != null && + !this.states.transitionValidForModel(this.current, to, this.model) + ) { + throw new StateMachineError( + `Transition not valid for model [from=${this.current}, to=${to}, id=${this.model.id}]` + ); + } + } + + afterSave() { + if (this.fromState == null) return; + + const fromState = this.fromState; + this.fromState = undefined; + this.states.afterTransitionHooks?.[this.current]?.(fromState, this.model); + } +} From d7df6cdccc150ed710290f3345f4033a6ff87165 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 7 Apr 2025 14:52:12 -0700 Subject: [PATCH 02/98] [TM-1861] Implement entity status state machine for project. --- .../src/entities/dto/project.dto.ts | 3 +- jest/setup-jest.ts | 4 +- libs/database/src/lib/constants/status.ts | 26 +++++++- .../src/lib/entities/project.entity.ts | 8 +-- .../src/lib/sequelize-config.service.ts | 33 +++++----- .../lib/util/model-column-state-machine.ts | 61 +++++++++++-------- 6 files changed, 84 insertions(+), 51 deletions(-) diff --git a/apps/entity-service/src/entities/dto/project.dto.ts b/apps/entity-service/src/entities/dto/project.dto.ts index ec5e02a5..42e4d5c3 100644 --- a/apps/entity-service/src/entities/dto/project.dto.ts +++ b/apps/entity-service/src/entities/dto/project.dto.ts @@ -44,11 +44,10 @@ export class ProjectLightDto extends EntityDto { organisationName: string | null; @ApiProperty({ - nullable: true, description: "Entity status for this project", enum: ENTITY_STATUSES }) - status: EntityStatus | null; + status: EntityStatus; @ApiProperty({ nullable: true, diff --git a/jest/setup-jest.ts b/jest/setup-jest.ts index 143d620a..7dc1362c 100644 --- a/jest/setup-jest.ts +++ b/jest/setup-jest.ts @@ -1,6 +1,7 @@ import { Sequelize } from "sequelize-typescript"; import { FactoryGirl, SequelizeAdapter } from "factory-girl-ts"; import * as Entities from "@terramatch-microservices/database/entities"; +import { SEQUELIZE_GLOBAL_HOOKS } from "@terramatch-microservices/database/sequelize-config.service"; let sequelize: Sequelize; @@ -14,7 +15,8 @@ beforeAll(async () => { password: "wri", database: "terramatch_microservices_test", models: Object.values(Entities), - // Switch to true locally to debug SQL statements in unit tests, especially table/index creation problems. + hooks: SEQUELIZE_GLOBAL_HOOKS, + // Switch to console.log locally to debug SQL statements in unit tests, especially table/index creation problems. logging: false }); diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 45bff790..578499ea 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -1,11 +1,35 @@ +import { States } from "../util/model-column-state-machine"; +import { Nursery, Project, Site } from "../entities"; + export const STARTED = "started"; export const AWAITING_APPROVAL = "awaiting-approval"; export const APPROVED = "approved"; export const RESTORATION_IN_PROGRESS = "restoration-in-progress"; export const NEEDS_MORE_INFORMATION = "needs-more-information"; export const ENTITY_STATUSES = [STARTED, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; -export const SITE_STATUSES = [...ENTITY_STATUSES, RESTORATION_IN_PROGRESS] as const; export type EntityStatus = (typeof ENTITY_STATUSES)[number]; + +export const EntityStatusStates: States = { + default: STARTED, + transitions: { + [STARTED]: [AWAITING_APPROVAL], + [AWAITING_APPROVAL]: [APPROVED, NEEDS_MORE_INFORMATION], + [NEEDS_MORE_INFORMATION]: [APPROVED, AWAITING_APPROVAL], + [APPROVED]: [NEEDS_MORE_INFORMATION] + }, + afterTransitionHooks: { + [APPROVED]: (from, model) => { + console.log("After approved", { from, uuid: model.uuid }); + // TODO Send status change email + }, + [NEEDS_MORE_INFORMATION]: (from, model) => { + console.log("After needs more info", { from, uuid: model.uuid }); + // TODO Send status change email + } + } +}; + +export const SITE_STATUSES = [...ENTITY_STATUSES, RESTORATION_IN_PROGRESS] as const; export type SiteStatus = (typeof SITE_STATUSES)[number]; export const DUE = "due"; diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts index 3f15eea2..aa4117e7 100644 --- a/libs/database/src/lib/entities/project.entity.ts +++ b/libs/database/src/lib/entities/project.entity.ts @@ -21,8 +21,9 @@ import { Nursery } from "./nursery.entity"; import { JsonColumn } from "../decorators/json-column.decorator"; import { FrameworkKey } from "../constants/framework"; import { Framework } from "./framework.entity"; -import { EntityStatus, UpdateRequestStatus } from "../constants/status"; +import { EntityStatus, EntityStatusStates, UpdateRequestStatus } from "../constants/status"; import { Subquery } from "../util/subquery.builder"; +import { StateMachineColumn } from "../util/model-column-state-machine"; @Table({ tableName: "v2_projects", underscored: true, paranoid: true }) export class Project extends Model { @@ -87,9 +88,8 @@ export class Project extends Model { @Column(BIGINT.UNSIGNED) applicationId: number | null; - @AllowNull - @Column(STRING) - status: EntityStatus | null; + @StateMachineColumn(EntityStatusStates) + status: EntityStatus; @AllowNull @Default("no-update") diff --git a/libs/database/src/lib/sequelize-config.service.ts b/libs/database/src/lib/sequelize-config.service.ts index 2d773ca5..87f3d9d7 100644 --- a/libs/database/src/lib/sequelize-config.service.ts +++ b/libs/database/src/lib/sequelize-config.service.ts @@ -3,7 +3,19 @@ import { ConfigService } from "@nestjs/config"; import * as Entities from "./entities"; import { SequelizeModuleOptions, SequelizeOptionsFactory } from "@nestjs/sequelize"; import { Model } from "sequelize-typescript"; -import { StateMachineModel } from "./util/model-column-state-machine"; +import { getStateMachine, getStateMachineProperties } from "./util/model-column-state-machine"; + +export const SEQUELIZE_GLOBAL_HOOKS = { + afterSave: function (model: Model) { + // After any model saves, check if we have a state machine defined on one or more of its + // columns, and if so, call afterSave on the state machine instance for the possible + // processing of afterTransitionHooks. See StateMachineColumn decorator in + // model-column-state-machine.ts + for (const key of getStateMachineProperties(model)) { + getStateMachine(model, key)?.afterSave(); + } + } +}; @Injectable() export class SequelizeConfigService implements SequelizeOptionsFactory { @@ -20,23 +32,8 @@ export class SequelizeConfigService implements SequelizeOptionsFactory { database: this.configService.get("DB_DATABASE"), synchronize: false, models: Object.values(Entities), - logging: sql => logger.log(sql), - hooks: { - afterSave: function (model: Model) { - // After any model saves, check if we have a state machine defined on one or more of its - // columns, and if so, call afterSave on the state machine instance for the possible - // processing of afterTransitionHooks. See model-column-state-machine.ts - const stateMachineMetadataKeys = Reflect.getMetadataKeys(model).filter(key => - key.startsWith("model-column-state-machine:") - ); - for (const key of stateMachineMetadataKeys) { - const propertyName = key.split(":").pop(); - if (!Reflect.getMetadata(key, model)) continue; - - (model as StateMachineModel)._stateMachines?.[propertyName]?.afterSave(); - } - } - } + hooks: SEQUELIZE_GLOBAL_HOOKS, + logging: sql => logger.log(sql) }; } } diff --git a/libs/database/src/lib/util/model-column-state-machine.ts b/libs/database/src/lib/util/model-column-state-machine.ts index 4c910da2..62ea2f6f 100644 --- a/libs/database/src/lib/util/model-column-state-machine.ts +++ b/libs/database/src/lib/util/model-column-state-machine.ts @@ -1,8 +1,7 @@ import { Column, Model } from "sequelize-typescript"; import { Attributes, STRING } from "sequelize"; -import { applyDecorators, CustomDecorator, SetMetadata } from "@nestjs/common"; -export type States, S extends string> = { +export type States = { default: S; transitions: Partial>; @@ -31,11 +30,11 @@ export type States, S extends string> = { export class StateMachineError extends Error {} -export type StateMachineModel, S extends string> = M & { +export type StateMachineModel = M & { _stateMachines?: Record>; }; -function ensureStateMachine, S extends string>( +function ensureStateMachine( model: StateMachineModel, propertyName: keyof Attributes, states: States @@ -47,29 +46,42 @@ function ensureStateMachine, S extends string>( return model._stateMachines[propertyName as string]; } +export function getStateMachine(model: M, propertyName: keyof Attributes) { + return (model as StateMachineModel)._stateMachines?.[propertyName as string]; +} + +const METADATA_KEY = Symbol("model-column-state-machine"); + +export const getStateMachineProperties = (model: M): (keyof Attributes)[] => + Reflect.getMetadata(METADATA_KEY, model) ?? []; + +/** + * Apply to any string column in a database model class to enforce state machine mechanics. + */ export const StateMachineColumn = - , S extends string>(states: States) => - (target: object, propertyName: string, propertyDescriptor?: PropertyDescriptor) => { - applyDecorators( - // Will cause the `afterSave` method of the state machine to be called when the model is - // saved. See sequelize-config.service.ts - SetMetadata(`model-column-state-machine:${propertyName}`, true), - Column({ - type: STRING, - defaultValue: states.default, - - get(this: StateMachineModel) { - return ensureStateMachine(this, propertyName as keyof Attributes, states); - }, - - set(this: StateMachineModel, value: S) { - ensureStateMachine(this, propertyName as keyof Attributes, states).transitionTo(value); - } - }) as CustomDecorator - )(target, propertyName, propertyDescriptor); + (states: States) => + (target: M, propertyName: string, propertyDescriptor?: PropertyDescriptor) => { + // Will cause the `afterSave` method of the state machine to be called when the model is + // saved. See sequelize-config.service.ts + Reflect.defineMetadata(METADATA_KEY, [...getStateMachineProperties(target), propertyName], target); + + // Define the sequelize column as we need it. To consumers of the model, it will appear to be a + // column of type S. If access to the underlying state machine is needed, use getStateMachine() + Column({ + type: STRING, + defaultValue: states.default, + + get(this: StateMachineModel) { + return ensureStateMachine(this, propertyName as keyof Attributes, states).current; + }, + + set(this: StateMachineModel, value: S) { + ensureStateMachine(this, propertyName as keyof Attributes, states).transitionTo(value); + } + })(target, propertyName, propertyDescriptor); }; -export class ModelColumnStateMachine, S extends string> { +export class ModelColumnStateMachine { constructor( protected readonly model: M, protected readonly column: keyof Attributes, @@ -101,7 +113,6 @@ export class ModelColumnStateMachine, S extends string> { this.validateTransition(to); this.fromState = this.current; - // @ts-expect-error force a string into the column this is applied to. this.model.setDataValue(this.column, to); } From baab788872c353b5bf50612817ef35b57cace9e4 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 7 Apr 2025 15:54:53 -0700 Subject: [PATCH 03/98] [TM-1861] Common module depends on database module; move the dependency there. --- apps/entity-service/src/app.module.ts | 3 +-- apps/job-service/src/app.module.ts | 3 +-- apps/research-service/src/app.module.ts | 3 +-- apps/unified-database-service/src/airtable/airtable.module.ts | 2 -- apps/unified-database-service/src/app.module.ts | 3 +-- apps/user-service/src/app.module.ts | 3 +-- libs/common/src/lib/common.module.ts | 4 +++- 7 files changed, 8 insertions(+), 13 deletions(-) diff --git a/apps/entity-service/src/app.module.ts b/apps/entity-service/src/app.module.ts index 748bbb5e..cbdd5e8a 100644 --- a/apps/entity-service/src/app.module.ts +++ b/apps/entity-service/src/app.module.ts @@ -1,5 +1,4 @@ import { Module } from "@nestjs/common"; -import { DatabaseModule } from "@terramatch-microservices/database"; import { CommonModule } from "@terramatch-microservices/common"; import { HealthModule } from "./health/health.module"; import { TreesController } from "./trees/trees.controller"; @@ -11,7 +10,7 @@ import { EntitiesController } from "./entities/entities.controller"; import { EntityAssociationsController } from "./entities/entity-associations.controller"; @Module({ - imports: [SentryModule.forRoot(), DatabaseModule, CommonModule, HealthModule], + imports: [SentryModule.forRoot(), CommonModule, HealthModule], controllers: [EntitiesController, EntityAssociationsController, TreesController], providers: [ { diff --git a/apps/job-service/src/app.module.ts b/apps/job-service/src/app.module.ts index 345b2516..eb0a2005 100644 --- a/apps/job-service/src/app.module.ts +++ b/apps/job-service/src/app.module.ts @@ -1,5 +1,4 @@ import { Module } from "@nestjs/common"; -import { DatabaseModule } from "@terramatch-microservices/database"; import { CommonModule } from "@terramatch-microservices/common"; import { DelayedJobsController } from "./jobs/delayed-jobs.controller"; import { HealthModule } from "./health/health.module"; @@ -7,7 +6,7 @@ import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; import { APP_FILTER } from "@nestjs/core"; @Module({ - imports: [SentryModule.forRoot(), DatabaseModule, CommonModule, HealthModule], + imports: [SentryModule.forRoot(), CommonModule, HealthModule], controllers: [DelayedJobsController], providers: [ { diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts index feb5d57c..bb7908b9 100644 --- a/apps/research-service/src/app.module.ts +++ b/apps/research-service/src/app.module.ts @@ -1,5 +1,4 @@ import { Module } from "@nestjs/common"; -import { DatabaseModule } from "@terramatch-microservices/database"; import { CommonModule } from "@terramatch-microservices/common"; import { HealthModule } from "./health/health.module"; import { SitePolygonsController } from "./site-polygons/site-polygons.controller"; @@ -8,7 +7,7 @@ import { APP_FILTER } from "@nestjs/core"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; @Module({ - imports: [SentryModule.forRoot(), DatabaseModule, CommonModule, HealthModule], + imports: [SentryModule.forRoot(), CommonModule, HealthModule], controllers: [SitePolygonsController], providers: [ { diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts index 2bc38cbe..e93a86c0 100644 --- a/apps/unified-database-service/src/airtable/airtable.module.ts +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -1,4 +1,3 @@ -import { DatabaseModule } from "@terramatch-microservices/database"; import { CommonModule } from "@terramatch-microservices/common"; import { ConfigModule, ConfigService } from "@nestjs/config"; import { BullModule } from "@nestjs/bullmq"; @@ -10,7 +9,6 @@ import { TerminusModule } from "@nestjs/terminus"; @Module({ imports: [ - DatabaseModule, CommonModule, ConfigModule.forRoot({ isGlobal: true }), BullModule.forRootAsync({ diff --git a/apps/unified-database-service/src/app.module.ts b/apps/unified-database-service/src/app.module.ts index 7e9ae861..2d0742a9 100644 --- a/apps/unified-database-service/src/app.module.ts +++ b/apps/unified-database-service/src/app.module.ts @@ -3,12 +3,11 @@ import { HealthModule } from "./health/health.module"; import { WebhookController } from "./webhook/webhook.controller"; import { AirtableModule } from "./airtable/airtable.module"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; -import { DatabaseModule } from "@terramatch-microservices/database"; import { APP_FILTER } from "@nestjs/core"; import { ScheduleModule } from "@nestjs/schedule"; @Module({ - imports: [SentryModule.forRoot(), ScheduleModule.forRoot(), DatabaseModule, HealthModule, AirtableModule], + imports: [SentryModule.forRoot(), ScheduleModule.forRoot(), HealthModule, AirtableModule], controllers: [WebhookController], providers: [ { diff --git a/apps/user-service/src/app.module.ts b/apps/user-service/src/app.module.ts index 0852c246..321f69ab 100644 --- a/apps/user-service/src/app.module.ts +++ b/apps/user-service/src/app.module.ts @@ -1,7 +1,6 @@ import { Module } from "@nestjs/common"; import { LoginController } from "./auth/login.controller"; import { AuthService } from "./auth/auth.service"; -import { DatabaseModule } from "@terramatch-microservices/database"; import { UsersController } from "./users/users.controller"; import { CommonModule } from "@terramatch-microservices/common"; import { HealthModule } from "./health/health.module"; @@ -14,7 +13,7 @@ import { VerificationUserService } from "./auth/verification-user.service"; import { UserCreationService } from "./users/user-creation.service"; @Module({ - imports: [SentryModule.forRoot(), DatabaseModule, CommonModule, HealthModule], + imports: [SentryModule.forRoot(), CommonModule, HealthModule], controllers: [LoginController, UsersController, ResetPasswordController, VerificationUserController], providers: [ { diff --git a/libs/common/src/lib/common.module.ts b/libs/common/src/lib/common.module.ts index 100013bc..1b3513d5 100644 --- a/libs/common/src/lib/common.module.ts +++ b/libs/common/src/lib/common.module.ts @@ -10,6 +10,7 @@ import { EmailService } from "./email/email.service"; import { MediaService } from "./media/media.service"; import { TemplateService } from "./email/template.service"; import { SlackService } from "./slack/slack.service"; +import { DatabaseModule } from "@terramatch-microservices/database"; @Module({ imports: [ @@ -23,7 +24,8 @@ import { SlackService } from "./slack/slack.service"; }), ConfigModule.forRoot({ isGlobal: true - }) + }), + DatabaseModule ], providers: [ PolicyService, From 605d883598ae6556a65452408cfe11fa093846c8 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 7 Apr 2025 16:31:42 -0700 Subject: [PATCH 04/98] [TM-1861] The user creation service doesn't use JWTs. --- .../src/users/user-creation.service.spec.ts | 13 ------------- .../user-service/src/users/user-creation.service.ts | 2 -- 2 files changed, 15 deletions(-) diff --git a/apps/user-service/src/users/user-creation.service.spec.ts b/apps/user-service/src/users/user-creation.service.spec.ts index 9df02817..868064ab 100644 --- a/apps/user-service/src/users/user-creation.service.spec.ts +++ b/apps/user-service/src/users/user-creation.service.spec.ts @@ -1,5 +1,4 @@ import { Test, TestingModule } from "@nestjs/testing"; -import { JwtService } from "@nestjs/jwt"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { ModelHasRole, Role, User, Verification } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; @@ -13,7 +12,6 @@ import { TemplateService } from "@terramatch-microservices/common/email/template describe("UserCreationService", () => { let service: UserCreationService; - let jwtService: DeepMocked; let emailService: DeepMocked; let localizationService: DeepMocked; let templateService: DeepMocked; @@ -68,10 +66,6 @@ describe("UserCreationService", () => { const module: TestingModule = await Test.createTestingModule({ providers: [ UserCreationService, - { - provide: JwtService, - useValue: (jwtService = createMock()) - }, { provide: EmailService, useValue: (emailService = createMock()) @@ -306,8 +300,6 @@ describe("UserCreationService", () => { jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - const token = "fake token"; - jwtService.signAsync.mockReturnValue(Promise.resolve(token)); jest.spyOn(Verification, "findOrCreate").mockImplementation(() => Promise.reject()); const result = await service.createNewUser(userNewRequest); @@ -334,9 +326,6 @@ describe("UserCreationService", () => { Promise.resolve([localizationBody, localizationSubject, localizationTitle, localizationCta]) ); - const token = "fake token"; - jwtService.signAsync.mockReturnValue(Promise.resolve(token)); - emailService.sendEmail.mockRejectedValue(null); const result = await service.createNewUser(userNewRequest); @@ -408,8 +397,6 @@ describe("UserCreationService", () => { jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - const token = "fake token"; - jwtService.signAsync.mockReturnValue(Promise.resolve(token)); jest.spyOn(Verification, "findOrCreate").mockRejectedValue(new Error("Verification creation failed")); await expect(service.createNewUser(userNewRequest)).resolves.toBeNull(); diff --git a/apps/user-service/src/users/user-creation.service.ts b/apps/user-service/src/users/user-creation.service.ts index 5c4c3f62..44c02081 100644 --- a/apps/user-service/src/users/user-creation.service.ts +++ b/apps/user-service/src/users/user-creation.service.ts @@ -1,5 +1,4 @@ import { Injectable, NotFoundException, UnprocessableEntityException } from "@nestjs/common"; -import { JwtService } from "@nestjs/jwt"; import { ModelHasRole, Role, User, Verification } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; @@ -16,7 +15,6 @@ export class UserCreationService { private roles = ["project-developer", "funder", "government"]; constructor( - private readonly jwtService: JwtService, private readonly emailService: EmailService, private readonly templateService: TemplateService, private readonly localizationService: LocalizationService From d8deae8e84063ad8fd9a1706006cc8b20b9702a5 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 7 Apr 2025 16:34:21 -0700 Subject: [PATCH 05/98] [TM-1861] Set up an event emitter pattern from the database lib. --- libs/common/src/lib/common.module.ts | 8 +++++-- libs/common/src/lib/events/event.service.ts | 16 ++++++++++++++ libs/database/src/lib/constants/status.ts | 16 +++++++------- libs/database/src/lib/database.module.ts | 23 ++++++++++++++++++-- package-lock.json | 24 ++++++++++++++++++--- package.json | 1 + 6 files changed, 73 insertions(+), 15 deletions(-) create mode 100644 libs/common/src/lib/events/event.service.ts diff --git a/libs/common/src/lib/common.module.ts b/libs/common/src/lib/common.module.ts index 1b3513d5..b5ff35cc 100644 --- a/libs/common/src/lib/common.module.ts +++ b/libs/common/src/lib/common.module.ts @@ -11,6 +11,8 @@ import { MediaService } from "./media/media.service"; import { TemplateService } from "./email/template.service"; import { SlackService } from "./slack/slack.service"; import { DatabaseModule } from "@terramatch-microservices/database"; +import { EventEmitterModule } from "@nestjs/event-emitter"; +import { EventService } from "./events/event.service"; @Module({ imports: [ @@ -25,7 +27,8 @@ import { DatabaseModule } from "@terramatch-microservices/database"; ConfigModule.forRoot({ isGlobal: true }), - DatabaseModule + DatabaseModule, + EventEmitterModule.forRoot() ], providers: [ PolicyService, @@ -34,7 +37,8 @@ import { DatabaseModule } from "@terramatch-microservices/database"; LocalizationService, MediaService, TemplateService, - SlackService + SlackService, + EventService ], exports: [PolicyService, JwtModule, EmailService, LocalizationService, MediaService, TemplateService, SlackService] }) diff --git a/libs/common/src/lib/events/event.service.ts b/libs/common/src/lib/events/event.service.ts new file mode 100644 index 00000000..2002cfb1 --- /dev/null +++ b/libs/common/src/lib/events/event.service.ts @@ -0,0 +1,16 @@ +import { Injectable } from "@nestjs/common"; +import { OnEvent } from "@nestjs/event-emitter"; +import { Project } from "@terramatch-microservices/database/entities"; +import { Model } from "sequelize-typescript"; + +/** + * A service to handle general events that are emitted in the common or database libraries, and + * should be handled in all of our various microservice apps. + */ +@Injectable() +export class EventService { + @OnEvent("database.statusUpdated") + handleStatusUpdated(model: Model) { + console.log("handle status change", { model: model instanceof Project }); + } +} diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 578499ea..b84b842b 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -1,5 +1,7 @@ import { States } from "../util/model-column-state-machine"; import { Nursery, Project, Site } from "../entities"; +import { Model } from "sequelize-typescript"; +import { DatabaseModule } from "../database.module"; export const STARTED = "started"; export const AWAITING_APPROVAL = "awaiting-approval"; @@ -9,6 +11,9 @@ export const NEEDS_MORE_INFORMATION = "needs-more-information"; export const ENTITY_STATUSES = [STARTED, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; export type EntityStatus = (typeof ENTITY_STATUSES)[number]; +const emitStatusUpdateHook = (from: string, model: Model) => { + DatabaseModule.emitModelEvent("statusUpdated", model); +}; export const EntityStatusStates: States = { default: STARTED, transitions: { @@ -18,14 +23,9 @@ export const EntityStatusStates: States [APPROVED]: [NEEDS_MORE_INFORMATION] }, afterTransitionHooks: { - [APPROVED]: (from, model) => { - console.log("After approved", { from, uuid: model.uuid }); - // TODO Send status change email - }, - [NEEDS_MORE_INFORMATION]: (from, model) => { - console.log("After needs more info", { from, uuid: model.uuid }); - // TODO Send status change email - } + [APPROVED]: emitStatusUpdateHook, + [AWAITING_APPROVAL]: emitStatusUpdateHook, + [NEEDS_MORE_INFORMATION]: emitStatusUpdateHook } }; diff --git a/libs/database/src/lib/database.module.ts b/libs/database/src/lib/database.module.ts index db3c6d53..29629b4c 100644 --- a/libs/database/src/lib/database.module.ts +++ b/libs/database/src/lib/database.module.ts @@ -2,6 +2,8 @@ import { Module } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; import { SequelizeModule } from "@nestjs/sequelize"; import { SequelizeConfigService } from "./sequelize-config.service"; +import { EventEmitter2, EventEmitterModule } from "@nestjs/event-emitter"; +import { Model } from "sequelize-typescript"; @Module({ imports: [ @@ -9,7 +11,24 @@ import { SequelizeConfigService } from "./sequelize-config.service"; SequelizeModule.forRootAsync({ useClass: SequelizeConfigService, imports: [ConfigModule.forRoot({ isGlobal: true })] - }) + }), + EventEmitterModule.forRoot() ] }) -export class DatabaseModule {} +export class DatabaseModule { + /** + * This is made statically available _only_ for use in the database model classes and their + * direct dependencies. Doing it this way is a bit of an anti-pattern, but our DB layer does + * not support NestJS dependency injection in a way that would give access to this event emitter + * in model code. + */ + private static eventEmitter?: EventEmitter2; + + constructor(readonly eventEmitter: EventEmitter2) { + DatabaseModule.eventEmitter = eventEmitter; + } + + static emitModelEvent(eventName: string, model: Model) { + this.eventEmitter?.emit(`database.${eventName}`, model); + } +} diff --git a/package-lock.json b/package-lock.json index 050ddba8..970fd950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@nestjs/common": "^11.0.11", "@nestjs/config": "^4.0.1", "@nestjs/core": "^11.0.11", + "@nestjs/event-emitter": "^3.0.1", "@nestjs/jwt": "^11.0.0", "@nestjs/platform-express": "^11.0.11", "@nestjs/schedule": "^5.0.1", @@ -3530,9 +3531,9 @@ } }, "node_modules/@nestjs/common": { - "version": "11.0.11", - "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.0.11.tgz", - "integrity": "sha512-b3zYiho5/XGCnLa7W2hHv5ecSBR1huQrXCHu6pxd+g2HY2B7sKP5CXHMv4gHYqpIqu4ClOb7Q4tLKXMp9LyLUg==", + "version": "11.0.13", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.0.13.tgz", + "integrity": "sha512-cXqXJPQTcJIYqT8GtBYqjYY9sklCBqp/rh9z1R40E60gWnsU598YIQWkojSFRI9G7lT/+uF+jqSrg/CMPBk7QQ==", "dependencies": { "iterare": "1.2.1", "tslib": "2.8.1", @@ -3611,6 +3612,18 @@ } } }, + "node_modules/@nestjs/event-emitter": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-3.0.1.tgz", + "integrity": "sha512-0Ln/x+7xkU6AJFOcQI9tIhUMXVF7D5itiaQGOyJbXtlAfAIt8gzDdJm+Im7cFzKoWkiW5nCXCPh6GSvdQd/3Dw==", + "dependencies": { + "eventemitter2": "6.4.9" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, "node_modules/@nestjs/jwt": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/@nestjs/jwt/-/jwt-11.0.0.tgz", @@ -10309,6 +10322,11 @@ "node": ">=6" } }, + "node_modules/eventemitter2": { + "version": "6.4.9", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", + "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" + }, "node_modules/eventemitter3": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", diff --git a/package.json b/package.json index 78ee1804..fae62a3b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@nestjs/common": "^11.0.11", "@nestjs/config": "^4.0.1", "@nestjs/core": "^11.0.11", + "@nestjs/event-emitter": "^3.0.1", "@nestjs/jwt": "^11.0.0", "@nestjs/platform-express": "^11.0.11", "@nestjs/schedule": "^5.0.1", From 9aed845eb7d40043f97ebb347b9753a1bb935f79 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 8 Apr 2025 11:29:04 -0700 Subject: [PATCH 06/98] [TM-1861] Wire up BullMQ for email sends. --- .../src/airtable/airtable.module.ts | 15 +---- .../src/airtable/queue-health.indicator.ts | 1 + libs/common/src/lib/common.module.ts | 25 ++++++- libs/common/src/lib/email/email.processor.ts | 65 +++++++++++++++++++ libs/common/src/lib/events/event.service.ts | 27 +++++++- tm-v3-cli/src/repl/command.ts | 2 +- 6 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 libs/common/src/lib/email/email.processor.ts diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts index e93a86c0..f52253c7 100644 --- a/apps/unified-database-service/src/airtable/airtable.module.ts +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -1,5 +1,5 @@ import { CommonModule } from "@terramatch-microservices/common"; -import { ConfigModule, ConfigService } from "@nestjs/config"; +import { ConfigModule } from "@nestjs/config"; import { BullModule } from "@nestjs/bullmq"; import { Module } from "@nestjs/common"; import { AirtableService } from "./airtable.service"; @@ -11,19 +11,6 @@ import { TerminusModule } from "@nestjs/terminus"; imports: [ CommonModule, ConfigModule.forRoot({ isGlobal: true }), - BullModule.forRootAsync({ - imports: [ConfigModule.forRoot({ isGlobal: true })], - inject: [ConfigService], - useFactory: (configService: ConfigService) => ({ - connection: { - host: configService.get("REDIS_HOST"), - port: configService.get("REDIS_PORT"), - prefix: "unified-database-service", - // Use TLS in AWS - ...(process.env.NODE_ENV !== "development" ? { tls: {} } : null) - } - }) - }), BullModule.registerQueue({ name: "airtable" }), TerminusModule ], diff --git a/apps/unified-database-service/src/airtable/queue-health.indicator.ts b/apps/unified-database-service/src/airtable/queue-health.indicator.ts index fd54e802..a9a1fd82 100644 --- a/apps/unified-database-service/src/airtable/queue-health.indicator.ts +++ b/apps/unified-database-service/src/airtable/queue-health.indicator.ts @@ -3,6 +3,7 @@ import { Injectable } from "@nestjs/common"; import { HealthIndicatorService } from "@nestjs/terminus"; import { Queue } from "bullmq"; +// TODO Add to common for all services to use. @Injectable() export class QueueHealthIndicator { constructor( diff --git a/libs/common/src/lib/common.module.ts b/libs/common/src/lib/common.module.ts index b5ff35cc..89a33079 100644 --- a/libs/common/src/lib/common.module.ts +++ b/libs/common/src/lib/common.module.ts @@ -13,6 +13,8 @@ import { SlackService } from "./slack/slack.service"; import { DatabaseModule } from "@terramatch-microservices/database"; import { EventEmitterModule } from "@nestjs/event-emitter"; import { EventService } from "./events/event.service"; +import { BullModule } from "@nestjs/bullmq"; +import { EmailProcessor } from "./email/email.processor"; @Module({ imports: [ @@ -28,7 +30,25 @@ import { EventService } from "./events/event.service"; isGlobal: true }), DatabaseModule, - EventEmitterModule.forRoot() + // Event Emitter is used for sending lightweight messages about events typically from the DB + // layer to processes that want to hear about specific types of DB updates. + EventEmitterModule.forRoot(), + // BullMQ is used for scheduling tasks that should not happen in the context of a typical API + // request (like sending en email). + BullModule.forRootAsync({ + imports: [ConfigModule.forRoot({ isGlobal: true })], + inject: [ConfigService], + useFactory: (configService: ConfigService) => ({ + connection: { + host: configService.get("REDIS_HOST"), + port: configService.get("REDIS_PORT"), + prefix: "terramatch-microservices", + // Use TLS in AWS + ...(process.env["NODE_ENV"] !== "development" ? { tls: {} } : null) + } + }) + }), + BullModule.registerQueue({ name: "email" }) ], providers: [ PolicyService, @@ -38,7 +58,8 @@ import { EventService } from "./events/event.service"; MediaService, TemplateService, SlackService, - EventService + EventService, + EmailProcessor ], exports: [PolicyService, JwtModule, EmailService, LocalizationService, MediaService, TemplateService, SlackService] }) diff --git a/libs/common/src/lib/email/email.processor.ts b/libs/common/src/lib/email/email.processor.ts new file mode 100644 index 00000000..296a33e6 --- /dev/null +++ b/libs/common/src/lib/email/email.processor.ts @@ -0,0 +1,65 @@ +import { OnWorkerEvent, Processor, WorkerHost } from "@nestjs/bullmq"; +import { EmailService } from "./email.service"; +import { Job } from "bullmq"; +import { ENTITY_MODELS, EntityType } from "@terramatch-microservices/database/constants/entities"; +import { InternalServerErrorException, NotImplementedException } from "@nestjs/common"; +import * as Sentry from "@sentry/node"; +import { TMLogger } from "../util/tm-logger"; +import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; + +export type StatusUpdateData = { + type: EntityType; + id: number; +}; + +/** + * Watches for jobs related to sending email + */ +@Processor("email") +export class EmailProcessor extends WorkerHost { + private readonly logger = new TMLogger(EmailProcessor.name); + + constructor(private readonly emailService: EmailService) { + super(); + } + + async process(job: Job) { + const { name, data } = job; + switch (name) { + case "statusUpdate": + return await this.sendStatusUpdateEmail(data as StatusUpdateData); + + default: + throw new NotImplementedException(`Unknown job type: ${name}`); + } + } + + @OnWorkerEvent("failed") + async onFailed(job: Job, error: Error) { + Sentry.captureException(error); + this.logger.error(`Worker event failed: ${JSON.stringify(job)}`, error.stack); + } + + private async sendStatusUpdateEmail({ type, id }: StatusUpdateData) { + const entityClass = ENTITY_MODELS[type]; + if (entityClass == null) { + throw new InternalServerErrorException(`Entity model class not found for entity type [${type}]`); + } + + const entity = await entityClass.findOne({ where: { id } }); + if (entity == null) { + throw new InternalServerErrorException(`Entity instance not found for id [type=${type}, id=${id}]`); + } + + if ( + ![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.status) && + entity.updateRequestStatus !== NEEDS_MORE_INFORMATION + ) { + return; + } + + this.logger.log( + `Sending status update email [type=${type}, id=${id}, status=${entity.status}, urStatus=${entity.updateRequestStatus}]` + ); + } +} diff --git a/libs/common/src/lib/events/event.service.ts b/libs/common/src/lib/events/event.service.ts index 2002cfb1..d7421569 100644 --- a/libs/common/src/lib/events/event.service.ts +++ b/libs/common/src/lib/events/event.service.ts @@ -1,7 +1,11 @@ import { Injectable } from "@nestjs/common"; import { OnEvent } from "@nestjs/event-emitter"; -import { Project } from "@terramatch-microservices/database/entities"; import { Model } from "sequelize-typescript"; +import { InjectQueue } from "@nestjs/bullmq"; +import { TMLogger } from "../util/tm-logger"; +import { Queue } from "bullmq"; +import { ENTITY_MODELS } from "@terramatch-microservices/database/constants/entities"; +import { StatusUpdateData } from "../email/email.processor"; /** * A service to handle general events that are emitted in the common or database libraries, and @@ -9,8 +13,25 @@ import { Model } from "sequelize-typescript"; */ @Injectable() export class EventService { + private readonly logger = new TMLogger(EventService.name); + + constructor(@InjectQueue("email") private readonly emailQueue: Queue) {} + @OnEvent("database.statusUpdated") - handleStatusUpdated(model: Model) { - console.log("handle status change", { model: model instanceof Project }); + async handleStatusUpdated(model: Model & { status: string }) { + this.logger.log("Received model status update", { + type: model.constructor.name, + id: model.id, + status: model.status + }); + + const type = Object.entries(ENTITY_MODELS).find(([, entityClass]) => model instanceof entityClass)?.[0]; + if (type == null) { + this.logger.error("Status update not an entity model", { type: model.constructor.name }); + return; + } + + this.logger.log("Sending status update to email queue", { type, id: model.id }); + await this.emailQueue.add("statusUpdate", { type, id: model.id } as StatusUpdateData); } } diff --git a/tm-v3-cli/src/repl/command.ts b/tm-v3-cli/src/repl/command.ts index d044da20..e2a24f84 100644 --- a/tm-v3-cli/src/repl/command.ts +++ b/tm-v3-cli/src/repl/command.ts @@ -29,7 +29,7 @@ const startLocalRepl = async (service: Service) => { }); debug(`Launching REPL process for ${service}`); - spawn("node", [`dist/apps/${service}-repl`], { stdio: "inherit" }); + spawn("node", [`dist/apps/${service}-repl`], { stdio: "inherit", env: { ...process.env, NODE_ENV: "development" } }); }; const getTaskId = async (service: Service, env: Environment) => { From 3bd10e0ac148254a3ff16c9c978ba66c8444c35c Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 8 Apr 2025 13:23:52 -0700 Subject: [PATCH 07/98] [TM-1861] Improve i18n template email handling. --- .../src/auth/reset-password.service.ts | 84 +++------------ .../src/users/user-creation.service.ts | 102 +++--------------- apps/user-service/src/util/constants.ts | 1 + libs/common/src/lib/email/TemplateParams.ts | 13 --- libs/common/src/lib/email/email.processor.ts | 51 +++++++-- libs/common/src/lib/email/email.service.ts | 35 +++++- libs/common/src/lib/email/template.service.ts | 9 +- .../lib/localization/localization.service.ts | 33 ++++-- libs/database/src/lib/constants/entities.ts | 16 +++ .../src/lib/entities/project-user.entity.ts | 4 + 10 files changed, 155 insertions(+), 193 deletions(-) create mode 100644 apps/user-service/src/util/constants.ts delete mode 100644 libs/common/src/lib/email/TemplateParams.ts diff --git a/apps/user-service/src/auth/reset-password.service.ts b/apps/user-service/src/auth/reset-password.service.ts index b06c5674..28b6d76b 100644 --- a/apps/user-service/src/auth/reset-password.service.ts +++ b/apps/user-service/src/auth/reset-password.service.ts @@ -3,88 +3,38 @@ import bcrypt from "bcryptjs"; import { JwtService } from "@nestjs/jwt"; import { User } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; -import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; -import { TemplateService } from "@terramatch-microservices/common/email/template.service"; +import { EMAIL_TEMPLATE } from "../util/constants"; + +const EMAIL_KEYS = { + body: "reset-password.body", + subject: "reset-password.subject", + title: "reset-password.title", + cta: "reset-password.cta" +} as const; @Injectable() export class ResetPasswordService { protected readonly logger = new TMLogger(ResetPasswordService.name); - constructor( - private readonly jwtService: JwtService, - private readonly emailService: EmailService, - private readonly localizationService: LocalizationService, - private readonly templateService: TemplateService - ) {} + constructor(private readonly jwtService: JwtService, private readonly emailService: EmailService) {} async sendResetPasswordEmail(emailAddress: string, callbackUrl: string) { - const user = await User.findOne({ where: { emailAddress }, attributes: ["id", "uuid", "locale", "emailAddress"] }); + const user = await User.findOne({ where: { emailAddress }, attributes: ["id", "uuid", "locale"] }); if (user == null) { throw new NotFoundException("User not found"); } - const bodyKey = "reset-password.body"; - const subjectKey = "reset-password.subject"; - const titleKey = "reset-password.title"; - const ctaKey = "reset-password.cta"; - - const localizationKeys = await this.localizationService.getLocalizationKeys([ - bodyKey, - subjectKey, - titleKey, - ctaKey - ]); - - if (!localizationKeys.length) { - throw new NotFoundException("Localizations not found"); - } - - const bodyLocalization = localizationKeys.find(x => x.key == bodyKey); - const subjectLocalization = localizationKeys.find(x => x.key == subjectKey); - const titleLocalization = localizationKeys.find(x => x.key == titleKey); - const ctaLocalization = localizationKeys.find(x => x.key == ctaKey); - - if (bodyLocalization == null) { - throw new NotFoundException("Localization body not found"); - } - - if (subjectLocalization == null) { - throw new NotFoundException("Localization subject not found"); - } - - if (titleLocalization == null) { - throw new NotFoundException("Localization title not found"); - } - - if (ctaLocalization == null) { - throw new NotFoundException("Localization CTA not found"); - } - - const resetToken = await this.jwtService.signAsync({ sub: user.uuid }, { expiresIn: "2h" }); + const { uuid, locale } = user; + const resetToken = await this.jwtService.signAsync({ sub: uuid }, { expiresIn: "2h" }); const resetLink = `${callbackUrl}/${resetToken}`; - const bodyEmail = await this.formatEmail( - user.locale, - bodyLocalization.value, - titleLocalization.value, - ctaLocalization.value, - resetLink - ); - await this.emailService.sendEmail(user.emailAddress, subjectLocalization.value, bodyEmail); - - return { email: user.emailAddress, uuid: user.uuid }; - } - - private async formatEmail(locale: string, body: string, title: string, cta: string, callbackUrl: string) { - const emailData = { - title: await this.localizationService.translate(title, locale), - body: await this.localizationService.translate(body, locale), - link: callbackUrl, - cta: await this.localizationService.translate(cta, locale), + await this.emailService.sendI18nTemplateEmail(EMAIL_TEMPLATE, emailAddress, locale, EMAIL_KEYS, { + link: resetLink, monitoring: "monitoring" - }; - return this.templateService.render("user-service/views/default-email.hbs", emailData); + }); + + return { email: emailAddress, uuid }; } async resetPassword(resetToken: string, newPassword: string) { diff --git a/apps/user-service/src/users/user-creation.service.ts b/apps/user-service/src/users/user-creation.service.ts index 44c02081..3e82621d 100644 --- a/apps/user-service/src/users/user-creation.service.ts +++ b/apps/user-service/src/users/user-creation.service.ts @@ -1,71 +1,34 @@ import { Injectable, NotFoundException, UnprocessableEntityException } from "@nestjs/common"; import { ModelHasRole, Role, User, Verification } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; -import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { UserNewRequest } from "./dto/user-new-request.dto"; import crypto from "node:crypto"; -import { TemplateService } from "@terramatch-microservices/common/email/template.service"; import { omit } from "lodash"; import bcrypt from "bcryptjs"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; +import { EMAIL_TEMPLATE } from "../util/constants"; + +const EMAIL_KEYS = { + body: "user-verification.body", + subjectKey: "user-verification.subject", + titleKey: "user-verification.title", + ctaKey: "user-verification.cta" +} as const; @Injectable() export class UserCreationService { protected readonly logger = new TMLogger(UserCreationService.name); private roles = ["project-developer", "funder", "government"]; - constructor( - private readonly emailService: EmailService, - private readonly templateService: TemplateService, - private readonly localizationService: LocalizationService - ) {} + constructor(private readonly emailService: EmailService) {} async createNewUser(request: UserNewRequest): Promise { - const bodyKey = "user-verification.body"; - const subjectKey = "user-verification.subject"; - const titleKey = "user-verification.title"; - const ctaKey = "user-verification.cta"; - - const localizationKeys = await this.localizationService.getLocalizationKeys([ - bodyKey, - subjectKey, - titleKey, - ctaKey - ]); - - if (!localizationKeys.length) { - throw new NotFoundException("Localizations not found"); - } - - const bodyLocalization = localizationKeys.find(x => x.key == bodyKey); - const subjectLocalization = localizationKeys.find(x => x.key == subjectKey); - const titleLocalization = localizationKeys.find(x => x.key == titleKey); - const ctaLocalization = localizationKeys.find(x => x.key == ctaKey); - - if (bodyLocalization == null) { - throw new NotFoundException("Localization body not found"); - } - - if (subjectLocalization == null) { - throw new NotFoundException("Localization subject not found"); - } - - if (titleLocalization == null) { - throw new NotFoundException("Localization title not found"); - } - - if (ctaLocalization == null) { - throw new NotFoundException("Localization CTA not found"); - } - const role = request.role; - if (!this.roles.includes(role)) { throw new UnprocessableEntityException("Role not valid"); } const roleEntity = await Role.findOne({ where: { name: role } }); - if (roleEntity == null) { throw new NotFoundException("Role not found"); } @@ -91,15 +54,7 @@ export class UserCreationService { const token = crypto.randomBytes(32).toString("hex"); await this.saveUserVerification(user.id, token); - await this.sendEmailVerification( - user, - token, - subjectLocalization.value, - bodyLocalization.value, - titleLocalization.value, - ctaLocalization.value, - callbackUrl - ); + await this.sendEmailVerification(user, token, callbackUrl); return user; } catch (error) { this.logger.error(error); @@ -107,42 +62,11 @@ export class UserCreationService { } } - private async formatEmail( - locale: string, - token: string, - body: string, - title: string, - cta: string, - callbackUrl: string - ) { - const emailData = { - title: await this.localizationService.translate(title, locale), - body: await this.localizationService.translate(body, locale), + private async sendEmailVerification({ emailAddress, locale }: User, token: string, callbackUrl: string) { + await this.emailService.sendI18nTemplateEmail(EMAIL_TEMPLATE, emailAddress, locale, EMAIL_KEYS, { link: `${callbackUrl}/${token}`, - cta: await this.localizationService.translate(cta, locale), monitoring: "monitoring" - }; - return this.templateService.render("user-service/views/default-email.hbs", emailData); - } - - private async sendEmailVerification( - user: User, - token: string, - subject: string, - bodyLocalization: string, - titleLocalization: string, - ctaLocalization: string, - callbackUrl: string - ) { - const body = await this.formatEmail( - user.locale, - token, - bodyLocalization, - titleLocalization, - ctaLocalization, - callbackUrl - ); - await this.emailService.sendEmail(user.emailAddress, subject, body); + }); } private async saveUserVerification(userId: number, token: string) { diff --git a/apps/user-service/src/util/constants.ts b/apps/user-service/src/util/constants.ts new file mode 100644 index 00000000..a55f8bd4 --- /dev/null +++ b/apps/user-service/src/util/constants.ts @@ -0,0 +1 @@ +export const EMAIL_TEMPLATE = "user-service/views/default-email.hbf"; diff --git a/libs/common/src/lib/email/TemplateParams.ts b/libs/common/src/lib/email/TemplateParams.ts deleted file mode 100644 index 18c5c322..00000000 --- a/libs/common/src/lib/email/TemplateParams.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface TemplateParams { - backendUrl?: string; - banner?: string; - year?: number; - monitoring?: string; - invite?: string; - transactional?: string; - unsubscribe?: string; - title: string; - body: string; - link: string; - cta: string; -} diff --git a/libs/common/src/lib/email/email.processor.ts b/libs/common/src/lib/email/email.processor.ts index 296a33e6..72b8cb04 100644 --- a/libs/common/src/lib/email/email.processor.ts +++ b/libs/common/src/lib/email/email.processor.ts @@ -1,11 +1,19 @@ import { OnWorkerEvent, Processor, WorkerHost } from "@nestjs/bullmq"; import { EmailService } from "./email.service"; import { Job } from "bullmq"; -import { ENTITY_MODELS, EntityType } from "@terramatch-microservices/database/constants/entities"; +import { + ENTITY_MODELS, + EntityModel, + EntityType, + getProjectId +} from "@terramatch-microservices/database/constants/entities"; import { InternalServerErrorException, NotImplementedException } from "@nestjs/common"; import * as Sentry from "@sentry/node"; import { TMLogger } from "../util/tm-logger"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; +import { ProjectUser, User } from "@terramatch-microservices/database/entities"; +import { isEmpty, map } from "lodash"; +import { Op } from "sequelize"; export type StatusUpdateData = { type: EntityType; @@ -46,20 +54,47 @@ export class EmailProcessor extends WorkerHost { throw new InternalServerErrorException(`Entity model class not found for entity type [${type}]`); } - const entity = await entityClass.findOne({ where: { id } }); + const attributes = ["status", "updateRequestStatus"]; + const attributeKeys = Object.keys(entityClass.getAttributes()); + for (const parentId of ["projectId", "siteId", "nurseryId"]) { + if (attributeKeys.includes(parentId)) attributes.push(parentId); + } + const entity = await entityClass.findOne({ where: { id }, attributes }); if (entity == null) { throw new InternalServerErrorException(`Entity instance not found for id [type=${type}, id=${id}]`); } - if ( - ![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.status) && - entity.updateRequestStatus !== NEEDS_MORE_INFORMATION - ) { + const status = + entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION + ? NEEDS_MORE_INFORMATION + : entity.status; + if (![APPROVED, NEEDS_MORE_INFORMATION].includes(status)) return; + + const logExtras = + `[type=${type}, id=${id}, status=${entity.status}, updateRequestStatus=${entity.updateRequestStatus}` as const; + this.logger.log(`Sending status update email ${logExtras}`); + + const to = await this.getEntityUserEmails(entity); + if (isEmpty(to)) { + this.logger.debug(`No addresses found to send entity update to ${logExtras}`); + return; + } + } + + private async getEntityUserEmails(entity: EntityModel) { + const projectId = await getProjectId(entity); + if (projectId == null) { + this.logger.error(`Could not find project ID for entity [type=${entity.constructor.name}, id=${entity.id}]`); return; } - this.logger.log( - `Sending status update email [type=${type}, id=${id}, status=${entity.status}, urStatus=${entity.updateRequestStatus}]` + const emailAddresses = map( + await User.findAll({ + where: { id: { [Op.in]: ProjectUser.projectUsersSubquery(projectId) } }, + attributes: ["emailAddress"] + }), + "emailAddress" ); + return this.emailService.filterEntityEmailRecipients(emailAddresses); } } diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index e64c2d27..3cec3032 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -1,13 +1,20 @@ -import { Injectable } from "@nestjs/common"; +import { Injectable, InternalServerErrorException } from "@nestjs/common"; import * as nodemailer from "nodemailer"; import { ConfigService } from "@nestjs/config"; import * as Mail from "nodemailer/lib/mailer"; +import { Dictionary, isEmpty } from "lodash"; +import { LocalizationService } from "../localization/localization.service"; +import { TemplateService } from "./template.service"; @Injectable() export class EmailService { private transporter: nodemailer.Transporter; - constructor(private readonly configService: ConfigService) { + constructor( + private readonly configService: ConfigService, + private readonly localizationService: LocalizationService, + private readonly templateService: TemplateService + ) { this.transporter = nodemailer.createTransport({ host: this.configService.get("MAIL_HOST"), port: this.configService.get("MAIL_PORT"), @@ -19,7 +26,29 @@ export class EmailService { }); } - async sendEmail(to: string, subject: string, body: string) { + filterEntityEmailRecipients(recipients: string[]) { + const entityDoNotEmailList = this.configService.get("ENTITY_UPDATE_DO_NOT_EMAIL"); + if (isEmpty(entityDoNotEmailList)) return recipients; + + const doNotEmail = entityDoNotEmailList.split(","); + return recipients.filter(address => !doNotEmail.includes(address)); + } + + async sendI18nTemplateEmail( + templatePath: string, + to: string | string[], + locale: string, + i18nKeys: Dictionary, + additionalValues: Dictionary = {} + ) { + if (i18nKeys["subject"] == null) throw new InternalServerErrorException("Email subject is required"); + const { subject, ...translated } = await this.localizationService.translateKeys(i18nKeys, locale); + + const renderedBody = this.templateService.render(templatePath, { ...translated, ...additionalValues }); + await this.sendEmail(to, subject, renderedBody); + } + + async sendEmail(to: string | string[], subject: string, body: string) { const headers = {} as { [p: string]: string | string[] | { prepared: boolean; value: string } }; const mailOptions: Mail.Options = { from: this.configService.get("MAIL_FROM_ADDRESS"), diff --git a/libs/common/src/lib/email/template.service.ts b/libs/common/src/lib/email/template.service.ts index 72af2ae4..5f27b7c1 100644 --- a/libs/common/src/lib/email/template.service.ts +++ b/libs/common/src/lib/email/template.service.ts @@ -3,7 +3,6 @@ import { ConfigService } from "@nestjs/config"; import * as Handlebars from "handlebars"; import * as fs from "fs"; import * as path from "path"; -import { TemplateParams } from "./TemplateParams"; import { Dictionary } from "factory-girl-ts"; @Injectable() @@ -20,15 +19,15 @@ export class TemplateService { return (this.templates[template] = Handlebars.compile(templateSource)); } - render(templatePath: string, data: TemplateParams): string { + render(templatePath: string, data: Dictionary): string { const params = { - ...data, backendUrl: this.configService.get("EMAIL_IMAGE_BASE_URL"), banner: null, invite: null, monitoring: null, - transactional: data.transactional || null, - year: new Date().getFullYear() + transactional: null, + year: new Date().getFullYear(), + ...data }; return this.getCompiledTemplate(templatePath)(params); } diff --git a/libs/common/src/lib/localization/localization.service.ts b/libs/common/src/lib/localization/localization.service.ts index c79fc676..8e026c34 100644 --- a/libs/common/src/lib/localization/localization.service.ts +++ b/libs/common/src/lib/localization/localization.service.ts @@ -1,9 +1,10 @@ -import { Injectable } from "@nestjs/common"; +import { Injectable, NotFoundException } from "@nestjs/common"; import { i18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; import { i18nItem } from "@terramatch-microservices/database/entities/i18n-item.entity"; import { Op } from "sequelize"; import { ConfigService } from "@nestjs/config"; import { normalizeLocale, tx } from "@transifex/native"; +import { Dictionary } from "lodash"; @Injectable() export class LocalizationService { @@ -29,16 +30,32 @@ export class LocalizationService { return await i18nTranslation.findOne({ where: { i18nItemId: itemId, language: locale } }); } + async translateKeys(keyMap: Dictionary, locale: string) { + const keyStrings = Object.values(keyMap); + const keys = (await this.getLocalizationKeys(keyStrings)) ?? []; + + const result: Dictionary = {}; + await Promise.all( + Object.entries(keyMap).map(async ([prop, key]) => { + const value = keys.find(record => record.key === key)?.value; + if (value == null) throw new NotFoundException(`Localization key not found [${key}]`); + + const translated = await this.translate(value, locale); + if (translated != null) result[prop] = translated; + }) + ); + + return result; + } + async translate(content: string, locale: string): Promise { const itemResult = await this.getItemI18n(content); - if (itemResult == null) { - return content; - } + if (itemResult == null) return content; + const translationResult = await this.getTranslateItem(itemResult.id, locale); - if (translationResult == null) { - return content; - } - return translationResult.shortValue || translationResult.longValue; + if (translationResult == null) return content; + + return translationResult.shortValue ?? translationResult.longValue; } /** diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 38326b06..31acc9a3 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -24,3 +24,19 @@ export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { sites: Site, nurseries: Nursery }; + +/** + * Get the project ID associated with the given entity, which may be any one of EntityModels defined in this file. + * + * Note: this method does require that for sites, nurseries and project reports, the entity's projectId must have + * been loaded when it was fetched from the DB, or `undefined` will be returned. Likewise, For site reports and + * nursery reports, the associated parent entity's id must be included. + */ +export async function getProjectId(entity: EntityModel) { + if (entity instanceof Project) return entity.id; + if (entity instanceof Site || entity instanceof Nursery || entity instanceof ProjectReport) return entity.projectId; + + const parentClass: ModelCtor = entity instanceof SiteReport ? Site : Nursery; + const parentId = entity instanceof SiteReport ? entity.siteId : entity.nurseryId; + return (await parentClass.findOne({ where: { id: parentId }, attributes: ["projectId"] }))?.projectId; +} diff --git a/libs/database/src/lib/entities/project-user.entity.ts b/libs/database/src/lib/entities/project-user.entity.ts index 72fc7162..f83e3e5b 100644 --- a/libs/database/src/lib/entities/project-user.entity.ts +++ b/libs/database/src/lib/entities/project-user.entity.ts @@ -14,6 +14,10 @@ export class ProjectUser extends Model { return Subquery.select(ProjectUser, "projectId").eq("userId", userId).eq("isManaging", true).literal; } + static projectUsersSubquery(projectId: number) { + return Subquery.select(ProjectUser, "userId").eq("projectId", projectId).literal; + } + @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) From 7d9429b223a5120e72b862b612403b3d1de6a684 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 8 Apr 2025 16:51:55 -0700 Subject: [PATCH 08/98] [TM-1861] Get the entity status email fully wired up. --- .env.local.sample | 10 +- .../src/auth/reset-password.service.ts | 6 +- .../src/users/user-creation.service.ts | 6 +- apps/user-service/src/util/constants.ts | 1 - apps/user-service/webpack.config.js | 2 +- libs/common/src/lib/email/email-processor.ts | 5 + libs/common/src/lib/email/email.processor.ts | 79 ++------- libs/common/src/lib/email/email.service.ts | 41 ++++- .../lib/email/entity-status-update.email.ts | 159 ++++++++++++++++++ libs/common/src/lib/email/template.service.ts | 14 +- .../localization/localization.service.spec.ts | 18 +- .../lib/localization/localization.service.ts | 68 +++----- libs/common/src/lib/util/bootstrap-repl.ts | 3 +- .../common/src/lib}/views/default-email.hbs | 0 libs/database/src/lib/constants/entities.ts | 9 + .../src/lib/entities/i18n-item.entity.ts | 2 +- .../lib/entities/i18n-translation.entity.ts | 2 +- libs/database/src/lib/entities/index.ts | 1 + .../lib/entities/localization-keys.entity.ts | 6 +- .../src/lib/entities/update-request.entity.ts | 84 +++++++++ 20 files changed, 366 insertions(+), 150 deletions(-) delete mode 100644 apps/user-service/src/util/constants.ts create mode 100644 libs/common/src/lib/email/email-processor.ts create mode 100644 libs/common/src/lib/email/entity-status-update.email.ts rename {apps/user-service/src => libs/common/src/lib}/views/default-email.hbs (100%) create mode 100644 libs/database/src/lib/entities/update-request.entity.ts diff --git a/.env.local.sample b/.env.local.sample index c44b3f41..0c7a2b5f 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -28,11 +28,10 @@ AIRTABLE_BASE_ID= SLACK_API_KEY= UDB_SLACK_CHANNEL= -MAIL_HOST=email-smtp.eu-west-1.amazonaws.com -MAIL_PORT=587 -MAIL_USERNAME= -MAIL_PASSWORD= -MAIL_FROM_ADDRESS=staging@terramatch.org +# Points to the mail catcher running from the PHP docker-compose. Visit localhost:8000 to see "sent" emails. +MAIL_HOST=localhost +MAIL_PORT=1025 +MAIL_FROM_ADDRESS=local@terramatch.org # If set, email is redirected from the original recipient to this list of addresses. The original # recipients are encoded in the X-Original-Recipients header MAIL_RECIPIENTS= @@ -40,3 +39,4 @@ MAIL_RECIPIENTS= TRANSIFEX_TOKEN= EMAIL_IMAGE_BASE_URL=https://api.terramatch.org/images +APP_FRONT_END=http://localhost:3000 diff --git a/apps/user-service/src/auth/reset-password.service.ts b/apps/user-service/src/auth/reset-password.service.ts index 28b6d76b..5d7ea978 100644 --- a/apps/user-service/src/auth/reset-password.service.ts +++ b/apps/user-service/src/auth/reset-password.service.ts @@ -4,7 +4,6 @@ import { JwtService } from "@nestjs/jwt"; import { User } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; -import { EMAIL_TEMPLATE } from "../util/constants"; const EMAIL_KEYS = { body: "reset-password.body", @@ -29,9 +28,8 @@ export class ResetPasswordService { const resetToken = await this.jwtService.signAsync({ sub: uuid }, { expiresIn: "2h" }); const resetLink = `${callbackUrl}/${resetToken}`; - await this.emailService.sendI18nTemplateEmail(EMAIL_TEMPLATE, emailAddress, locale, EMAIL_KEYS, { - link: resetLink, - monitoring: "monitoring" + await this.emailService.sendI18nTemplateEmail(emailAddress, locale, EMAIL_KEYS, { + additionalValues: { link: resetLink, monitoring: "monitoring" } }); return { email: emailAddress, uuid }; diff --git a/apps/user-service/src/users/user-creation.service.ts b/apps/user-service/src/users/user-creation.service.ts index 3e82621d..c1833cc0 100644 --- a/apps/user-service/src/users/user-creation.service.ts +++ b/apps/user-service/src/users/user-creation.service.ts @@ -6,7 +6,6 @@ import crypto from "node:crypto"; import { omit } from "lodash"; import bcrypt from "bcryptjs"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; -import { EMAIL_TEMPLATE } from "../util/constants"; const EMAIL_KEYS = { body: "user-verification.body", @@ -63,9 +62,8 @@ export class UserCreationService { } private async sendEmailVerification({ emailAddress, locale }: User, token: string, callbackUrl: string) { - await this.emailService.sendI18nTemplateEmail(EMAIL_TEMPLATE, emailAddress, locale, EMAIL_KEYS, { - link: `${callbackUrl}/${token}`, - monitoring: "monitoring" + await this.emailService.sendI18nTemplateEmail(emailAddress, locale, EMAIL_KEYS, { + additionalValues: { link: `${callbackUrl}/${token}`, monitoring: "monitoring" } }); } diff --git a/apps/user-service/src/util/constants.ts b/apps/user-service/src/util/constants.ts deleted file mode 100644 index a55f8bd4..00000000 --- a/apps/user-service/src/util/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const EMAIL_TEMPLATE = "user-service/views/default-email.hbf"; diff --git a/apps/user-service/webpack.config.js b/apps/user-service/webpack.config.js index 143e0519..19593752 100644 --- a/apps/user-service/webpack.config.js +++ b/apps/user-service/webpack.config.js @@ -11,7 +11,7 @@ module.exports = { compiler: "tsc", main: "./src/main.ts", tsConfig: "./tsconfig.app.json", - assets: ["./src/assets", "./src/views"], + assets: ["./src/assets"], optimization: false, outputHashing: "none", generatePackageJson: true diff --git a/libs/common/src/lib/email/email-processor.ts b/libs/common/src/lib/email/email-processor.ts new file mode 100644 index 00000000..226f1b7f --- /dev/null +++ b/libs/common/src/lib/email/email-processor.ts @@ -0,0 +1,5 @@ +import { EmailService } from "./email.service"; + +export abstract class EmailProcessor { + abstract send(emailService: EmailService): Promise; +} diff --git a/libs/common/src/lib/email/email.processor.ts b/libs/common/src/lib/email/email.processor.ts index 72b8cb04..b64588fb 100644 --- a/libs/common/src/lib/email/email.processor.ts +++ b/libs/common/src/lib/email/email.processor.ts @@ -1,25 +1,21 @@ import { OnWorkerEvent, Processor, WorkerHost } from "@nestjs/bullmq"; import { EmailService } from "./email.service"; import { Job } from "bullmq"; -import { - ENTITY_MODELS, - EntityModel, - EntityType, - getProjectId -} from "@terramatch-microservices/database/constants/entities"; -import { InternalServerErrorException, NotImplementedException } from "@nestjs/common"; +import { EntityType } from "@terramatch-microservices/database/constants/entities"; +import { NotImplementedException } from "@nestjs/common"; import * as Sentry from "@sentry/node"; import { TMLogger } from "../util/tm-logger"; -import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; -import { ProjectUser, User } from "@terramatch-microservices/database/entities"; -import { isEmpty, map } from "lodash"; -import { Op } from "sequelize"; +import { EntityStatusUpdateEmail } from "./entity-status-update.email"; export type StatusUpdateData = { type: EntityType; id: number; }; +const EMAIL_PROCESSORS = { + statusUpdate: EntityStatusUpdateEmail +}; + /** * Watches for jobs related to sending email */ @@ -33,13 +29,12 @@ export class EmailProcessor extends WorkerHost { async process(job: Job) { const { name, data } = job; - switch (name) { - case "statusUpdate": - return await this.sendStatusUpdateEmail(data as StatusUpdateData); - - default: - throw new NotImplementedException(`Unknown job type: ${name}`); + const processor = EMAIL_PROCESSORS[name as keyof typeof EMAIL_PROCESSORS]; + if (processor == null) { + throw new NotImplementedException(`Unknown job type: ${name}`); } + + await new processor(data).send(this.emailService); } @OnWorkerEvent("failed") @@ -47,54 +42,4 @@ export class EmailProcessor extends WorkerHost { Sentry.captureException(error); this.logger.error(`Worker event failed: ${JSON.stringify(job)}`, error.stack); } - - private async sendStatusUpdateEmail({ type, id }: StatusUpdateData) { - const entityClass = ENTITY_MODELS[type]; - if (entityClass == null) { - throw new InternalServerErrorException(`Entity model class not found for entity type [${type}]`); - } - - const attributes = ["status", "updateRequestStatus"]; - const attributeKeys = Object.keys(entityClass.getAttributes()); - for (const parentId of ["projectId", "siteId", "nurseryId"]) { - if (attributeKeys.includes(parentId)) attributes.push(parentId); - } - const entity = await entityClass.findOne({ where: { id }, attributes }); - if (entity == null) { - throw new InternalServerErrorException(`Entity instance not found for id [type=${type}, id=${id}]`); - } - - const status = - entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION - ? NEEDS_MORE_INFORMATION - : entity.status; - if (![APPROVED, NEEDS_MORE_INFORMATION].includes(status)) return; - - const logExtras = - `[type=${type}, id=${id}, status=${entity.status}, updateRequestStatus=${entity.updateRequestStatus}` as const; - this.logger.log(`Sending status update email ${logExtras}`); - - const to = await this.getEntityUserEmails(entity); - if (isEmpty(to)) { - this.logger.debug(`No addresses found to send entity update to ${logExtras}`); - return; - } - } - - private async getEntityUserEmails(entity: EntityModel) { - const projectId = await getProjectId(entity); - if (projectId == null) { - this.logger.error(`Could not find project ID for entity [type=${entity.constructor.name}, id=${entity.id}]`); - return; - } - - const emailAddresses = map( - await User.findAll({ - where: { id: { [Op.in]: ProjectUser.projectUsersSubquery(projectId) } }, - attributes: ["emailAddress"] - }), - "emailAddress" - ); - return this.emailService.filterEntityEmailRecipients(emailAddresses); - } } diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index 3cec3032..3e66d7d3 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -5,6 +5,14 @@ import * as Mail from "nodemailer/lib/mailer"; import { Dictionary, isEmpty } from "lodash"; import { LocalizationService } from "../localization/localization.service"; import { TemplateService } from "./template.service"; +import { User } from "@terramatch-microservices/database/entities"; + +type I18nEmailOptions = { + i18nReplacements?: Dictionary; + additionalValues?: Dictionary; +}; + +const EMAIL_TEMPLATE = "libs/common/src/lib/views/default-email.hbs"; @Injectable() export class EmailService { @@ -26,26 +34,41 @@ export class EmailService { }); } - filterEntityEmailRecipients(recipients: string[]) { + filterEntityEmailRecipients(recipients: User[]) { const entityDoNotEmailList = this.configService.get("ENTITY_UPDATE_DO_NOT_EMAIL"); if (isEmpty(entityDoNotEmailList)) return recipients; const doNotEmail = entityDoNotEmailList.split(","); - return recipients.filter(address => !doNotEmail.includes(address)); + return recipients.filter(({ emailAddress }) => !doNotEmail.includes(emailAddress)); } - async sendI18nTemplateEmail( - templatePath: string, - to: string | string[], + async renderI18nTemplateEmail( locale: string, i18nKeys: Dictionary, - additionalValues: Dictionary = {} + { i18nReplacements, additionalValues }: I18nEmailOptions = {} ) { if (i18nKeys["subject"] == null) throw new InternalServerErrorException("Email subject is required"); - const { subject, ...translated } = await this.localizationService.translateKeys(i18nKeys, locale); + const { subject, ...translated } = await this.localizationService.translateKeys( + i18nKeys, + locale, + i18nReplacements ?? {} + ); + + const body = this.templateService.render(EMAIL_TEMPLATE, { ...translated, ...(additionalValues ?? {}) }); + return { subject, body }; + } - const renderedBody = this.templateService.render(templatePath, { ...translated, ...additionalValues }); - await this.sendEmail(to, subject, renderedBody); + async sendI18nTemplateEmail( + to: string | string[], + locale: string, + i18nKeys: Dictionary, + { i18nReplacements, additionalValues }: I18nEmailOptions = {} + ) { + const { subject, body } = await this.renderI18nTemplateEmail(locale, i18nKeys, { + i18nReplacements, + additionalValues + }); + await this.sendEmail(to, subject, body); } async sendEmail(to: string | string[], subject: string, body: string) { diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts new file mode 100644 index 00000000..4336c58e --- /dev/null +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -0,0 +1,159 @@ +import { EmailProcessor } from "./email-processor"; +import { EmailService } from "./email.service"; +import { StatusUpdateData } from "./email.processor"; +import { + ENTITY_MODELS, + EntityModel, + EntityType, + getProjectId, + getViewLinkPath, + isReport, + ReportModel +} from "@terramatch-microservices/database/constants/entities"; +import { Dictionary, groupBy, isEmpty } from "lodash"; +import { + ProjectReport, + ProjectUser, + SiteReport, + UpdateRequest, + User +} from "@terramatch-microservices/database/entities"; +import { Op } from "sequelize"; +import { TMLogger } from "../util/tm-logger"; +import { InternalServerErrorException } from "@nestjs/common"; +import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; + +export class EntityStatusUpdateEmail extends EmailProcessor { + private readonly logger = new TMLogger(EntityStatusUpdateEmail.name); + + private readonly type: EntityType; + private readonly id: number; + + constructor({ type, id }: StatusUpdateData) { + super(); + this.type = type; + this.id = id; + } + + async send(emailService: EmailService) { + const entity = await this.getEntity(); + const status = + entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION + ? NEEDS_MORE_INFORMATION + : entity.status; + if (![APPROVED, NEEDS_MORE_INFORMATION].includes(status)) return; + + const logExtras = `[type=${this.type}, id=${this.id}, status=${status}]` as const; + this.logger.log(`Sending status update email ${logExtras}`); + + const to = emailService.filterEntityEmailRecipients(await this.getEntityUsers(entity)); + if (isEmpty(to)) { + this.logger.debug(`No addresses found to send entity update to ${logExtras}`); + return; + } + + const i18nKeys: Dictionary = { + subject: + status === APPROVED + ? "entity-status-change.subject-approved" + : "entity-status-change.subject-needs-more-information", + cta: "entity-status-change.cta" + }; + i18nKeys["title"] = i18nKeys["subject"]; + + if (isReport(entity)) { + i18nKeys["body"] = + status === APPROVED + ? "entity-status-change.body-report-approved" + : "entity-status-change.body-report-needs-more-information"; + } else { + i18nKeys["body"] = + status === APPROVED + ? "entity-status-change.body-entity-approved" + : "entity-status-change.body-entity-needs-more-information"; + } + + const entityTypeName = isReport(entity) ? "Report" : entity.constructor.name; + const i18nReplacements: Dictionary = { + "{entityTypeName}": entityTypeName, + "{lowerEntityTypeName}": entityTypeName.toLowerCase(), + "{entityName}": (isReport(entity) ? "" : entity.name) ?? "", + "{feedback}": (await this.getFeedback(entity)) ?? "(No feedback)" + }; + if (isReport(entity)) i18nReplacements["{parentEntityName"] = this.getParentName(entity) ?? ""; + + const additionalValues = { + link: getViewLinkPath(entity), + transactional: "transactional" + }; + + // Group the users by locale and then send the email to each locale group. + await Promise.all( + Object.entries(groupBy(to, "locale")).map(([locale, users]) => + emailService.sendI18nTemplateEmail( + users.map(({ emailAddress }) => emailAddress), + locale, + i18nKeys, + { + i18nReplacements, + additionalValues + } + ) + ) + ); + } + + private getParentName(report: ReportModel) { + if (report instanceof ProjectReport) return report.projectName; + if (report instanceof SiteReport) return report.siteName; + return report.nurseryName; + } + + private async getFeedback(entity: EntityModel) { + if (![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.updateRequestStatus ?? "")) { + return entity.feedback; + } + + const updateRequest = await UpdateRequest.entity(entity).findOne({ + order: [["updatedAt", "DESC"]], + attributes: ["feedback"] + }); + return updateRequest?.feedback ?? entity.feedback; + } + + private async getEntity() { + const entityClass = ENTITY_MODELS[this.type]; + if (entityClass == null) { + throw new InternalServerErrorException(`Entity model class not found for entity type [${this.type}]`); + } + + const attributes = ["id", "uuid", "status", "updateRequestStatus", "name", "feedback"]; + const include = []; + const attributeKeys = Object.keys(entityClass.getAttributes()); + for (const parentId of ["projectId", "siteId", "nurseryId"]) { + if (attributeKeys.includes(parentId)) { + attributes.push(parentId); + include.push({ association: parentId.substring(0, parentId.length - 2), attributes: ["name"] }); + } + } + const entity = await entityClass.findOne({ where: { id: this.id }, attributes, include }); + if (entity == null) { + throw new InternalServerErrorException(`Entity instance not found for id [type=${this.type}, id=${this.id}]`); + } + + return entity; + } + + private async getEntityUsers(entity: EntityModel) { + const projectId = await getProjectId(entity); + if (projectId == null) { + this.logger.error(`Could not find project ID for entity [type=${entity.constructor.name}, id=${entity.id}]`); + return []; + } + + return await User.findAll({ + where: { id: { [Op.in]: ProjectUser.projectUsersSubquery(projectId) } }, + attributes: ["emailAddress", "locale"] + }); + } +} diff --git a/libs/common/src/lib/email/template.service.ts b/libs/common/src/lib/email/template.service.ts index 5f27b7c1..1ed15948 100644 --- a/libs/common/src/lib/email/template.service.ts +++ b/libs/common/src/lib/email/template.service.ts @@ -4,6 +4,7 @@ import * as Handlebars from "handlebars"; import * as fs from "fs"; import * as path from "path"; import { Dictionary } from "factory-girl-ts"; +import { isString } from "lodash"; @Injectable() export class TemplateService { @@ -14,13 +15,14 @@ export class TemplateService { private getCompiledTemplate(template: string) { if (this.templates[template] != null) return this.templates[template]; - const templatePath = path.join(__dirname, "..", template); + // reference the template path from the NX root. + const templatePath = path.join(__dirname, "../../..", template); const templateSource = fs.readFileSync(templatePath, "utf-8"); return (this.templates[template] = Handlebars.compile(templateSource)); } - render(templatePath: string, data: Dictionary): string { - const params = { + render(templatePath: string, data: Dictionary): string { + const params: Dictionary = { backendUrl: this.configService.get("EMAIL_IMAGE_BASE_URL"), banner: null, invite: null, @@ -29,6 +31,12 @@ export class TemplateService { year: new Date().getFullYear(), ...data }; + + if (isString(params["link"]) && params["link"].substring(0, 4).toLowerCase() !== "http") { + // If we're given a link that's pointing to a raw path, assume it should be prepended with + // the configured app FE base URL. + params["link"] = `${this.configService.get("APP_FRONT_END")}${params["link"]}`; + } return this.getCompiledTemplate(templatePath)(params); } } diff --git a/libs/common/src/lib/localization/localization.service.spec.ts b/libs/common/src/lib/localization/localization.service.spec.ts index d5ca4ca7..4f6a45c0 100644 --- a/libs/common/src/lib/localization/localization.service.spec.ts +++ b/libs/common/src/lib/localization/localization.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from "@nestjs/testing"; import { LocalizationService } from "./localization.service"; -import { i18nItem, i18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; +import { I18nItem, I18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; import { ConfigService } from "@nestjs/config"; import { tx } from "@transifex/native"; import { createMock } from "@golevelup/ts-jest"; @@ -41,25 +41,25 @@ describe("LocalizationService", () => { }); it("should return the translated text when a matching i18n item and translation record are found", async () => { - const i18Record = new i18nTranslation(); + const i18Record = new I18nTranslation(); i18Record.shortValue = "contenido traducido"; - const i18Item = new i18nItem(); - jest.spyOn(i18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); - jest.spyOn(i18nTranslation, "findOne").mockImplementation(() => Promise.resolve(i18Record)); + const i18Item = new I18nItem(); + jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); + jest.spyOn(I18nTranslation, "findOne").mockImplementation(() => Promise.resolve(i18Record)); const result = await service.translate("content translate", "es-MX"); expect(result).toBe(i18Record.shortValue); }); it("should return the original text when no matching i18n item is found", async () => { - jest.spyOn(i18nItem, "findOne").mockImplementation(() => Promise.resolve(null)); + jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(null)); const result = await service.translate("content translate", "es-MX"); expect(result).toBe("content translate"); }); it("should return the original text when no translation record is found for the given locale", async () => { - const i18Item = new i18nItem(); - jest.spyOn(i18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); - jest.spyOn(i18nTranslation, "findOne").mockImplementation(() => Promise.resolve(null)); + const i18Item = new I18nItem(); + jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); + jest.spyOn(I18nTranslation, "findOne").mockImplementation(() => Promise.resolve(null)); const result = await service.translate("content translate", "es-MX"); expect(result).toBe("content translate"); }); diff --git a/libs/common/src/lib/localization/localization.service.ts b/libs/common/src/lib/localization/localization.service.ts index 8e026c34..d2d554c2 100644 --- a/libs/common/src/lib/localization/localization.service.ts +++ b/libs/common/src/lib/localization/localization.service.ts @@ -1,6 +1,5 @@ import { Injectable, NotFoundException } from "@nestjs/common"; -import { i18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; -import { i18nItem } from "@terramatch-microservices/database/entities/i18n-item.entity"; +import { I18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; import { Op } from "sequelize"; import { ConfigService } from "@nestjs/config"; import { normalizeLocale, tx } from "@transifex/native"; @@ -14,48 +13,33 @@ export class LocalizationService { }); } - async getLocalizationKeys(keys: string[]): Promise { - return await LocalizationKey.findAll({ where: { key: { [Op.in]: keys } } }); - } - - private async getItemI18n(value: string): Promise { - return await i18nItem.findOne({ - where: { - [Op.or]: [{ shortValue: value }, { longValue: value }] - } - }); - } - - private async getTranslateItem(itemId: number, locale: string): Promise { - return await i18nTranslation.findOne({ where: { i18nItemId: itemId, language: locale } }); - } - - async translateKeys(keyMap: Dictionary, locale: string) { + async translateKeys(keyMap: Dictionary, locale: string, replacements: Dictionary = {}) { const keyStrings = Object.values(keyMap); - const keys = (await this.getLocalizationKeys(keyStrings)) ?? []; - - const result: Dictionary = {}; - await Promise.all( - Object.entries(keyMap).map(async ([prop, key]) => { - const value = keys.find(record => record.key === key)?.value; - if (value == null) throw new NotFoundException(`Localization key not found [${key}]`); - - const translated = await this.translate(value, locale); - if (translated != null) result[prop] = translated; - }) - ); - - return result; + const keys = await this.getLocalizationKeys(keyStrings); + const i18nItemIds = keys.map(({ valueId }) => valueId).filter(id => id != null); + const translations = + i18nItemIds.length === 0 + ? [] + : await I18nTranslation.findAll({ + where: { i18nItemId: { [Op.in]: i18nItemIds }, language: locale } + }); + + return Object.entries(keyMap).reduce((result, [prop, key]) => { + const { value, valueId } = keys.find(record => record.key === key) ?? {}; + if (value == null) throw new NotFoundException(`Localization key not found [${key}]`); + + const translation = translations.find(({ i18nItemId }) => i18nItemId === valueId); + const translated = Object.entries(replacements).reduce( + (translated, [replacementKey, replacementValue]) => translated.replaceAll(replacementKey, replacementValue), + translation?.longValue ?? translation?.shortValue ?? value + ); + + return { ...result, [prop]: translated }; + }, {} as Dictionary); } - async translate(content: string, locale: string): Promise { - const itemResult = await this.getItemI18n(content); - if (itemResult == null) return content; - - const translationResult = await this.getTranslateItem(itemResult.id, locale); - if (translationResult == null) return content; - - return translationResult.shortValue ?? translationResult.longValue; + private async getLocalizationKeys(keys: string[]): Promise { + return (await LocalizationKey.findAll({ where: { key: { [Op.in]: keys } } })) ?? []; } /** @@ -66,9 +50,7 @@ export class LocalizationService { */ async localizeText(text: string, locale: string): Promise { // Set the locale for the SDK - const txLocale = normalizeLocale(locale); - await tx.setCurrentLocale(txLocale); // Translate the text diff --git a/libs/common/src/lib/util/bootstrap-repl.ts b/libs/common/src/lib/util/bootstrap-repl.ts index 272f5df8..7679e842 100644 --- a/libs/common/src/lib/util/bootstrap-repl.ts +++ b/libs/common/src/lib/util/bootstrap-repl.ts @@ -7,6 +7,7 @@ import { join } from "node:path"; import { existsSync, mkdirSync } from "node:fs"; import { Dictionary } from "lodash"; import { buildJsonApi } from "./json-api-builder"; +import { Op } from "sequelize"; const logger = new TMLogger("REPL"); @@ -22,7 +23,7 @@ export async function bootstrapRepl(serviceName: string, module: Type | DynamicM const replServer = await repl(module); // By default, we make lodash, the JSON API Builder, and the Sequelize models available - context = { lodash, buildJsonApi, ...replServer.context["get"](Sequelize).models, ...context }; + context = { lodash, buildJsonApi, ...replServer.context["get"](Sequelize).models, Op, ...context }; for (const [name, model] of Object.entries(context as Dictionary)) { // For in REPL auto-complete functionality diff --git a/apps/user-service/src/views/default-email.hbs b/libs/common/src/lib/views/default-email.hbs similarity index 100% rename from apps/user-service/src/views/default-email.hbs rename to libs/common/src/lib/views/default-email.hbs diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 31acc9a3..156b32c2 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -1,6 +1,7 @@ import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport } from "../entities"; import { ModelCtor } from "sequelize-typescript"; import { ModelStatic } from "sequelize"; +import { kebabCase } from "lodash"; export const REPORT_TYPES = ["projectReports", "siteReports", "nurseryReports"] as const; export type ReportType = (typeof REPORT_TYPES)[number]; @@ -25,6 +26,9 @@ export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { nurseries: Nursery }; +export const isReport = (entity: EntityModel): entity is ReportModel => + Object.values(REPORT_MODELS).find(model => entity instanceof model) != null; + /** * Get the project ID associated with the given entity, which may be any one of EntityModels defined in this file. * @@ -40,3 +44,8 @@ export async function getProjectId(entity: EntityModel) { const parentId = entity instanceof SiteReport ? entity.siteId : entity.nurseryId; return (await parentClass.findOne({ where: { id: parentId }, attributes: ["projectId"] }))?.projectId; } + +export function getViewLinkPath(entity: EntityModel) { + const prefix = isReport(entity) ? "/reports/" : "/"; + return `${prefix}${kebabCase(entity.constructor.name)}/${entity.uuid}`; +} diff --git a/libs/database/src/lib/entities/i18n-item.entity.ts b/libs/database/src/lib/entities/i18n-item.entity.ts index 9e2444f6..65784e3d 100644 --- a/libs/database/src/lib/entities/i18n-item.entity.ts +++ b/libs/database/src/lib/entities/i18n-item.entity.ts @@ -2,7 +2,7 @@ import { AllowNull, AutoIncrement, Column, Model, PrimaryKey, Table } from "sequ import { BIGINT, STRING, TEXT } from "sequelize"; @Table({ tableName: "i18n_items", underscored: true }) -export class i18nItem extends Model { +export class I18nItem extends Model { @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) diff --git a/libs/database/src/lib/entities/i18n-translation.entity.ts b/libs/database/src/lib/entities/i18n-translation.entity.ts index ae4a76c9..4df85647 100644 --- a/libs/database/src/lib/entities/i18n-translation.entity.ts +++ b/libs/database/src/lib/entities/i18n-translation.entity.ts @@ -2,7 +2,7 @@ import { AllowNull, AutoIncrement, Column, Model, PrimaryKey, Table, Unique } fr import { BIGINT, INTEGER, STRING, TEXT } from "sequelize"; @Table({ tableName: "i18n_translations", underscored: true }) -export class i18nTranslation extends Model { +export class I18nTranslation extends Model { @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 337f404e..e25e329f 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -36,6 +36,7 @@ export * from "./site-polygon.entity"; export * from "./site-report.entity"; export * from "./tree-species.entity"; export * from "./tree-species-research.entity"; +export * from "./update-request.entity"; export * from "./user.entity"; export * from "./verification.entity"; export * from "./task.entity"; diff --git a/libs/database/src/lib/entities/localization-keys.entity.ts b/libs/database/src/lib/entities/localization-keys.entity.ts index 2e0a1895..d8c4410b 100644 --- a/libs/database/src/lib/entities/localization-keys.entity.ts +++ b/libs/database/src/lib/entities/localization-keys.entity.ts @@ -1,5 +1,6 @@ -import { AllowNull, AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { AllowNull, AutoIncrement, BelongsTo, Column, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, INTEGER, STRING, TEXT } from "sequelize"; +import { I18nItem } from "./i18n-item.entity"; @Table({ tableName: "localization_keys", underscored: true }) export class LocalizationKey extends Model { @@ -17,4 +18,7 @@ export class LocalizationKey extends Model { @AllowNull @Column(INTEGER({ length: 11 })) valueId: number; + + @BelongsTo(() => I18nItem, { foreignKey: "value_id" }) + i18nItem: I18nItem; } diff --git a/libs/database/src/lib/entities/update-request.entity.ts b/libs/database/src/lib/entities/update-request.entity.ts new file mode 100644 index 00000000..b8fd4624 --- /dev/null +++ b/libs/database/src/lib/entities/update-request.entity.ts @@ -0,0 +1,84 @@ +import { + AllowNull, + AutoIncrement, + Column, + ForeignKey, + Model, + PrimaryKey, + Scopes, + Table, + Unique +} from "sequelize-typescript"; +import { BIGINT, STRING, TEXT, UUID } from "sequelize"; +import { Organisation } from "./organisation.entity"; +import { Project } from "./project.entity"; +import { User } from "./user.entity"; +import { FrameworkKey } from "../constants/framework"; +import { JsonColumn } from "../decorators/json-column.decorator"; +import { EntityClass, EntityModel } from "../constants/entities"; +import { chainScope } from "../util/chain-scope"; + +@Scopes(() => ({ + entity: (entity: T) => ({ + where: { + updateRequestableType: (entity.constructor as EntityClass).LARAVEL_TYPE, + updateRequestableId: entity.id + } + }) +})) +@Table({ tableName: "v2_update_requests", underscored: true, paranoid: true }) +export class UpdateRequest extends Model { + static entity(entity: T) { + return chainScope(this, "entity", entity) as typeof UpdateRequest; + } + + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique + @Column(UUID) + uuid: string; + + @AllowNull + @ForeignKey(() => Organisation) + @Column(BIGINT.UNSIGNED) + organisationId: number | null; + + @AllowNull + @ForeignKey(() => Project) + @Column(BIGINT.UNSIGNED) + projectId: number | null; + + @AllowNull + @ForeignKey(() => User) + @Column(BIGINT.UNSIGNED) + createdById: number | null; + + @AllowNull + @Column(STRING) + frameworkKey: FrameworkKey | null; + + @AllowNull + @Column(STRING) + status: string; + + @AllowNull + @JsonColumn() + content: object | null; + + @AllowNull + @Column(TEXT) + feedback: string | null; + + @AllowNull + @JsonColumn() + feedbackFields: string[] | null; + + @Column({ type: STRING, field: "updaterequestable_type" }) + updateRequestableType: string; + + @Column({ type: BIGINT.UNSIGNED, field: "updaterequestable_id" }) + updateRequestableId: number; +} From 075de902214d5cc6f420e0cee10cde2158254302 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 11:56:28 -0700 Subject: [PATCH 09/98] [TM-1861] Use a single central health module for all services. --- README.md | 3 +- apps/entity-service/src/app.module.ts | 2 +- .../src/health/health.controller.ts | 23 --------- .../src/health/health.module.ts | 9 ---- apps/job-service/src/app.module.ts | 2 +- .../src/health/health.controller.ts | 23 --------- apps/job-service/src/health/health.module.ts | 9 ---- apps/research-service/src/app.module.ts | 2 +- .../src/health/health.controller.ts | 23 --------- .../src/health/health.module.ts | 9 ---- .../src/airtable/airtable.module.ts | 13 ++--- .../src/airtable/queue-health.indicator.ts | 33 ------------ .../src/app.module.ts | 18 ++++--- .../src/health/health.module.ts | 10 ---- apps/user-service/src/app.module.ts | 2 +- .../src/health/health.controller.ts | 23 --------- apps/user-service/src/health/health.module.ts | 9 ---- libs/common/src/lib/common.module.ts | 4 +- .../src/lib}/health/health.controller.ts | 10 ++-- libs/common/src/lib/health/health.module.ts | 29 +++++++++++ .../src/lib/health/queue-health.indicator.ts | 51 +++++++++++++++++++ libs/common/src/lib/util/bootstrap-repl.ts | 9 +++- 22 files changed, 116 insertions(+), 200 deletions(-) delete mode 100644 apps/entity-service/src/health/health.controller.ts delete mode 100644 apps/entity-service/src/health/health.module.ts delete mode 100644 apps/job-service/src/health/health.controller.ts delete mode 100644 apps/job-service/src/health/health.module.ts delete mode 100644 apps/research-service/src/health/health.controller.ts delete mode 100644 apps/research-service/src/health/health.module.ts delete mode 100644 apps/unified-database-service/src/airtable/queue-health.indicator.ts delete mode 100644 apps/unified-database-service/src/health/health.module.ts delete mode 100644 apps/user-service/src/health/health.controller.ts delete mode 100644 apps/user-service/src/health/health.module.ts rename {apps/unified-database-service/src => libs/common/src/lib}/health/health.controller.ts (66%) create mode 100644 libs/common/src/lib/health/health.module.ts create mode 100644 libs/common/src/lib/health/queue-health.indicator.ts diff --git a/README.md b/README.md index 5990b35e..f6e5d4b8 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ and main branches. - In the root directory: `nx g @nx/nest:app apps/foo-service` - Set up the new `main.ts` similarly to existing services. - - Make sure swagger docs and the `/health` endpoint are implemented + - Make sure swagger docs are implemented - Pick a default local port that is unique from other services - Make sure the top of `main.ts` has these two lines: ``` @@ -95,6 +95,7 @@ and main branches. import "../../../instrument-sentry"; ``` - Add the `SentryModule` and `SentryGlobalFilter` to your main `app.module.ts`. See an existing service for an example. + - Add the `HealthModule` to your main `app.module.ts`. You will likely need `CommonModule` as well. - Set up REPL access: - Copy `repl.ts` from an existing service (and modify to specify the new service's name) - Add the `build-repl` target to `project.json`, which an empty definition. diff --git a/apps/entity-service/src/app.module.ts b/apps/entity-service/src/app.module.ts index cbdd5e8a..006c28ac 100644 --- a/apps/entity-service/src/app.module.ts +++ b/apps/entity-service/src/app.module.ts @@ -1,6 +1,5 @@ import { Module } from "@nestjs/common"; import { CommonModule } from "@terramatch-microservices/common"; -import { HealthModule } from "./health/health.module"; import { TreesController } from "./trees/trees.controller"; import { TreeService } from "./trees/tree.service"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; @@ -8,6 +7,7 @@ import { APP_FILTER } from "@nestjs/core"; import { EntitiesService } from "./entities/entities.service"; import { EntitiesController } from "./entities/entities.controller"; import { EntityAssociationsController } from "./entities/entity-associations.controller"; +import { HealthModule } from "@terramatch-microservices/common/health/health.module"; @Module({ imports: [SentryModule.forRoot(), CommonModule, HealthModule], diff --git a/apps/entity-service/src/health/health.controller.ts b/apps/entity-service/src/health/health.controller.ts deleted file mode 100644 index 950e84da..00000000 --- a/apps/entity-service/src/health/health.controller.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Controller, Get } from "@nestjs/common"; -import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; -import { ApiExcludeController } from "@nestjs/swagger"; -import { User } from "@terramatch-microservices/database/entities"; - -@Controller("health") -@ApiExcludeController() -export class HealthController { - constructor(private readonly health: HealthCheckService, private readonly db: SequelizeHealthIndicator) {} - - @Get() - @HealthCheck() - @NoBearerAuth - async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); - try { - return this.health.check([() => this.db.pingCheck("database", { connection })]); - } finally { - User.sequelize.connectionManager.releaseConnection(connection); - } - } -} diff --git a/apps/entity-service/src/health/health.module.ts b/apps/entity-service/src/health/health.module.ts deleted file mode 100644 index 0f68be79..00000000 --- a/apps/entity-service/src/health/health.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from "@nestjs/common"; -import { TerminusModule } from "@nestjs/terminus"; -import { HealthController } from "./health.controller"; - -@Module({ - imports: [TerminusModule], - controllers: [HealthController] -}) -export class HealthModule {} diff --git a/apps/job-service/src/app.module.ts b/apps/job-service/src/app.module.ts index eb0a2005..872883d6 100644 --- a/apps/job-service/src/app.module.ts +++ b/apps/job-service/src/app.module.ts @@ -1,9 +1,9 @@ import { Module } from "@nestjs/common"; import { CommonModule } from "@terramatch-microservices/common"; import { DelayedJobsController } from "./jobs/delayed-jobs.controller"; -import { HealthModule } from "./health/health.module"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; import { APP_FILTER } from "@nestjs/core"; +import { HealthModule } from "@terramatch-microservices/common/health/health.module"; @Module({ imports: [SentryModule.forRoot(), CommonModule, HealthModule], diff --git a/apps/job-service/src/health/health.controller.ts b/apps/job-service/src/health/health.controller.ts deleted file mode 100644 index 950e84da..00000000 --- a/apps/job-service/src/health/health.controller.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Controller, Get } from "@nestjs/common"; -import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; -import { ApiExcludeController } from "@nestjs/swagger"; -import { User } from "@terramatch-microservices/database/entities"; - -@Controller("health") -@ApiExcludeController() -export class HealthController { - constructor(private readonly health: HealthCheckService, private readonly db: SequelizeHealthIndicator) {} - - @Get() - @HealthCheck() - @NoBearerAuth - async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); - try { - return this.health.check([() => this.db.pingCheck("database", { connection })]); - } finally { - User.sequelize.connectionManager.releaseConnection(connection); - } - } -} diff --git a/apps/job-service/src/health/health.module.ts b/apps/job-service/src/health/health.module.ts deleted file mode 100644 index 0f68be79..00000000 --- a/apps/job-service/src/health/health.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from "@nestjs/common"; -import { TerminusModule } from "@nestjs/terminus"; -import { HealthController } from "./health.controller"; - -@Module({ - imports: [TerminusModule], - controllers: [HealthController] -}) -export class HealthModule {} diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts index bb7908b9..a6b1f8c0 100644 --- a/apps/research-service/src/app.module.ts +++ b/apps/research-service/src/app.module.ts @@ -1,10 +1,10 @@ import { Module } from "@nestjs/common"; import { CommonModule } from "@terramatch-microservices/common"; -import { HealthModule } from "./health/health.module"; import { SitePolygonsController } from "./site-polygons/site-polygons.controller"; import { SitePolygonsService } from "./site-polygons/site-polygons.service"; import { APP_FILTER } from "@nestjs/core"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; +import { HealthModule } from "@terramatch-microservices/common/health/health.module"; @Module({ imports: [SentryModule.forRoot(), CommonModule, HealthModule], diff --git a/apps/research-service/src/health/health.controller.ts b/apps/research-service/src/health/health.controller.ts deleted file mode 100644 index 950e84da..00000000 --- a/apps/research-service/src/health/health.controller.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Controller, Get } from "@nestjs/common"; -import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; -import { ApiExcludeController } from "@nestjs/swagger"; -import { User } from "@terramatch-microservices/database/entities"; - -@Controller("health") -@ApiExcludeController() -export class HealthController { - constructor(private readonly health: HealthCheckService, private readonly db: SequelizeHealthIndicator) {} - - @Get() - @HealthCheck() - @NoBearerAuth - async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); - try { - return this.health.check([() => this.db.pingCheck("database", { connection })]); - } finally { - User.sequelize.connectionManager.releaseConnection(connection); - } - } -} diff --git a/apps/research-service/src/health/health.module.ts b/apps/research-service/src/health/health.module.ts deleted file mode 100644 index 0f68be79..00000000 --- a/apps/research-service/src/health/health.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from "@nestjs/common"; -import { TerminusModule } from "@nestjs/terminus"; -import { HealthController } from "./health.controller"; - -@Module({ - imports: [TerminusModule], - controllers: [HealthController] -}) -export class HealthModule {} diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts index f52253c7..87e7220e 100644 --- a/apps/unified-database-service/src/airtable/airtable.module.ts +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -4,17 +4,10 @@ import { BullModule } from "@nestjs/bullmq"; import { Module } from "@nestjs/common"; import { AirtableService } from "./airtable.service"; import { AirtableProcessor } from "./airtable.processor"; -import { QueueHealthIndicator } from "./queue-health.indicator"; -import { TerminusModule } from "@nestjs/terminus"; @Module({ - imports: [ - CommonModule, - ConfigModule.forRoot({ isGlobal: true }), - BullModule.registerQueue({ name: "airtable" }), - TerminusModule - ], - providers: [AirtableService, AirtableProcessor, QueueHealthIndicator], - exports: [AirtableService, QueueHealthIndicator] + imports: [CommonModule, ConfigModule.forRoot({ isGlobal: true }), BullModule.registerQueue({ name: "airtable" })], + providers: [AirtableService, AirtableProcessor], + exports: [AirtableService] }) export class AirtableModule {} diff --git a/apps/unified-database-service/src/airtable/queue-health.indicator.ts b/apps/unified-database-service/src/airtable/queue-health.indicator.ts deleted file mode 100644 index a9a1fd82..00000000 --- a/apps/unified-database-service/src/airtable/queue-health.indicator.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { InjectQueue } from "@nestjs/bullmq"; -import { Injectable } from "@nestjs/common"; -import { HealthIndicatorService } from "@nestjs/terminus"; -import { Queue } from "bullmq"; - -// TODO Add to common for all services to use. -@Injectable() -export class QueueHealthIndicator { - constructor( - @InjectQueue("airtable") private readonly airtableQueue: Queue, - private readonly healthIndicatorService: HealthIndicatorService - ) {} - - public async isHealthy() { - const indicator = this.healthIndicatorService.check("redis-queue:airtable"); - - const isHealthy = (await (await this.airtableQueue.client).ping()) === "PONG"; - if (!isHealthy) { - return indicator.down({ message: "Redis connection for Airtable queue unavailable" }); - } - - try { - const data = { - waitingJobs: await this.airtableQueue.getWaitingCount(), - totalJobs: await this.airtableQueue.count() - }; - - return indicator.up(data); - } catch (error) { - return indicator.down({ message: "Error fetching Airtable queue stats", error }); - } - } -} diff --git a/apps/unified-database-service/src/app.module.ts b/apps/unified-database-service/src/app.module.ts index 2d0742a9..c3b76597 100644 --- a/apps/unified-database-service/src/app.module.ts +++ b/apps/unified-database-service/src/app.module.ts @@ -1,19 +1,21 @@ import { Module } from "@nestjs/common"; -import { HealthModule } from "./health/health.module"; import { WebhookController } from "./webhook/webhook.controller"; import { AirtableModule } from "./airtable/airtable.module"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; import { APP_FILTER } from "@nestjs/core"; import { ScheduleModule } from "@nestjs/schedule"; +import { CommonModule } from "@terramatch-microservices/common"; +import { HealthModule } from "@terramatch-microservices/common/health/health.module"; @Module({ - imports: [SentryModule.forRoot(), ScheduleModule.forRoot(), HealthModule, AirtableModule], + imports: [ + SentryModule.forRoot(), + ScheduleModule.forRoot(), + CommonModule, + HealthModule.configure({ additionalQueues: ["airtable"] }), + AirtableModule + ], controllers: [WebhookController], - providers: [ - { - provide: APP_FILTER, - useClass: SentryGlobalFilter - } - ] + providers: [{ provide: APP_FILTER, useClass: SentryGlobalFilter }] }) export class AppModule {} diff --git a/apps/unified-database-service/src/health/health.module.ts b/apps/unified-database-service/src/health/health.module.ts deleted file mode 100644 index 1b7de1b3..00000000 --- a/apps/unified-database-service/src/health/health.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from "@nestjs/common"; -import { TerminusModule } from "@nestjs/terminus"; -import { HealthController } from "./health.controller"; -import { AirtableModule } from "../airtable/airtable.module"; - -@Module({ - imports: [TerminusModule, AirtableModule], - controllers: [HealthController] -}) -export class HealthModule {} diff --git a/apps/user-service/src/app.module.ts b/apps/user-service/src/app.module.ts index 321f69ab..764afdb2 100644 --- a/apps/user-service/src/app.module.ts +++ b/apps/user-service/src/app.module.ts @@ -3,7 +3,6 @@ import { LoginController } from "./auth/login.controller"; import { AuthService } from "./auth/auth.service"; import { UsersController } from "./users/users.controller"; import { CommonModule } from "@terramatch-microservices/common"; -import { HealthModule } from "./health/health.module"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; import { APP_FILTER } from "@nestjs/core"; import { ResetPasswordController } from "./auth/reset-password.controller"; @@ -11,6 +10,7 @@ import { ResetPasswordService } from "./auth/reset-password.service"; import { VerificationUserController } from "./auth/verification-user.controller"; import { VerificationUserService } from "./auth/verification-user.service"; import { UserCreationService } from "./users/user-creation.service"; +import { HealthModule } from "@terramatch-microservices/common/health/health.module"; @Module({ imports: [SentryModule.forRoot(), CommonModule, HealthModule], diff --git a/apps/user-service/src/health/health.controller.ts b/apps/user-service/src/health/health.controller.ts deleted file mode 100644 index 950e84da..00000000 --- a/apps/user-service/src/health/health.controller.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Controller, Get } from "@nestjs/common"; -import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; -import { ApiExcludeController } from "@nestjs/swagger"; -import { User } from "@terramatch-microservices/database/entities"; - -@Controller("health") -@ApiExcludeController() -export class HealthController { - constructor(private readonly health: HealthCheckService, private readonly db: SequelizeHealthIndicator) {} - - @Get() - @HealthCheck() - @NoBearerAuth - async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); - try { - return this.health.check([() => this.db.pingCheck("database", { connection })]); - } finally { - User.sequelize.connectionManager.releaseConnection(connection); - } - } -} diff --git a/apps/user-service/src/health/health.module.ts b/apps/user-service/src/health/health.module.ts deleted file mode 100644 index 0f68be79..00000000 --- a/apps/user-service/src/health/health.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from "@nestjs/common"; -import { TerminusModule } from "@nestjs/terminus"; -import { HealthController } from "./health.controller"; - -@Module({ - imports: [TerminusModule], - controllers: [HealthController] -}) -export class HealthModule {} diff --git a/libs/common/src/lib/common.module.ts b/libs/common/src/lib/common.module.ts index 89a33079..b3f63ca1 100644 --- a/libs/common/src/lib/common.module.ts +++ b/libs/common/src/lib/common.module.ts @@ -16,6 +16,8 @@ import { EventService } from "./events/event.service"; import { BullModule } from "@nestjs/bullmq"; import { EmailProcessor } from "./email/email.processor"; +export const QUEUES = ["email"]; + @Module({ imports: [ RequestContextModule, @@ -48,7 +50,7 @@ import { EmailProcessor } from "./email/email.processor"; } }) }), - BullModule.registerQueue({ name: "email" }) + ...QUEUES.map(name => BullModule.registerQueue({ name })) ], providers: [ PolicyService, diff --git a/apps/unified-database-service/src/health/health.controller.ts b/libs/common/src/lib/health/health.controller.ts similarity index 66% rename from apps/unified-database-service/src/health/health.controller.ts rename to libs/common/src/lib/health/health.controller.ts index 58625d2f..da838c6e 100644 --- a/apps/unified-database-service/src/health/health.controller.ts +++ b/libs/common/src/lib/health/health.controller.ts @@ -1,9 +1,9 @@ import { Controller, Get } from "@nestjs/common"; import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; import { ApiExcludeController } from "@nestjs/swagger"; import { User } from "@terramatch-microservices/database/entities"; -import { QueueHealthIndicator } from "../airtable/queue-health.indicator"; +import { QueueHealthIndicator } from "./queue-health.indicator"; +import { NoBearerAuth } from "../guards"; @Controller("health") @ApiExcludeController() @@ -18,11 +18,13 @@ export class HealthController { @HealthCheck() @NoBearerAuth async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const sequelize = User.sequelize!; + const connection = await sequelize.connectionManager.getConnection({ type: "read" }); try { return this.health.check([() => this.db.pingCheck("database", { connection }), () => this.queue.isHealthy()]); } finally { - User.sequelize.connectionManager.releaseConnection(connection); + sequelize.connectionManager.releaseConnection(connection); } } } diff --git a/libs/common/src/lib/health/health.module.ts b/libs/common/src/lib/health/health.module.ts new file mode 100644 index 00000000..1641f668 --- /dev/null +++ b/libs/common/src/lib/health/health.module.ts @@ -0,0 +1,29 @@ +import { DynamicModule, Module } from "@nestjs/common"; +import { TerminusModule } from "@nestjs/terminus"; +import { HealthController } from "./health.controller"; +import { QueueHealthIndicator } from "./queue-health.indicator"; +import { QUEUES } from "../common.module"; + +export const QUEUE_LIST = Symbol("QUEUE_LIST"); + +type HealthModuleOptions = { + // Queues to add to what is defined in the common module. + additionalQueues?: string[]; +}; + +@Module({ + imports: [TerminusModule], + providers: [QueueHealthIndicator, { provide: QUEUE_LIST, useValue: QUEUES }], + controllers: [HealthController] +}) +export class HealthModule { + static configure(options: HealthModuleOptions): DynamicModule { + const queueList = [...QUEUES, ...(options.additionalQueues ?? [])]; + return { + module: HealthModule, + imports: [TerminusModule], + providers: [QueueHealthIndicator, { provide: QUEUE_LIST, useValue: queueList }], + controllers: [HealthController] + }; + } +} diff --git a/libs/common/src/lib/health/queue-health.indicator.ts b/libs/common/src/lib/health/queue-health.indicator.ts new file mode 100644 index 00000000..fe781d39 --- /dev/null +++ b/libs/common/src/lib/health/queue-health.indicator.ts @@ -0,0 +1,51 @@ +import { getQueueToken } from "@nestjs/bullmq"; +import { Injectable, OnModuleInit } from "@nestjs/common"; +import { HealthIndicatorService } from "@nestjs/terminus"; +import { Queue } from "bullmq"; +import { ModuleRef } from "@nestjs/core"; +import { Dictionary } from "lodash"; +import { QUEUE_LIST } from "./health.module"; + +type QueueData = { + waitingJobs: number; + totalJobs: number; +}; + +@Injectable() +export class QueueHealthIndicator implements OnModuleInit { + private queues: Queue[] = []; + + constructor(private readonly moduleRef: ModuleRef, private readonly healthIndicatorService: HealthIndicatorService) {} + + async isHealthy() { + const indicator = this.healthIndicatorService.check("redis:bullmq"); + + try { + const data: Dictionary = {}; + for (const queue of this.queues) { + data[queue.name] = await this.checkQueue(queue); + } + + return indicator.up(data); + } catch (error) { + return indicator.down({ message: "Error fetching Airtable queue stats", error }); + } + } + + onModuleInit() { + const queueList = this.moduleRef.get(QUEUE_LIST) as string[]; + for (const name of queueList) { + this.queues.push(this.moduleRef.get(getQueueToken(name), { strict: false })); + } + } + + private async checkQueue(queue: Queue) { + const isHealthy = (await (await queue.client).ping()) === "PONG"; + if (!isHealthy) throw new Error("Redis connection for airtable queue unavailable"); + + return { + waitingJobs: await queue.getWaitingCount(), + totalJobs: await queue.count() + }; + } +} diff --git a/libs/common/src/lib/util/bootstrap-repl.ts b/libs/common/src/lib/util/bootstrap-repl.ts index 7679e842..2c986a09 100644 --- a/libs/common/src/lib/util/bootstrap-repl.ts +++ b/libs/common/src/lib/util/bootstrap-repl.ts @@ -23,7 +23,14 @@ export async function bootstrapRepl(serviceName: string, module: Type | DynamicM const replServer = await repl(module); // By default, we make lodash, the JSON API Builder, and the Sequelize models available - context = { lodash, buildJsonApi, ...replServer.context["get"](Sequelize).models, Op, ...context }; + context = { + lodash, + buildJsonApi, + Op, + Reflect, + ...replServer.context["get"](Sequelize).models, + ...context + }; for (const [name, model] of Object.entries(context as Dictionary)) { // For in REPL auto-complete functionality From b543cbf294f71545c67dabec4aecfc717f56f05a Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 12:02:23 -0700 Subject: [PATCH 10/98] [TM-1861] Specify the entity update do not email list in github secrets so we don't expose folks' email addresses. --- .env.local.sample | 2 ++ .github/workflows/deploy-service.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.env.local.sample b/.env.local.sample index 0c7a2b5f..2ba7dc1b 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -35,6 +35,8 @@ MAIL_FROM_ADDRESS=local@terramatch.org # If set, email is redirected from the original recipient to this list of addresses. The original # recipients are encoded in the X-Original-Recipients header MAIL_RECIPIENTS= +# If set, email addresses on this list will be exempted from receiving updates to entity statuses +ENTITY_UPDATE_DO_NOT_EMAIL= TRANSIFEX_TOKEN= diff --git a/.github/workflows/deploy-service.yml b/.github/workflows/deploy-service.yml index ccb7b48c..594554c1 100644 --- a/.github/workflows/deploy-service.yml +++ b/.github/workflows/deploy-service.yml @@ -78,6 +78,7 @@ jobs: echo "AIRTABLE_API_KEY=\"${{ secrets.AIRTABLE_API_KEY }}\"" >> .env echo "SENTRY_DSN=\"${{ secrets.SENTRY_DSN }}\"" >> .env echo "SLACK_API_KEY=\"${{ secrets.SLACK_API_KEY }}\"" >> .env + echo "ENTITY_UPDATE_DO_NOT_EMAIL=\"${{ secrets.ENTITY_UPDATE_DO_NOT_EMAIL }}\"" >> .env : # Don't build the base image with NODE_ENV because it'll limit the packages that are installed docker build -t terramatch-microservices-base:nx-base . SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:${{ env.IMAGE_TAG }} From 533c20bc505f818461db7ec9bdccd6d242831c3f Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 12:10:26 -0700 Subject: [PATCH 11/98] [TM-1861] Add a note about ENV configuration. --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index f6e5d4b8..ebf44db1 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,24 @@ to take effect. Once this project is live in production, we can explore continuous deployment to at least staging and prod envs on the staging and main branches. +# Environment + +The Environment for a given service deployment is configured in Github Actions secrets / variables. Some are repo-wide, and +some apply only to a given environment. During the build process, the contents of the variables applied to .env are visible +to the general public, so we need to be very careful about what is included there. Nothing sensitive (passwords, email +addresses, API tokens, etc) may be included in Variables, and must instead be in Secrets + +- If you need to update a _non-secret_ ENV variable, add / update it in the given environment's ENV variable +- If you need to add a _secret_ ENV variable, create the secret in Github actions, and then add a line to `deploy-service.yml` + to append that secret to the generated `.env` variable. +- The current value of secrets in GitHub actions may not be read by anyone, including repository admins. If you need to + inspect the current value of a configured secret, the recommended approach is to access that deployed service's REPL, and + pull the value using the `ConfigService`: + +``` +> $(ConfigService).get("SUPER_SECRETE_ENV_VALUE"); +``` + # Creating a new service - In the root directory: `nx g @nx/nest:app apps/foo-service` From 739819ff934b9a31029aa2589ade8e2872453a97 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 17:43:18 -0700 Subject: [PATCH 12/98] [TM-1861] WIP generic update endpoint. --- .../src/entities/dto/entity-update.dto.ts | 40 +++++ .../src/entities/entities.controller.ts | 31 +++- .../src/users/dto/user-update.dto.ts | 28 +--- .../src/users/users.controller.ts | 4 +- .../src/lib/util/json-api-update-dto.ts | 141 ++++++++++++++++++ 5 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 apps/entity-service/src/entities/dto/entity-update.dto.ts create mode 100644 libs/common/src/lib/util/json-api-update-dto.ts diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts new file mode 100644 index 00000000..8c40a81b --- /dev/null +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -0,0 +1,40 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { ENTITY_STATUSES, SITE_STATUSES } from "@terramatch-microservices/database/constants/status"; +import { IsBoolean, IsIn, IsOptional } from "class-validator"; +import { JsonApiDataDto, JsonApiMultiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; + +class EntityUpdateAttributes { + @IsOptional() + @IsIn(ENTITY_STATUSES) + @ApiProperty({ + description: "Request to change to the status of the given entity", + nullable: true, + enum: ENTITY_STATUSES + }) + status?: string | null; +} + +class ProjectUpdateAttributes extends EntityUpdateAttributes { + @IsOptional() + @IsBoolean() + @ApiProperty({ description: "Update the isTest flag.", nullable: true }) + isTest?: boolean; +} + +class ProjectData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} + +// Temporary stub while implementing projects +class SiteUpdateAttributes extends EntityUpdateAttributes { + @IsOptional() + @IsIn(SITE_STATUSES) + @ApiProperty({ + description: "Request to change to the status of the given entity", + nullable: true, + enum: SITE_STATUSES + }) + status?: string | null; +} + +class SiteData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} + +export class EntityUpdateBody extends JsonApiMultiBodyDto([ProjectData, SiteData] as const) {} diff --git a/apps/entity-service/src/entities/entities.controller.ts b/apps/entity-service/src/entities/entities.controller.ts index 67526248..37699a43 100644 --- a/apps/entity-service/src/entities/entities.controller.ts +++ b/apps/entity-service/src/entities/entities.controller.ts @@ -1,10 +1,12 @@ import { BadRequestException, + Body, Controller, Delete, Get, NotFoundException, Param, + Patch, Query, UnauthorizedException } from "@nestjs/common"; @@ -24,7 +26,8 @@ import { NurseryFullDto, NurseryLightDto } from "./dto/nursery.dto"; import { EntityModel } from "@terramatch-microservices/database/constants/entities"; import { JsonApiDeletedResponse } from "@terramatch-microservices/common/decorators/json-api-response.decorator"; import { NurseryReportFullDto, NurseryReportLightDto } from "./dto/nursery-report.dto"; -import { SiteReportLightDto, SiteReportFullDto } from "./dto/site-report.dto"; +import { SiteReportFullDto, SiteReportLightDto } from "./dto/site-report.dto"; +import { EntityUpdateBody } from "./dto/entity-update.dto"; @Controller("entities/v3") @ApiExtraModels(ANRDto, ProjectApplicationDto, MediaDto, EntitySideload) @@ -108,4 +111,30 @@ export class EntitiesController { return buildDeletedResponse(getDtoType(processor.FULL_DTO), model.uuid); } + + @Patch(":entity/:uuid") + @ApiOperation({ + operationId: "entityUpdate", + summary: "Update various supported entity fields directly. Typically used for status transitions" + }) + @ExceptionResponse(UnauthorizedException, { + description: "Authentication failed, or resource unavailable to current user." + }) + @ExceptionResponse(NotFoundException, { description: "Resource not found." }) + @ExceptionResponse(BadRequestException, { description: "Request params are malformed." }) + async entityUpdate( + @Param() { entity, uuid }: SpecificEntityDto, + @Body() updatePayload: EntityUpdateBody + ) { + if (entity !== updatePayload.data.type) { + throw new BadRequestException("Entity type in path and payload do not match"); + } + + const processor = this.entitiesService.createEntityProcessor(entity); + const model = await processor.findOne(uuid); + if (model == null) throw new NotFoundException(); + + // await this.policyService.authorize("update", model); + return { entity, uuid, updatePayload, type: typeof updatePayload.data }; + } } diff --git a/apps/user-service/src/users/dto/user-update.dto.ts b/apps/user-service/src/users/dto/user-update.dto.ts index d8f95a74..618f5af1 100644 --- a/apps/user-service/src/users/dto/user-update.dto.ts +++ b/apps/user-service/src/users/dto/user-update.dto.ts @@ -1,6 +1,6 @@ -import { Equals, IsEnum, IsUUID, ValidateNested } from "class-validator"; +import { IsEnum } from "class-validator"; import { ApiProperty } from "@nestjs/swagger"; -import { Type } from "class-transformer"; +import { JsonApiBodyDto, JsonApiDataDto } from "@terramatch-microservices/common/util/json-api-update-dto"; const VALID_LOCALES = ["en-US", "es-MX", "fr-FR", "pt-BR"]; @@ -10,24 +10,6 @@ class UserUpdateAttributes { locale?: string | null; } -class UserUpdate { - @Equals("users") - @ApiProperty({ enum: ["users"] }) - type: string; - - @IsUUID() - @ApiProperty({ format: "uuid" }) - id: string; - - @ValidateNested() - @Type(() => UserUpdateAttributes) - @ApiProperty({ type: () => UserUpdateAttributes }) - attributes: UserUpdateAttributes; -} - -export class UserUpdateBodyDto { - @ValidateNested() - @Type(() => UserUpdate) - @ApiProperty({ type: () => UserUpdate }) - data: UserUpdate; -} +export class UserUpdateBody extends JsonApiBodyDto( + class UserData extends JsonApiDataDto({ type: "users" }, UserUpdateAttributes) {} +) {} diff --git a/apps/user-service/src/users/users.controller.ts b/apps/user-service/src/users/users.controller.ts index 82d71eab..4db7a491 100644 --- a/apps/user-service/src/users/users.controller.ts +++ b/apps/user-service/src/users/users.controller.ts @@ -16,7 +16,7 @@ import { ApiOperation, ApiParam } from "@nestjs/swagger"; import { OrganisationDto, UserDto } from "@terramatch-microservices/common/dto"; import { ExceptionResponse, JsonApiResponse } from "@terramatch-microservices/common/decorators"; import { buildJsonApi, DocumentBuilder, JsonApiDocument } from "@terramatch-microservices/common/util"; -import { UserUpdateBodyDto } from "./dto/user-update.dto"; +import { UserUpdateBody } from "./dto/user-update.dto"; import { NoBearerAuth } from "@terramatch-microservices/common/guards"; import { UserNewRequest } from "./dto/user-new-request.dto"; import { UserCreationService } from "./user-creation.service"; @@ -73,7 +73,7 @@ export class UsersController { @ExceptionResponse(UnauthorizedException, { description: "Authorization failed" }) @ExceptionResponse(NotFoundException, { description: "User with that ID not found" }) @ExceptionResponse(BadRequestException, { description: "Something is malformed about the request" }) - async update(@Param("uuid") uuid: string, @Body() updatePayload: UserUpdateBodyDto) { + async update(@Param("uuid") uuid: string, @Body() updatePayload: UserUpdateBody) { if (uuid !== updatePayload.data.id) { throw new BadRequestException(`Path uuid and payload id do not match`); } diff --git a/libs/common/src/lib/util/json-api-update-dto.ts b/libs/common/src/lib/util/json-api-update-dto.ts new file mode 100644 index 00000000..5817d166 --- /dev/null +++ b/libs/common/src/lib/util/json-api-update-dto.ts @@ -0,0 +1,141 @@ +import { DtoOptions } from "../decorators/json-api-dto.decorator"; +import { Equals, IsIn, IsNotEmpty, IsNumberString, IsString, IsUUID, ValidateNested } from "class-validator"; +import { ApiExtraModels, ApiProperty, getSchemaPath } from "@nestjs/swagger"; +import { DiscriminatorDescriptor, Type } from "class-transformer"; +import { InternalServerErrorException } from "@nestjs/common"; +import { DECORATORS } from "@nestjs/swagger/dist/constants"; + +function UuidDataDto(type: string, AttributesDto: new () => T) { + class DataDto { + @Equals(type) + @ApiProperty({ enum: [type] }) + type: string; + + @IsUUID() + @ApiProperty({ format: "uuid" }) + id: string; + + @IsNotEmpty() + @ValidateNested() + @Type(() => AttributesDto) + @ApiProperty({ type: () => AttributesDto }) + attributes: T; + } + return DataDto; +} + +function NumberDataDto(type: string, AttributesDto: new () => T) { + class DataDto { + @Equals(type) + @ApiProperty({ enum: [type] }) + type: string; + + @IsNumberString({ no_symbols: true }) + @ApiProperty({ pattern: "^\\d{5}$" }) + id: string; + + @IsNotEmpty() + @ValidateNested() + @Type(() => AttributesDto) + @ApiProperty({ type: () => AttributesDto }) + attributes: T; + } + return DataDto; +} + +function StringDataDto(type: string, AttributesDto: new () => T) { + class DataDto { + @Equals(type) + @ApiProperty({ enum: [type] }) + type: string; + + @IsString() + @ApiProperty() + id: string; + + @IsNotEmpty() + @ValidateNested() + @Type(() => AttributesDto) + @ApiProperty({ type: () => AttributesDto }) + attributes: T; + } + return DataDto; +} + +export function JsonApiDataDto(options: DtoOptions, AttributesDto: new () => T) { + // It's tedious to have these three specified separately, but if we specify these differently as + // an intermediate base class and then a subclass with the correct id annotations, it mixes up + // the order of the properties in the resulting Swagger docs. + if (options.id === "uuid" || options.id == null) { + return UuidDataDto(options.type, AttributesDto); + } + + if (options.id === "number") { + return NumberDataDto(options.type, AttributesDto); + } + + if (options.id === "string") { + return StringDataDto(options.type, AttributesDto); + } + + throw new InternalServerErrorException(`Options id not recognized [${options.id}]`); +} + +export function JsonApiBodyDto(DataDto: new () => T) { + class BodyDto { + @IsNotEmpty() + @ValidateNested() + @Type(() => DataDto) + @ApiProperty({ type: () => DataDto }) + data: T; + } + + return BodyDto; +} + +/** + * Creates a DTO object for JSON:API update that can accept multiple different types of payload + * attributes, distinguished by the `type` param. See `entity-update.dto.ts` for example usage. + * + * Note: the DTOs passed in must be a const array in order to maintain the union type of all possible + * values in T, which is used for the typing of the `data` member of the main body DTO. + */ +export function JsonApiMultiBodyDto unknown>(dtos: readonly T[]) { + const subTypes: DiscriminatorDescriptor["subTypes"] = []; + for (const dto of dtos) { + const apiModelProperties = Reflect.getMetadata(DECORATORS.API_MODEL_PROPERTIES, dto.prototype, "type"); + if (apiModelProperties == null || apiModelProperties?.enum == null || apiModelProperties?.enum.length !== 1) { + throw new InternalServerErrorException( + "Multi body DTO must have a 'type' property defined with a single enum value" + ); + } + + subTypes.push({ name: apiModelProperties.enum[0], value: dto }); + } + + class GenericData { + @IsIn(subTypes.map(({ name }) => name)) + type: string; + } + + @ApiExtraModels(...dtos) + class MultiBodyDto { + @IsNotEmpty() + @ValidateNested() + @Type(() => GenericData, { + discriminator: { + property: "type", + subTypes + }, + keepDiscriminatorProperty: true + }) + @ApiProperty({ + oneOf: dtos.map(dto => ({ + $ref: getSchemaPath(dto) + })) + }) + data: InstanceType; + } + + return MultiBodyDto; +} From 181c22170500919bebe2a7395a34ff88bad0545c Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 20:23:04 -0700 Subject: [PATCH 13/98] [TM-1861] Basic status updates. --- .../src/entities/dto/entity-update.dto.ts | 20 ++++++++++------ .../src/entities/entities.controller.ts | 13 ++++++++-- .../src/entities/entities.service.ts | 7 +++++- .../entities/processors/entity-processor.ts | 24 ++++++++++++++++++- .../processors/nursery-report.processor.ts | 4 +++- .../entities/processors/nursery.processor.ts | 20 +++++++--------- .../processors/project-report.processor.ts | 4 +++- .../entities/processors/project.processor.ts | 23 ++++++++++++++++-- .../processors/site-report.processor.ts | 8 ++++++- .../src/entities/processors/site.processor.ts | 14 +++++++---- .../common/src/lib/policies/project.policy.ts | 11 ++++----- libs/database/src/lib/constants/status.ts | 7 ++++-- .../lib/util/model-column-state-machine.ts | 14 +++++++---- 13 files changed, 125 insertions(+), 44 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index 8c40a81b..6ee1a45d 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -3,7 +3,7 @@ import { ENTITY_STATUSES, SITE_STATUSES } from "@terramatch-microservices/databa import { IsBoolean, IsIn, IsOptional } from "class-validator"; import { JsonApiDataDto, JsonApiMultiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; -class EntityUpdateAttributes { +export class EntityUpdateAttributes { @IsOptional() @IsIn(ENTITY_STATUSES) @ApiProperty({ @@ -14,17 +14,16 @@ class EntityUpdateAttributes { status?: string | null; } -class ProjectUpdateAttributes extends EntityUpdateAttributes { +export class ProjectUpdateAttributes extends EntityUpdateAttributes { @IsOptional() @IsBoolean() @ApiProperty({ description: "Update the isTest flag.", nullable: true }) isTest?: boolean; } -class ProjectData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} +export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} -// Temporary stub while implementing projects -class SiteUpdateAttributes extends EntityUpdateAttributes { +export class SiteUpdateAttributes extends EntityUpdateAttributes { @IsOptional() @IsIn(SITE_STATUSES) @ApiProperty({ @@ -35,6 +34,13 @@ class SiteUpdateAttributes extends EntityUpdateAttributes { status?: string | null; } -class SiteData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} +// These are stubs, and most will need updating as we add support for these entity types +export class NurseryUpdateData extends JsonApiDataDto({ type: "nurseries" }, EntityUpdateAttributes) {} +export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, EntityUpdateAttributes) {} +export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, EntityUpdateAttributes) {} +export class NurseryReportUpdateData extends JsonApiDataDto({ type: "nurseryReports" }, EntityUpdateAttributes) {} -export class EntityUpdateBody extends JsonApiMultiBodyDto([ProjectData, SiteData] as const) {} +export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} + +export type EntityUpdateData = ProjectUpdateAttributes | SiteUpdateAttributes | EntityUpdateAttributes; +export class EntityUpdateBody extends JsonApiMultiBodyDto([ProjectUpdateData, SiteUpdateData] as const) {} diff --git a/apps/entity-service/src/entities/entities.controller.ts b/apps/entity-service/src/entities/entities.controller.ts index 37699a43..ffab5cee 100644 --- a/apps/entity-service/src/entities/entities.controller.ts +++ b/apps/entity-service/src/entities/entities.controller.ts @@ -126,6 +126,9 @@ export class EntitiesController { @Param() { entity, uuid }: SpecificEntityDto, @Body() updatePayload: EntityUpdateBody ) { + // The structure of the EntityUpdateBody ensures that the `type` field in the body controls + // which update body is used for validation, but it doesn't make sure that the body of the + // request matches the type in the URL path. if (entity !== updatePayload.data.type) { throw new BadRequestException("Entity type in path and payload do not match"); } @@ -134,7 +137,13 @@ export class EntitiesController { const model = await processor.findOne(uuid); if (model == null) throw new NotFoundException(); - // await this.policyService.authorize("update", model); - return { entity, uuid, updatePayload, type: typeof updatePayload.data }; + await this.policyService.authorize("update", model); + + await processor.update(model, updatePayload.data.attributes); + + const document = buildJsonApi(processor.FULL_DTO); + const { id, dto } = await processor.getFullDto(model); + document.addData(id, dto); + return document.serialize(); } } diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index e3c7327f..892fb745 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -23,6 +23,7 @@ import { SeedingDto } from "./dto/seeding.dto"; import { TreeSpeciesDto } from "./dto/tree-species.dto"; import { DemographicDto } from "./dto/demographic.dto"; import { PolicyService } from "@terramatch-microservices/common"; +import { EntityUpdateData } from "./dto/entity-update.dto"; // The keys of this array must match the type in the resulting DTO. const ENTITY_PROCESSORS = { @@ -80,13 +81,17 @@ export class EntitiesService { await this.policyService.authorize(action, subject); } + async isFrameworkAdmin({ frameworkKey }: T) { + return (await this.getPermissions()).includes(`framework-${frameworkKey}`); + } + createEntityProcessor(entity: ProcessableEntity) { const processorClass = ENTITY_PROCESSORS[entity]; if (processorClass == null) { throw new BadRequestException(`Entity type invalid: ${entity}`); } - return new processorClass(this, entity) as unknown as EntityProcessor; + return new processorClass(this, entity) as unknown as EntityProcessor; } createAssociationProcessor, D extends AssociationDto>( diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index e6393847..962e2f74 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -7,6 +7,8 @@ import { BadRequestException, Type } from "@nestjs/common"; import { EntityDto } from "../dto/entity.dto"; import { EntityClass, EntityModel } from "@terramatch-microservices/database/constants/entities"; import { Action } from "@terramatch-microservices/database/entities/action.entity"; +import { EntityUpdateData } from "../dto/entity-update.dto"; +import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; export type Aggregate> = { func: string; @@ -50,11 +52,14 @@ const getIndexData = ( export abstract class EntityProcessor< ModelType extends EntityModel, LightDto extends EntityDto, - FullDto extends EntityDto + FullDto extends EntityDto, + UpdateDto extends EntityUpdateData > { abstract readonly LIGHT_DTO: Type; abstract readonly FULL_DTO: Type; + readonly APPROVAL_STATUSES = [APPROVED, NEEDS_MORE_INFORMATION]; + constructor(protected readonly entitiesService: EntitiesService, protected readonly resource: ProcessableEntity) {} abstract findOne(uuid: string): Promise; @@ -105,4 +110,21 @@ export abstract class EntityProcessor< await Action.targetable((model.constructor as EntityClass).LARAVEL_TYPE, model.id).destroy(); await model.destroy(); } + + /** + * Performs the basic function of setting the status and saving the model. If this concrete processor + * needs to support more fields on the update dto, override this method and set the appropriate fields + * and then call super.update() + */ + async update(model: ModelType, update: UpdateDto) { + if (update.status != null) { + if (this.APPROVAL_STATUSES.includes(update.status)) { + await this.entitiesService.authorize("approve", model); + } + + model.status = update.status; + } + + await model.save(); + } } diff --git a/apps/entity-service/src/entities/processors/nursery-report.processor.ts b/apps/entity-service/src/entities/processors/nursery-report.processor.ts index 840d0590..e1531dd7 100644 --- a/apps/entity-service/src/entities/processors/nursery-report.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery-report.processor.ts @@ -10,11 +10,13 @@ import { NurseryReportLightDto, NurseryReportMedia } from "../dto/nursery-report.dto"; +import { EntityUpdateAttributes } from "../dto/entity-update.dto"; export class NurseryReportProcessor extends EntityProcessor< NurseryReport, NurseryReportLightDto, - NurseryReportFullDto + NurseryReportFullDto, + EntityUpdateAttributes > { readonly LIGHT_DTO = NurseryReportLightDto; readonly FULL_DTO = NurseryReportFullDto; diff --git a/apps/entity-service/src/entities/processors/nursery.processor.ts b/apps/entity-service/src/entities/processors/nursery.processor.ts index 6c22d56b..5cededdf 100644 --- a/apps/entity-service/src/entities/processors/nursery.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery.processor.ts @@ -1,19 +1,18 @@ -import { - Action, - Media, - Nursery, - NurseryReport, - Project, - ProjectUser -} from "@terramatch-microservices/database/entities"; +import { Media, Nursery, NurseryReport, Project, ProjectUser } from "@terramatch-microservices/database/entities"; import { AdditionalNurseryFullProps, NurseryFullDto, NurseryLightDto, NurseryMedia } from "../dto/nursery.dto"; import { EntityProcessor } from "./entity-processor"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { col, fn, Includeable, Op } from "sequelize"; import { BadRequestException, NotAcceptableException } from "@nestjs/common"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; +import { EntityUpdateAttributes } from "../dto/entity-update.dto"; -export class NurseryProcessor extends EntityProcessor { +export class NurseryProcessor extends EntityProcessor< + Nursery, + NurseryLightDto, + NurseryFullDto, + EntityUpdateAttributes +> { readonly LIGHT_DTO = NurseryLightDto; readonly FULL_DTO = NurseryFullDto; @@ -158,7 +157,6 @@ export class NurseryProcessor extends EntityProcessor { readonly LIGHT_DTO = ProjectReportLightDto; readonly FULL_DTO = ProjectReportFullDto; diff --git a/apps/entity-service/src/entities/processors/project.processor.ts b/apps/entity-service/src/entities/processors/project.processor.ts index 6a11bbe6..46e12f64 100644 --- a/apps/entity-service/src/entities/processors/project.processor.ts +++ b/apps/entity-service/src/entities/processors/project.processor.ts @@ -26,11 +26,17 @@ import { } from "../dto/project.dto"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; -import { BadRequestException } from "@nestjs/common"; +import { BadRequestException, UnauthorizedException } from "@nestjs/common"; import { ProcessableEntity } from "../entities.service"; import { DocumentBuilder } from "@terramatch-microservices/common/util"; +import { ProjectUpdateAttributes } from "../dto/entity-update.dto"; -export class ProjectProcessor extends EntityProcessor { +export class ProjectProcessor extends EntityProcessor< + Project, + ProjectLightDto, + ProjectFullDto, + ProjectUpdateAttributes +> { readonly LIGHT_DTO = ProjectLightDto; readonly FULL_DTO = ProjectFullDto; @@ -100,6 +106,19 @@ export class ProjectProcessor extends EntityProcessor { +export class SiteReportProcessor extends EntityProcessor< + SiteReport, + SiteReportLightDto, + SiteReportFullDto, + EntityUpdateAttributes +> { readonly LIGHT_DTO = SiteReportLightDto; readonly FULL_DTO = SiteReportFullDto; diff --git a/apps/entity-service/src/entities/processors/site.processor.ts b/apps/entity-service/src/entities/processors/site.processor.ts index 3a57aa2c..b58e062c 100644 --- a/apps/entity-service/src/entities/processors/site.processor.ts +++ b/apps/entity-service/src/entities/processors/site.processor.ts @@ -17,12 +17,19 @@ import { FrameworkKey } from "@terramatch-microservices/database/constants/frame import { Includeable, Op } from "sequelize"; import { sumBy } from "lodash"; import { EntityQueryDto } from "../dto/entity-query.dto"; -import { Action } from "@terramatch-microservices/database/entities/action.entity"; +import { SiteUpdateAttributes } from "../dto/entity-update.dto"; +import { + APPROVED, + NEEDS_MORE_INFORMATION, + RESTORATION_IN_PROGRESS +} from "@terramatch-microservices/database/constants/status"; -export class SiteProcessor extends EntityProcessor { +export class SiteProcessor extends EntityProcessor { readonly LIGHT_DTO = SiteLightDto; readonly FULL_DTO = SiteFullDto; + readonly APPROVAL_STATUSES = [APPROVED, NEEDS_MORE_INFORMATION, RESTORATION_IN_PROGRESS]; + async findOne(uuid: string) { return await Site.findOne({ where: { uuid }, @@ -206,7 +213,6 @@ export class SiteProcessor extends EntityProcessor 0) { - this.builder.can("read", Project, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", Project, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], Project, { frameworkKey: { $in: this.frameworks } }); } if (this.permissions.includes("manage-own")) { const user = await this.getUser(); if (user != null) { - this.builder.can("read", Project, { organisationId: user.organisationId }); + this.builder.can(["read", "update"], Project, { organisationId: user.organisationId }); this.builder.can("delete", Project, { organisationId: user.organisationId, status: STARTED }); const projectIds = user.projects.map(({ id }) => id); if (projectIds.length > 0) { - this.builder.can("read", Project, { id: { $in: projectIds } }); + this.builder.can(["read", "update"], Project, { id: { $in: projectIds } }); this.builder.can("delete", Project, { id: { $in: projectIds }, status: STARTED }); } } @@ -32,8 +30,7 @@ export class ProjectPolicy extends UserPermissionsPolicy { if (user != null) { const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); if (projectIds.length > 0) { - this.builder.can("read", Project, { id: { $in: projectIds } }); - this.builder.can("delete", Project, { id: { $in: projectIds } }); + this.builder.can(["read", "delete", "update", "approve"], Project, { id: { $in: projectIds } }); } } } diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index b84b842b..52339b52 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -1,5 +1,5 @@ import { States } from "../util/model-column-state-machine"; -import { Nursery, Project, Site } from "../entities"; +import { Nursery, Project } from "../entities"; import { Model } from "sequelize-typescript"; import { DatabaseModule } from "../database.module"; @@ -14,14 +14,17 @@ export type EntityStatus = (typeof ENTITY_STATUSES)[number]; const emitStatusUpdateHook = (from: string, model: Model) => { DatabaseModule.emitModelEvent("statusUpdated", model); }; -export const EntityStatusStates: States = { + +export const EntityStatusStates: States = { default: STARTED, + transitions: { [STARTED]: [AWAITING_APPROVAL], [AWAITING_APPROVAL]: [APPROVED, NEEDS_MORE_INFORMATION], [NEEDS_MORE_INFORMATION]: [APPROVED, AWAITING_APPROVAL], [APPROVED]: [NEEDS_MORE_INFORMATION] }, + afterTransitionHooks: { [APPROVED]: emitStatusUpdateHook, [AWAITING_APPROVAL]: emitStatusUpdateHook, diff --git a/libs/database/src/lib/util/model-column-state-machine.ts b/libs/database/src/lib/util/model-column-state-machine.ts index 62ea2f6f..afe7db32 100644 --- a/libs/database/src/lib/util/model-column-state-machine.ts +++ b/libs/database/src/lib/util/model-column-state-machine.ts @@ -1,5 +1,6 @@ import { Column, Model } from "sequelize-typescript"; import { Attributes, STRING } from "sequelize"; +import { HttpException, HttpStatus } from "@nestjs/common"; export type States = { default: S; @@ -28,7 +29,12 @@ export type States = { afterTransitionHooks?: Partial void>>; }; -export class StateMachineError extends Error {} +// Extends HttpException so these errors bubble up to the API consumer +export class StateMachineException extends HttpException { + constructor(message: string) { + super(message, HttpStatus.BAD_REQUEST); + } +} export type StateMachineModel = M & { _stateMachines?: Record>; @@ -103,7 +109,7 @@ export class ModelColumnStateMachine { canBe(from: S, to: S) { if (!Object.keys(this.states.transitions).includes(from)) { - throw new StateMachineError(`Current state is not defined [${from}]`); + throw new StateMachineException(`Current state is not defined [${from}]`); } return from === to || this.states.transitions[from]?.includes(to) === true; @@ -120,14 +126,14 @@ export class ModelColumnStateMachine { if (this.model.isNewRecord) return; if (!this.canBe(this.current, to)) { - throw new StateMachineError(`Transition not valid [from=${this.current}, to=${to}]`); + throw new StateMachineException(`Transition not valid [from=${this.current}, to=${to}]`); } if ( this.states.transitionValidForModel != null && !this.states.transitionValidForModel(this.current, to, this.model) ) { - throw new StateMachineError( + throw new StateMachineException( `Transition not valid for model [from=${this.current}, to=${to}, id=${this.model.id}]` ); } From 03135986dbbabe1802333113e80100d456079d86 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 20:32:31 -0700 Subject: [PATCH 14/98] [TM-1861] Support for feedback / feedbackFields --- .../src/entities/dto/entity-update.dto.ts | 19 ++++++++++++++++++- .../entities/processors/entity-processor.ts | 12 +++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index 6ee1a45d..37382c9f 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -1,7 +1,8 @@ import { ApiProperty } from "@nestjs/swagger"; import { ENTITY_STATUSES, SITE_STATUSES } from "@terramatch-microservices/database/constants/status"; -import { IsBoolean, IsIn, IsOptional } from "class-validator"; +import { IsArray, IsBoolean, IsIn, IsOptional, IsString } from "class-validator"; import { JsonApiDataDto, JsonApiMultiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; +import { Type } from "class-transformer"; export class EntityUpdateAttributes { @IsOptional() @@ -12,6 +13,22 @@ export class EntityUpdateAttributes { enum: ENTITY_STATUSES }) status?: string | null; + + @IsOptional() + @IsString() + @ApiProperty({ description: "Specific feedback for the PD", nullable: true }) + feedback?: string | null; + + @IsOptional() + @IsArray() + @Type(() => String) + @ApiProperty({ + isArray: true, + type: String, + description: "The fields in the entity form that need attention from the PD", + nullable: true + }) + feedbackFields?: string[] | null; } export class ProjectUpdateAttributes extends EntityUpdateAttributes { diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index 962e2f74..f8383b5a 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -112,14 +112,20 @@ export abstract class EntityProcessor< } /** - * Performs the basic function of setting the status and saving the model. If this concrete processor - * needs to support more fields on the update dto, override this method and set the appropriate fields - * and then call super.update() + * Performs the basic function of setting fields in EntityUpdateAttributes and saving the model. + * If this concrete processor needs to support more fields on the update dto, override this method + * and set the appropriate fields and then call super.update() */ async update(model: ModelType, update: UpdateDto) { if (update.status != null) { if (this.APPROVAL_STATUSES.includes(update.status)) { await this.entitiesService.authorize("approve", model); + + // If an admin is doing an update, set the feedback / feedbackFields to whatever is in the + // request, even if it's null. We ignore feedback / feedbackFields if the status is not + // also being updated. + model.feedback = update.feedback; + model.feedbackFields = update.feedbackFields; } model.status = update.status; From 7db3b82bf4166c710507e82fde9214b87eb87962 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 20:40:44 -0700 Subject: [PATCH 15/98] [TM-1861] Specify the update DTO for each entity type. --- .../src/entities/dto/entity-update.dto.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index 37382c9f..b5b86f6b 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -51,13 +51,19 @@ export class SiteUpdateAttributes extends EntityUpdateAttributes { status?: string | null; } -// These are stubs, and most will need updating as we add support for these entity types +export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} + export class NurseryUpdateData extends JsonApiDataDto({ type: "nurseries" }, EntityUpdateAttributes) {} export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, EntityUpdateAttributes) {} export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, EntityUpdateAttributes) {} export class NurseryReportUpdateData extends JsonApiDataDto({ type: "nurseryReports" }, EntityUpdateAttributes) {} -export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} - export type EntityUpdateData = ProjectUpdateAttributes | SiteUpdateAttributes | EntityUpdateAttributes; -export class EntityUpdateBody extends JsonApiMultiBodyDto([ProjectUpdateData, SiteUpdateData] as const) {} +export class EntityUpdateBody extends JsonApiMultiBodyDto([ + ProjectUpdateData, + SiteUpdateData, + NurseryUpdateData, + ProjectReportUpdateData, + SiteReportUpdateData, + NurseryReportUpdateData +] as const) {} From b108f37b23d6f48fbf85b41ce9a6cbe7fbb9f289 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 20:54:52 -0700 Subject: [PATCH 16/98] [TM-1861] Double check the path / payload entity ID as well. --- apps/entity-service/src/entities/entities.controller.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/entity-service/src/entities/entities.controller.ts b/apps/entity-service/src/entities/entities.controller.ts index ffab5cee..7ecb4566 100644 --- a/apps/entity-service/src/entities/entities.controller.ts +++ b/apps/entity-service/src/entities/entities.controller.ts @@ -132,6 +132,9 @@ export class EntitiesController { if (entity !== updatePayload.data.type) { throw new BadRequestException("Entity type in path and payload do not match"); } + if (uuid !== updatePayload.data.id) { + throw new BadRequestException("Entity id in path and payload do not match"); + } const processor = this.entitiesService.createEntityProcessor(entity); const model = await processor.findOne(uuid); From 0841c37d785d6a40d97a96128e6e03034ed55d15 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 21:26:54 -0700 Subject: [PATCH 17/98] [TM-1861] Use the updated polymorphic chain scope pattern for Media. --- .../processors/nursery-report.processor.ts | 3 +- .../entities/processors/nursery.processor.ts | 5 +- .../processors/project-report.processor.ts | 3 +- .../entities/processors/project.processor.ts | 5 +- .../processors/site-report.processor.ts | 2 +- .../src/entities/processors/site.processor.ts | 2 +- .../lib/email/entity-status-update.email.ts | 2 +- .../database/src/lib/entities/media.entity.ts | 67 ++----------------- .../src/lib/entities/update-request.entity.ts | 2 +- libs/database/src/lib/types/util.ts | 7 +- 10 files changed, 19 insertions(+), 79 deletions(-) diff --git a/apps/entity-service/src/entities/processors/nursery-report.processor.ts b/apps/entity-service/src/entities/processors/nursery-report.processor.ts index e1531dd7..e049c04c 100644 --- a/apps/entity-service/src/entities/processors/nursery-report.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery-report.processor.ts @@ -134,8 +134,7 @@ export class NurseryReportProcessor extends EntityProcessor< } async getFullDto(nurseryReport: NurseryReport) { - const nurseryReportId = nurseryReport.id; - const mediaCollection = await Media.nurseryReport(nurseryReportId).findAll(); + const mediaCollection = await Media.for(nurseryReport).findAll(); const reportTitle = await this.getReportTitle(nurseryReport); const projectReportTitle = await this.getProjectReportTitle(nurseryReport); const migrated = nurseryReport.oldModel != null; diff --git a/apps/entity-service/src/entities/processors/nursery.processor.ts b/apps/entity-service/src/entities/processors/nursery.processor.ts index 5cededdf..1926ef59 100644 --- a/apps/entity-service/src/entities/processors/nursery.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery.processor.ts @@ -113,10 +113,7 @@ export class NurseryProcessor extends EntityProcessor< nurseryReportsTotal, overdueNurseryReportsTotal, - ...(this.entitiesService.mapMediaCollection( - await Media.nursery(nurseryId).findAll(), - Nursery.MEDIA - ) as NurseryMedia) + ...(this.entitiesService.mapMediaCollection(await Media.for(nursery).findAll(), Nursery.MEDIA) as NurseryMedia) }; return { id: nursery.uuid, dto: new NurseryFullDto(nursery, props) }; diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index 5d5000e9..283e6689 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -129,7 +129,6 @@ export class ProjectReportProcessor extends EntityProcessor< } async getFullDto(projectReport: ProjectReport) { - const projectReportId = projectReport.id; const taskId = projectReport.taskId; const reportTitle = await this.getReportTitle(projectReport); const siteReportsIdsTask = ProjectReport.siteReportIdsTaskSubquery([taskId]); @@ -161,7 +160,7 @@ export class ProjectReportProcessor extends EntityProcessor< readableCompletionStatus, createdByUser, ...(this.entitiesService.mapMediaCollection( - await Media.projectReport(projectReportId).findAll(), + await Media.for(projectReport).findAll(), ProjectReport.MEDIA ) as ProjectReportMedia) }; diff --git a/apps/entity-service/src/entities/processors/project.processor.ts b/apps/entity-service/src/entities/processors/project.processor.ts index 3e4b8a40..ce321d4c 100644 --- a/apps/entity-service/src/entities/processors/project.processor.ts +++ b/apps/entity-service/src/entities/processors/project.processor.ts @@ -191,10 +191,7 @@ export class ProjectProcessor extends EntityProcessor< application: project.application == null ? null : new ProjectApplicationDto(project.application), - ...(this.entitiesService.mapMediaCollection( - await Media.project(project.id).findAll(), - Project.MEDIA - ) as ProjectMedia) + ...(this.entitiesService.mapMediaCollection(await Media.for(project).findAll(), Project.MEDIA) as ProjectMedia) }; return { id: project.uuid, dto: new ProjectFullDto(project, props) }; diff --git a/apps/entity-service/src/entities/processors/site-report.processor.ts b/apps/entity-service/src/entities/processors/site-report.processor.ts index a072b87b..abd235a6 100644 --- a/apps/entity-service/src/entities/processors/site-report.processor.ts +++ b/apps/entity-service/src/entities/processors/site-report.processor.ts @@ -152,7 +152,7 @@ export class SiteReportProcessor extends EntityProcessor< (await TreeSpecies.visible().collection("non-tree").siteReports([siteReportId]).sum("amount")) ?? 0; const totalTreeReplantingCount = (await TreeSpecies.visible().collection("replanting").siteReports([siteReportId]).sum("amount")) ?? 0; - const mediaCollection = await Media.siteReport(siteReportId).findAll(); + const mediaCollection = await Media.for(siteReport).findAll(); const props: AdditionalSiteReportFullProps = { reportTitle, projectReportTitle, diff --git a/apps/entity-service/src/entities/processors/site.processor.ts b/apps/entity-service/src/entities/processors/site.processor.ts index 03b7d37c..b627e252 100644 --- a/apps/entity-service/src/entities/processors/site.processor.ts +++ b/apps/entity-service/src/entities/processors/site.processor.ts @@ -156,7 +156,7 @@ export class SiteProcessor extends EntityProcessor ({ order: ["orderColumn"] })) @Scopes(() => ({ collection: (collectionName: string) => ({ where: { collectionName } }), - project: (id: number) => ({ + association: (association: LaravelModel) => ({ where: { - modelType: Project.LARAVEL_TYPE, - modelId: id - } - }), - nursery: (id: number) => ({ - where: { - modelType: Nursery.LARAVEL_TYPE, - modelId: id - } - }), - site: (id: number) => ({ - where: { - modelType: Site.LARAVEL_TYPE, - modelId: id - } - }), - nurseryReport: (id: number) => ({ - where: { - modelType: NurseryReport.LARAVEL_TYPE, - modelId: id - } - }), - projectReport: (id: number) => ({ - where: { - modelType: ProjectReport.LARAVEL_TYPE, - modelId: id - } - }), - siteReport: (id: number) => ({ - where: { - modelType: SiteReport.LARAVEL_TYPE, - modelId: id + modelType: (association.constructor as LaravelModelCtor).LARAVEL_TYPE, + modelId: association.id } }) })) @@ -75,28 +40,8 @@ export class Media extends Model { return chainScope(this, "collection", collectionName) as typeof Media; } - static project(id: number) { - return chainScope(this, "project", id) as typeof Media; - } - - static site(id: number) { - return chainScope(this, "site", id) as typeof Media; - } - - static nursery(id: number) { - return chainScope(this, "nursery", id) as typeof Media; - } - - static nurseryReport(id: number) { - return chainScope(this, "nurseryReport", id) as typeof Media; - } - - static projectReport(id: number) { - return chainScope(this, "projectReport", id) as typeof Media; - } - - static siteReport(id: number) { - return chainScope(this, "siteReport", id) as typeof Media; + static for(model: LaravelModel) { + return chainScope(this, "association", model) as typeof Media; } @PrimaryKey diff --git a/libs/database/src/lib/entities/update-request.entity.ts b/libs/database/src/lib/entities/update-request.entity.ts index b8fd4624..a4319d78 100644 --- a/libs/database/src/lib/entities/update-request.entity.ts +++ b/libs/database/src/lib/entities/update-request.entity.ts @@ -28,7 +28,7 @@ import { chainScope } from "../util/chain-scope"; })) @Table({ tableName: "v2_update_requests", underscored: true, paranoid: true }) export class UpdateRequest extends Model { - static entity(entity: T) { + static for(entity: T) { return chainScope(this, "entity", entity) as typeof UpdateRequest; } diff --git a/libs/database/src/lib/types/util.ts b/libs/database/src/lib/types/util.ts index 24bfc93f..0ea63756 100644 --- a/libs/database/src/lib/types/util.ts +++ b/libs/database/src/lib/types/util.ts @@ -1,3 +1,6 @@ -import { Model } from "sequelize-typescript"; +import { Model, ModelCtor } from "sequelize-typescript"; -export type UuidModel = Model & { uuid: string }; +export type UuidModel = Model & { uuid: string }; + +export type LaravelModelCtor = ModelCtor & { LARAVEL_TYPE: string }; +export type LaravelModel = InstanceType; From ae50eb4273d4f10e42bc42f62bb541eb00dad927 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 21:37:05 -0700 Subject: [PATCH 18/98] [TM-1861] Generate UUIDs automatically. --- apps/user-service/src/users/user-creation.service.ts | 2 +- libs/database/src/lib/entities/action.entity.ts | 4 ++-- libs/database/src/lib/entities/application.entity.ts | 4 ++-- libs/database/src/lib/entities/delayed-job.entity.ts | 4 ++-- libs/database/src/lib/entities/demographic.entity.ts | 4 ++-- libs/database/src/lib/entities/form-submission.entity.ts | 4 ++-- libs/database/src/lib/entities/framework.entity.ts | 4 ++-- libs/database/src/lib/entities/funding-programme.entity.ts | 4 ++-- libs/database/src/lib/entities/media.entity.ts | 4 ++-- libs/database/src/lib/entities/nursery-report.entity.ts | 4 ++-- libs/database/src/lib/entities/nursery.entity.ts | 4 ++-- libs/database/src/lib/entities/organisation.entity.ts | 4 ++-- libs/database/src/lib/entities/point-geometry.entity.ts | 4 ++-- libs/database/src/lib/entities/polygon-geometry.entity.ts | 4 ++-- libs/database/src/lib/entities/project-pitch.entity.ts | 4 ++-- libs/database/src/lib/entities/project-report.entity.ts | 4 ++-- libs/database/src/lib/entities/project.entity.ts | 4 ++-- libs/database/src/lib/entities/seeding.entity.ts | 4 ++-- libs/database/src/lib/entities/site-polygon.entity.ts | 4 ++-- libs/database/src/lib/entities/site-report.entity.ts | 4 ++-- libs/database/src/lib/entities/site.entity.ts | 4 ++-- libs/database/src/lib/entities/task.entity.ts | 4 ++-- libs/database/src/lib/entities/tree-species.entity.ts | 4 ++-- libs/database/src/lib/entities/update-request.entity.ts | 4 ++-- libs/database/src/lib/entities/user.entity.ts | 4 ++-- libs/database/src/lib/factories/action.factory.ts | 1 - libs/database/src/lib/factories/application.factory.ts | 1 - libs/database/src/lib/factories/delayed-job.factory.ts | 1 - .../database/src/lib/factories/demographic-entry.factory.ts | 1 - libs/database/src/lib/factories/demographic.factory.ts | 1 - libs/database/src/lib/factories/form-submission.factory.ts | 1 - .../database/src/lib/factories/funding-programme.factory.ts | 1 - libs/database/src/lib/factories/media.factory.ts | 1 - libs/database/src/lib/factories/nursery-report.factory.ts | 1 - libs/database/src/lib/factories/nursery.factory.ts | 1 - libs/database/src/lib/factories/organisation.factory.ts | 1 - libs/database/src/lib/factories/polygon-geometry.factory.ts | 1 - libs/database/src/lib/factories/project-pitch.factory.ts | 4 +--- libs/database/src/lib/factories/project-report.factory.ts | 1 - libs/database/src/lib/factories/project.factory.ts | 1 - libs/database/src/lib/factories/seeding.factory.ts | 1 - libs/database/src/lib/factories/site-report.factory.ts | 1 - libs/database/src/lib/factories/site.factory.ts | 1 - libs/database/src/lib/factories/task.factory.ts | 6 +----- libs/database/src/lib/factories/tree-species.factory.ts | 1 - libs/database/src/lib/factories/user.factory.ts | 1 - 46 files changed, 51 insertions(+), 76 deletions(-) diff --git a/apps/user-service/src/users/user-creation.service.ts b/apps/user-service/src/users/user-creation.service.ts index c1833cc0..b6599d56 100644 --- a/apps/user-service/src/users/user-creation.service.ts +++ b/apps/user-service/src/users/user-creation.service.ts @@ -42,7 +42,7 @@ export class UserCreationService { const callbackUrl = request.callbackUrl; const newUser = omit(request, ["callbackUrl", "role", "password"]); - const user = await User.create({ ...newUser, uuid: crypto.randomUUID(), password: hashPassword }); + const user = await User.create({ ...newUser, password: hashPassword }); await user.reload(); diff --git a/libs/database/src/lib/entities/action.entity.ts b/libs/database/src/lib/entities/action.entity.ts index 62507be6..d81afad0 100644 --- a/libs/database/src/lib/entities/action.entity.ts +++ b/libs/database/src/lib/entities/action.entity.ts @@ -10,7 +10,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, STRING, UUID } from "sequelize"; +import { BIGINT, STRING, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { Project } from "./project.entity"; import { Literal } from "sequelize/types/utils"; @@ -44,7 +44,7 @@ export class Action extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/application.entity.ts b/libs/database/src/lib/entities/application.entity.ts index 4889c211..43489ad6 100644 --- a/libs/database/src/lib/entities/application.entity.ts +++ b/libs/database/src/lib/entities/application.entity.ts @@ -10,7 +10,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, UUID } from "sequelize"; +import { BIGINT, UUID, UUIDV4 } from "sequelize"; import { User } from "./user.entity"; import { FormSubmission } from "./form-submission.entity"; import { FundingProgramme } from "./funding-programme.entity"; @@ -30,7 +30,7 @@ export class Application extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/delayed-job.entity.ts b/libs/database/src/lib/entities/delayed-job.entity.ts index e58e717d..e5006078 100644 --- a/libs/database/src/lib/entities/delayed-job.entity.ts +++ b/libs/database/src/lib/entities/delayed-job.entity.ts @@ -9,7 +9,7 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, INTEGER, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, INTEGER, STRING, UUID, UUIDV4 } from "sequelize"; import { User } from "./user.entity"; import { JsonColumn } from "../decorators/json-column.decorator"; @@ -26,7 +26,7 @@ export class DelayedJob extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Default("pending") diff --git a/libs/database/src/lib/entities/demographic.entity.ts b/libs/database/src/lib/entities/demographic.entity.ts index 587e9f1a..67b20df7 100644 --- a/libs/database/src/lib/entities/demographic.entity.ts +++ b/libs/database/src/lib/entities/demographic.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, HasMany, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { DemographicEntry } from "./demographic-entry.entity"; import { Literal } from "sequelize/types/utils"; import { Subquery } from "../util/subquery.builder"; @@ -50,7 +50,7 @@ export class Demographic extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(STRING) diff --git a/libs/database/src/lib/entities/form-submission.entity.ts b/libs/database/src/lib/entities/form-submission.entity.ts index b7ae8af2..f70c06ca 100644 --- a/libs/database/src/lib/entities/form-submission.entity.ts +++ b/libs/database/src/lib/entities/form-submission.entity.ts @@ -9,7 +9,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, CHAR, STRING, UUID } from "sequelize"; +import { BIGINT, CHAR, STRING, UUID, UUIDV4 } from "sequelize"; import { Application } from "./application.entity"; import { FormSubmissionStatus } from "../constants/status"; @@ -22,7 +22,7 @@ export class FormSubmission extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(STRING) diff --git a/libs/database/src/lib/entities/framework.entity.ts b/libs/database/src/lib/entities/framework.entity.ts index 163d676c..a4f81964 100644 --- a/libs/database/src/lib/entities/framework.entity.ts +++ b/libs/database/src/lib/entities/framework.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, STRING, UUID } from "sequelize"; +import { BIGINT, STRING, UUID, UUIDV4 } from "sequelize"; // Incomplete stub @Table({ tableName: "frameworks", underscored: true }) @@ -10,7 +10,7 @@ export class Framework extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Index diff --git a/libs/database/src/lib/entities/funding-programme.entity.ts b/libs/database/src/lib/entities/funding-programme.entity.ts index 38ed498e..347450a3 100644 --- a/libs/database/src/lib/entities/funding-programme.entity.ts +++ b/libs/database/src/lib/entities/funding-programme.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, STRING, UUID } from "sequelize"; +import { BIGINT, STRING, UUID, UUIDV4 } from "sequelize"; // Incomplete stub @Table({ tableName: "funding_programmes", underscored: true, paranoid: true }) @@ -10,7 +10,7 @@ export class FundingProgramme extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(STRING) diff --git a/libs/database/src/lib/entities/media.entity.ts b/libs/database/src/lib/entities/media.entity.ts index 876ab5e8..13641d1e 100644 --- a/libs/database/src/lib/entities/media.entity.ts +++ b/libs/database/src/lib/entities/media.entity.ts @@ -10,7 +10,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DOUBLE, ENUM, INTEGER, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DOUBLE, ENUM, INTEGER, STRING, UUID, UUIDV4 } from "sequelize"; import { JsonColumn } from "../decorators/json-column.decorator"; import { User } from "./user.entity"; import { chainScope } from "../util/chain-scope"; @@ -50,7 +50,7 @@ export class Media extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(STRING) diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index 3288100f..2bf907cb 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { Nursery } from "./nursery.entity"; import { TreeSpecies } from "./tree-species.entity"; import { COMPLETE_REPORT_STATUSES, ReportStatus, UpdateRequestStatus } from "../constants/status"; @@ -72,7 +72,7 @@ export class NurseryReport extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/nursery.entity.ts b/libs/database/src/lib/entities/nursery.entity.ts index c95e3a22..1af7613d 100644 --- a/libs/database/src/lib/entities/nursery.entity.ts +++ b/libs/database/src/lib/entities/nursery.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { Project } from "./project.entity"; import { TreeSpecies } from "./tree-species.entity"; import { NurseryReport } from "./nursery-report.entity"; @@ -56,7 +56,7 @@ export class Nursery extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(STRING) diff --git a/libs/database/src/lib/entities/organisation.entity.ts b/libs/database/src/lib/entities/organisation.entity.ts index 3ff173a6..4fdb74a6 100644 --- a/libs/database/src/lib/entities/organisation.entity.ts +++ b/libs/database/src/lib/entities/organisation.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { JsonColumn } from "../decorators/json-column.decorator"; import { OrganisationStatus } from "../constants/status"; @@ -13,7 +13,7 @@ export class Organisation extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Default("draft") diff --git a/libs/database/src/lib/entities/point-geometry.entity.ts b/libs/database/src/lib/entities/point-geometry.entity.ts index 16822fe2..a21a9d73 100644 --- a/libs/database/src/lib/entities/point-geometry.entity.ts +++ b/libs/database/src/lib/entities/point-geometry.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, DECIMAL, GEOMETRY, UUID } from "sequelize"; +import { BIGINT, DECIMAL, GEOMETRY, UUID, UUIDV4 } from "sequelize"; import { Point } from "geojson"; import { User } from "./user.entity"; @@ -11,7 +11,7 @@ export class PointGeometry extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/polygon-geometry.entity.ts b/libs/database/src/lib/entities/polygon-geometry.entity.ts index 719ece09..89455b75 100644 --- a/libs/database/src/lib/entities/polygon-geometry.entity.ts +++ b/libs/database/src/lib/entities/polygon-geometry.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, GEOMETRY, UUID } from "sequelize"; +import { BIGINT, GEOMETRY, UUID, UUIDV4 } from "sequelize"; import { Polygon } from "geojson"; import { User } from "./user.entity"; @@ -11,7 +11,7 @@ export class PolygonGeometry extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/project-pitch.entity.ts b/libs/database/src/lib/entities/project-pitch.entity.ts index 5141ae20..af33d4d2 100644 --- a/libs/database/src/lib/entities/project-pitch.entity.ts +++ b/libs/database/src/lib/entities/project-pitch.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, UUID } from "sequelize"; +import { BIGINT, UUID, UUIDV4 } from "sequelize"; // Incomplete stub @Table({ tableName: "project_pitches", underscored: true, paranoid: true }) @@ -12,6 +12,6 @@ export class ProjectPitch extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; } diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index d07c8d06..a1a87648 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Project } from "./project.entity"; import { FrameworkKey } from "../constants/framework"; @@ -103,7 +103,7 @@ export class ProjectReport extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts index c508709b..ec83e17e 100644 --- a/libs/database/src/lib/entities/project.entity.ts +++ b/libs/database/src/lib/entities/project.entity.ts @@ -11,7 +11,7 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { TreeSpecies } from "./tree-species.entity"; import { ProjectReport } from "./project-report.entity"; @@ -56,7 +56,7 @@ export class Project extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/seeding.entity.ts b/libs/database/src/lib/entities/seeding.entity.ts index 62594392..37609540 100644 --- a/libs/database/src/lib/entities/seeding.entity.ts +++ b/libs/database/src/lib/entities/seeding.entity.ts @@ -9,7 +9,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DOUBLE, Op, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DOUBLE, Op, STRING, UUID, UUIDV4 } from "sequelize"; import { TreeSpeciesResearch } from "./tree-species-research.entity"; import { Literal } from "sequelize/types/utils"; import { SiteReport } from "./site-report.entity"; @@ -46,7 +46,7 @@ export class Seeding extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/site-polygon.entity.ts b/libs/database/src/lib/entities/site-polygon.entity.ts index 5929aac4..a9687cd9 100644 --- a/libs/database/src/lib/entities/site-polygon.entity.ts +++ b/libs/database/src/lib/entities/site-polygon.entity.ts @@ -12,7 +12,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, DOUBLE, INTEGER, Op, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, DOUBLE, INTEGER, Op, STRING, UUID, UUIDV4 } from "sequelize"; import { Site } from "./site.entity"; import { PointGeometry } from "./point-geometry.entity"; import { PolygonGeometry } from "./polygon-geometry.entity"; @@ -61,7 +61,7 @@ export class SitePolygon extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @Column(UUID) diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 11f07dcb..19e03534 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; @@ -93,7 +93,7 @@ export class SiteReport extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts index 8dd6aa57..56452ef3 100644 --- a/libs/database/src/lib/entities/site.entity.ts +++ b/libs/database/src/lib/entities/site.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, DECIMAL, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, DECIMAL, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { SiteReport } from "./site-report.entity"; import { Project } from "./project.entity"; @@ -79,7 +79,7 @@ export class Site extends Model { updateRequestStatus: UpdateRequestStatus | null; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/task.entity.ts b/libs/database/src/lib/entities/task.entity.ts index 1f0f5659..feba4a7a 100644 --- a/libs/database/src/lib/entities/task.entity.ts +++ b/libs/database/src/lib/entities/task.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, HasOne, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, UUID } from "sequelize"; +import { BIGINT, UUID, UUIDV4 } from "sequelize"; import { ProjectReport } from "./project-report.entity"; @Table({ tableName: "v2_tasks", underscored: true, paranoid: true }) @@ -12,7 +12,7 @@ export class Task extends Model { override id: number; @Index - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @HasOne(() => ProjectReport) diff --git a/libs/database/src/lib/entities/tree-species.entity.ts b/libs/database/src/lib/entities/tree-species.entity.ts index fde25cbe..46b9211b 100644 --- a/libs/database/src/lib/entities/tree-species.entity.ts +++ b/libs/database/src/lib/entities/tree-species.entity.ts @@ -10,7 +10,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, Op, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, Op, STRING, UUID, UUIDV4 } from "sequelize"; import { TreeSpeciesResearch } from "./tree-species-research.entity"; import { Literal } from "sequelize/types/utils"; import { SiteReport } from "./site-report.entity"; @@ -77,7 +77,7 @@ export class TreeSpecies extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/update-request.entity.ts b/libs/database/src/lib/entities/update-request.entity.ts index a4319d78..e88aa6d2 100644 --- a/libs/database/src/lib/entities/update-request.entity.ts +++ b/libs/database/src/lib/entities/update-request.entity.ts @@ -9,7 +9,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { Project } from "./project.entity"; import { User } from "./user.entity"; @@ -38,7 +38,7 @@ export class UpdateRequest extends Model { override id: number; @Unique - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; @AllowNull diff --git a/libs/database/src/lib/entities/user.entity.ts b/libs/database/src/lib/entities/user.entity.ts index 3926f6de..c4431d99 100644 --- a/libs/database/src/lib/entities/user.entity.ts +++ b/libs/database/src/lib/entities/user.entity.ts @@ -13,7 +13,7 @@ import { Table, Unique } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, col, DATE, fn, Op, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, col, DATE, fn, Op, STRING, UUID, UUIDV4 } from "sequelize"; import { Role } from "./role.entity"; import { ModelHasRole } from "./model-has-role.entity"; import { Permission } from "./permission.entity"; @@ -37,7 +37,7 @@ export class User extends Model { // index until that is fixed. @AllowNull @Index({ unique: false }) - @Column(UUID) + @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string | null; @ForeignKey(() => Organisation) diff --git a/libs/database/src/lib/factories/action.factory.ts b/libs/database/src/lib/factories/action.factory.ts index e3c8669f..999660ed 100644 --- a/libs/database/src/lib/factories/action.factory.ts +++ b/libs/database/src/lib/factories/action.factory.ts @@ -4,7 +4,6 @@ import { OrganisationFactory } from "./organisation.factory"; import { ProjectFactory } from "./project.factory"; const defaultAttributesFactory = async () => ({ - uuid: crypto.randomUUID(), organisationId: OrganisationFactory.associate("id"), projectId: ProjectFactory.associate("id") }); diff --git a/libs/database/src/lib/factories/application.factory.ts b/libs/database/src/lib/factories/application.factory.ts index a39ec929..972717d2 100644 --- a/libs/database/src/lib/factories/application.factory.ts +++ b/libs/database/src/lib/factories/application.factory.ts @@ -4,7 +4,6 @@ import { OrganisationFactory } from "./organisation.factory"; import { FundingProgrammeFactory } from "./funding-programme.factory"; export const ApplicationFactory = FactoryGirl.define(Application, async () => ({ - uuid: crypto.randomUUID(), organisationUuid: OrganisationFactory.associate("uuid"), fundingProgrammeUuid: FundingProgrammeFactory.associate("uuid") })); diff --git a/libs/database/src/lib/factories/delayed-job.factory.ts b/libs/database/src/lib/factories/delayed-job.factory.ts index ad31459a..8091a929 100644 --- a/libs/database/src/lib/factories/delayed-job.factory.ts +++ b/libs/database/src/lib/factories/delayed-job.factory.ts @@ -2,7 +2,6 @@ import { FactoryGirl } from "factory-girl-ts"; import { DelayedJob } from "../entities"; export const DelayedJobFactory = FactoryGirl.define(DelayedJob, async () => ({ - uuid: crypto.randomUUID(), status: "succeeded", statusCode: 200, payload: { data: "test" }, diff --git a/libs/database/src/lib/factories/demographic-entry.factory.ts b/libs/database/src/lib/factories/demographic-entry.factory.ts index fbe8d946..dbcb6fec 100644 --- a/libs/database/src/lib/factories/demographic-entry.factory.ts +++ b/libs/database/src/lib/factories/demographic-entry.factory.ts @@ -20,7 +20,6 @@ const NAMES: Record = { export const DemographicEntryFactory = FactoryGirl.define(DemographicEntry, async () => { const type = faker.helpers.arrayElement(TYPES); return { - uuid: crypto.randomUUID(), demographicId: DemographicFactory.forProjectReportWorkday.associate("id"), type, subtype: faker.helpers.arrayElement(SUBTYPES[type] ?? [null]), diff --git a/libs/database/src/lib/factories/demographic.factory.ts b/libs/database/src/lib/factories/demographic.factory.ts index cefd7e16..2ddd50db 100644 --- a/libs/database/src/lib/factories/demographic.factory.ts +++ b/libs/database/src/lib/factories/demographic.factory.ts @@ -16,7 +16,6 @@ import { ProjectPitchFactory } from "./project-pitch.factory"; import { ProjectFactory } from "./project.factory"; const defaultAttributesFactory = async () => ({ - uuid: crypto.randomUUID(), description: null, hidden: false }); diff --git a/libs/database/src/lib/factories/form-submission.factory.ts b/libs/database/src/lib/factories/form-submission.factory.ts index dfb3b2b4..4baf4211 100644 --- a/libs/database/src/lib/factories/form-submission.factory.ts +++ b/libs/database/src/lib/factories/form-submission.factory.ts @@ -5,7 +5,6 @@ import { ApplicationFactory } from "./application.factory"; import { FORM_SUBMISSION_STATUSES } from "../constants/status"; export const FormSubmissionFactory = FactoryGirl.define(FormSubmission, async () => ({ - uuid: crypto.randomUUID(), applicationId: ApplicationFactory.associate("id"), name: faker.animal.petName(), status: faker.helpers.arrayElement(FORM_SUBMISSION_STATUSES) diff --git a/libs/database/src/lib/factories/funding-programme.factory.ts b/libs/database/src/lib/factories/funding-programme.factory.ts index e2243b5a..30875d70 100644 --- a/libs/database/src/lib/factories/funding-programme.factory.ts +++ b/libs/database/src/lib/factories/funding-programme.factory.ts @@ -3,6 +3,5 @@ import { FundingProgramme } from "../entities"; import { faker } from "@faker-js/faker"; export const FundingProgrammeFactory = FactoryGirl.define(FundingProgramme, async () => ({ - uuid: crypto.randomUUID(), name: faker.animal.petName() })); diff --git a/libs/database/src/lib/factories/media.factory.ts b/libs/database/src/lib/factories/media.factory.ts index 1ccb47ae..f6470b38 100644 --- a/libs/database/src/lib/factories/media.factory.ts +++ b/libs/database/src/lib/factories/media.factory.ts @@ -4,7 +4,6 @@ import { ProjectFactory } from "./project.factory"; import { faker } from "@faker-js/faker"; const defaultAttributesFactory = async () => ({ - uuid: crypto.randomUUID(), collectionName: faker.lorem.words(1), name: faker.lorem.words(2), fileName: `${faker.lorem.words(1)}.jpg`, diff --git a/libs/database/src/lib/factories/nursery-report.factory.ts b/libs/database/src/lib/factories/nursery-report.factory.ts index bd40ced9..3f8b32fa 100644 --- a/libs/database/src/lib/factories/nursery-report.factory.ts +++ b/libs/database/src/lib/factories/nursery-report.factory.ts @@ -9,7 +9,6 @@ export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () = const dueAt = faker.date.past({ years: 2 }); dueAt.setMilliseconds(0); return { - uuid: crypto.randomUUID(), nurseryId: NurseryFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), diff --git a/libs/database/src/lib/factories/nursery.factory.ts b/libs/database/src/lib/factories/nursery.factory.ts index 58eaeb89..6f64ea48 100644 --- a/libs/database/src/lib/factories/nursery.factory.ts +++ b/libs/database/src/lib/factories/nursery.factory.ts @@ -5,7 +5,6 @@ import { faker } from "@faker-js/faker"; import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; export const NurseryFactory = FactoryGirl.define(Nursery, async () => ({ - uuid: crypto.randomUUID(), projectId: ProjectFactory.associate("id"), name: faker.animal.petName(), status: faker.helpers.arrayElement(ENTITY_STATUSES), diff --git a/libs/database/src/lib/factories/organisation.factory.ts b/libs/database/src/lib/factories/organisation.factory.ts index adf0beeb..b9dfdc6b 100644 --- a/libs/database/src/lib/factories/organisation.factory.ts +++ b/libs/database/src/lib/factories/organisation.factory.ts @@ -4,7 +4,6 @@ import { faker } from "@faker-js/faker"; import { ORGANISATION_STATUSES } from "../constants/status"; export const OrganisationFactory = FactoryGirl.define(Organisation, async () => ({ - uuid: crypto.randomUUID(), status: faker.helpers.arrayElement(ORGANISATION_STATUSES), type: "non-profit-organisation", name: faker.company.name() diff --git a/libs/database/src/lib/factories/polygon-geometry.factory.ts b/libs/database/src/lib/factories/polygon-geometry.factory.ts index 268fe3c9..e94c9d06 100644 --- a/libs/database/src/lib/factories/polygon-geometry.factory.ts +++ b/libs/database/src/lib/factories/polygon-geometry.factory.ts @@ -16,7 +16,6 @@ export const POLYGON = { }; export const PolygonGeometryFactory = FactoryGirl.define(PolygonGeometry, async () => ({ - uuid: crypto.randomUUID(), polygon: POLYGON, createdBy: UserFactory.associate("id") })); diff --git a/libs/database/src/lib/factories/project-pitch.factory.ts b/libs/database/src/lib/factories/project-pitch.factory.ts index 1e85a690..f8c12dc0 100644 --- a/libs/database/src/lib/factories/project-pitch.factory.ts +++ b/libs/database/src/lib/factories/project-pitch.factory.ts @@ -1,6 +1,4 @@ import { FactoryGirl } from "factory-girl-ts"; import { ProjectPitch } from "../entities"; -export const ProjectPitchFactory = FactoryGirl.define(ProjectPitch, async () => ({ - uuid: crypto.randomUUID() -})); +export const ProjectPitchFactory = FactoryGirl.define(ProjectPitch, async () => ({})); diff --git a/libs/database/src/lib/factories/project-report.factory.ts b/libs/database/src/lib/factories/project-report.factory.ts index 4dcc8be9..39b08acb 100644 --- a/libs/database/src/lib/factories/project-report.factory.ts +++ b/libs/database/src/lib/factories/project-report.factory.ts @@ -10,7 +10,6 @@ export const ProjectReportFactory = FactoryGirl.define(ProjectReport, async () = const dueAt = faker.date.past({ years: 2 }); dueAt.setMilliseconds(0); return { - uuid: crypto.randomUUID(), projectId: ProjectFactory.associate("id"), frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), dueAt, diff --git a/libs/database/src/lib/factories/project.factory.ts b/libs/database/src/lib/factories/project.factory.ts index 6311f914..93b3cf77 100644 --- a/libs/database/src/lib/factories/project.factory.ts +++ b/libs/database/src/lib/factories/project.factory.ts @@ -9,7 +9,6 @@ import { FRAMEWORK_KEYS } from "../constants/framework"; const CONTINENTS = ["africa", "australia", "south-america", "asia", "north-america"]; export const ProjectFactory = FactoryGirl.define(Project, async () => ({ - uuid: crypto.randomUUID(), name: faker.animal.petName(), frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), status: faker.helpers.arrayElement(ENTITY_STATUSES), diff --git a/libs/database/src/lib/factories/seeding.factory.ts b/libs/database/src/lib/factories/seeding.factory.ts index f354c852..8f78bc20 100644 --- a/libs/database/src/lib/factories/seeding.factory.ts +++ b/libs/database/src/lib/factories/seeding.factory.ts @@ -5,7 +5,6 @@ import { SiteReportFactory } from "./site-report.factory"; import { SiteFactory } from "./site.factory"; const defaultAttributesFactory = async () => ({ - uuid: crypto.randomUUID(), name: faker.lorem.words(2), amount: faker.number.int({ min: 10, max: 1000 }), seedsInSample: faker.number.int({ min: 10, max: 1000 }), diff --git a/libs/database/src/lib/factories/site-report.factory.ts b/libs/database/src/lib/factories/site-report.factory.ts index 27e18881..17b13bf1 100644 --- a/libs/database/src/lib/factories/site-report.factory.ts +++ b/libs/database/src/lib/factories/site-report.factory.ts @@ -9,7 +9,6 @@ export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { const dueAt = faker.date.past({ years: 2 }); dueAt.setMilliseconds(0); return { - uuid: crypto.randomUUID(), siteId: SiteFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), diff --git a/libs/database/src/lib/factories/site.factory.ts b/libs/database/src/lib/factories/site.factory.ts index 84579744..8766a2a7 100644 --- a/libs/database/src/lib/factories/site.factory.ts +++ b/libs/database/src/lib/factories/site.factory.ts @@ -6,7 +6,6 @@ import { SITE_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; import { SITING_STRATEGIES } from "../constants/entity-selects"; export const SiteFactory = FactoryGirl.define(Site, async () => ({ - uuid: crypto.randomUUID(), projectId: ProjectFactory.associate("id"), name: faker.animal.petName(), status: faker.helpers.arrayElement(SITE_STATUSES), diff --git a/libs/database/src/lib/factories/task.factory.ts b/libs/database/src/lib/factories/task.factory.ts index dd97a7fb..2d816be1 100644 --- a/libs/database/src/lib/factories/task.factory.ts +++ b/libs/database/src/lib/factories/task.factory.ts @@ -1,8 +1,4 @@ import { FactoryGirl } from "factory-girl-ts"; import { Task } from "../entities"; -export const TaskFactory = FactoryGirl.define(Task, async () => { - return { - uuid: crypto.randomUUID() - }; -}); +export const TaskFactory = FactoryGirl.define(Task, async () => ({})); diff --git a/libs/database/src/lib/factories/tree-species.factory.ts b/libs/database/src/lib/factories/tree-species.factory.ts index 2bc46dd3..02c9b501 100644 --- a/libs/database/src/lib/factories/tree-species.factory.ts +++ b/libs/database/src/lib/factories/tree-species.factory.ts @@ -9,7 +9,6 @@ import { ProjectReportFactory } from "./project-report.factory"; import { NurseryFactory } from "./nursery.factory"; const defaultAttributesFactory = async () => ({ - uuid: crypto.randomUUID(), name: faker.lorem.words(2), taxonId: null, amount: faker.number.int({ min: 10, max: 1000 }), diff --git a/libs/database/src/lib/factories/user.factory.ts b/libs/database/src/lib/factories/user.factory.ts index 9e99f534..759d7ef1 100644 --- a/libs/database/src/lib/factories/user.factory.ts +++ b/libs/database/src/lib/factories/user.factory.ts @@ -5,7 +5,6 @@ import { User } from "../entities"; // TODO: generate correctly hashed passwords. This will be easily accomplished once user signup // has been implemented in this codebase. export const UserFactory = FactoryGirl.define(User, async () => ({ - uuid: crypto.randomUUID(), firstName: faker.person.firstName(), lastName: faker.person.lastName(), emailAddress: await generateUniqueEmail(), From 427ca569164347ba19ebdff73c79f96536b74f22 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 9 Apr 2025 21:47:34 -0700 Subject: [PATCH 19/98] [TM-1861] Spec out the audit statuses DB table. --- .../src/lib/entities/audit-status.entity.ts | 72 +++++++++++++++++++ libs/database/src/lib/entities/index.ts | 1 + 2 files changed, 73 insertions(+) create mode 100644 libs/database/src/lib/entities/audit-status.entity.ts diff --git a/libs/database/src/lib/entities/audit-status.entity.ts b/libs/database/src/lib/entities/audit-status.entity.ts new file mode 100644 index 00000000..371cc362 --- /dev/null +++ b/libs/database/src/lib/entities/audit-status.entity.ts @@ -0,0 +1,72 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, BOOLEAN, DATE, ENUM, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; +import { User } from "./user.entity"; + +@Table({ + tableName: "audit_statuses", + underscored: true, + paranoid: true, + // @Index doesn't work with underscored column names in all contexts + indexes: [{ name: "audit_statuses_auditable_type_auditable_id_index", fields: ["auditable_type", "auditable_id"] }] +}) +export class AuditStatus extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column({ type: UUID, defaultValue: UUIDV4 }) + uuid: string; + + @AllowNull + @Column(STRING) + status: string; + + @AllowNull + @Column(TEXT) + comment: string; + + @AllowNull + @Column(STRING) + firstName: string; + + @AllowNull + @Column(STRING) + lastName: string; + + @AllowNull + @Column({ + type: ENUM, + values: ["change-request", "status", "submission", "comment", "change-request-updated", "reminder-sent"] + }) + type: string; + + @AllowNull + @Column(BOOLEAN) + isSubmitted: boolean; + + @AllowNull + @Column(BOOLEAN) + isActive: boolean; + + @AllowNull + @Column(BOOLEAN) + requestRemoved: boolean; + + /** @deprecated */ + @AllowNull + @Column(DATE) + dateCreated: Date; + + @AllowNull + @ForeignKey(() => User) + @Column(BIGINT.UNSIGNED) + createdBy: number | null; + + @Column(STRING) + auditableType: string; + + @Column(BIGINT.UNSIGNED) + auditableId: number; +} diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 26313fa5..9aaa1aa5 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -1,4 +1,5 @@ export * from "./action.entity"; +export * from "./audit-status.entity"; export * from "./application.entity"; export * from "./delayed-job.entity"; export * from "./demographic.entity"; From 3bacc4d2bf8a79828630beb1b0b3fc0c706ea5d6 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 10:01:36 -0700 Subject: [PATCH 20/98] [TM-1861] Avoid multiple EmailProcessor classes. --- .../src/lib/email/{email-processor.ts => email-sender.ts} | 2 +- libs/common/src/lib/email/entity-status-update.email.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename libs/common/src/lib/email/{email-processor.ts => email-sender.ts} (74%) diff --git a/libs/common/src/lib/email/email-processor.ts b/libs/common/src/lib/email/email-sender.ts similarity index 74% rename from libs/common/src/lib/email/email-processor.ts rename to libs/common/src/lib/email/email-sender.ts index 226f1b7f..fd5ba97d 100644 --- a/libs/common/src/lib/email/email-processor.ts +++ b/libs/common/src/lib/email/email-sender.ts @@ -1,5 +1,5 @@ import { EmailService } from "./email.service"; -export abstract class EmailProcessor { +export abstract class EmailSender { abstract send(emailService: EmailService): Promise; } diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index 432827bd..ef0f8504 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -1,4 +1,4 @@ -import { EmailProcessor } from "./email-processor"; +import { EmailSender } from "./email-sender"; import { EmailService } from "./email.service"; import { StatusUpdateData } from "./email.processor"; import { @@ -23,7 +23,7 @@ import { TMLogger } from "../util/tm-logger"; import { InternalServerErrorException } from "@nestjs/common"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; -export class EntityStatusUpdateEmail extends EmailProcessor { +export class EntityStatusUpdateEmail extends EmailSender { private readonly logger = new TMLogger(EntityStatusUpdateEmail.name); private readonly type: EntityType; From 03865417d32b59fc741b62e3a4a46bddb8866374 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 14:43:41 -0700 Subject: [PATCH 21/98] [TM-1861] Finish implementing side effects for project status update. --- .../src/entities/entities.service.ts | 2 +- .../processors/association-processor.ts | 4 +- .../entities/processors/entity-processor.ts | 4 +- .../entity-status-update.event-processor.ts | 128 ++++++++++++++++++ libs/common/src/lib/events/event.processor.ts | 22 +++ libs/common/src/lib/events/event.service.ts | 26 +--- libs/database/src/lib/constants/entities.ts | 7 + libs/database/src/lib/constants/status.ts | 34 ++++- .../src/lib/entities/action.entity.ts | 19 ++- .../src/lib/entities/audit-status.entity.ts | 62 ++++++--- .../src/lib/entities/form-question.entity.ts | 120 ++++++++++++++++ libs/database/src/lib/entities/index.ts | 1 + .../database/src/lib/entities/media.entity.ts | 4 +- libs/database/src/lib/types/util.ts | 6 + 14 files changed, 377 insertions(+), 62 deletions(-) create mode 100644 libs/common/src/lib/events/entity-status-update.event-processor.ts create mode 100644 libs/common/src/lib/events/event.processor.ts create mode 100644 libs/database/src/lib/entities/form-question.entity.ts diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index f2295e8d..1bb4c180 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -101,7 +101,7 @@ export class EntitiesService { return new processorClass(this, entity) as unknown as EntityProcessor; } - createAssociationProcessor, D extends AssociationDto>( + createAssociationProcessor>( entityType: EntityType, uuid: string, association: ProcessableAssociation diff --git a/apps/entity-service/src/entities/processors/association-processor.ts b/apps/entity-service/src/entities/processors/association-processor.ts index 1369a730..b042d1fe 100644 --- a/apps/entity-service/src/entities/processors/association-processor.ts +++ b/apps/entity-service/src/entities/processors/association-processor.ts @@ -5,7 +5,7 @@ import { EntityClass, EntityModel, EntityType } from "@terramatch-microservices/ import { intersection } from "lodash"; import { UuidModel } from "@terramatch-microservices/database/types/util"; -export abstract class AssociationProcessor, D extends AssociationDto> { +export abstract class AssociationProcessor> { abstract readonly DTO: Type; constructor( @@ -19,7 +19,7 @@ export abstract class AssociationProcessor, D extends Ass * are simple enough that providing a reference to the DTO class, and a getter of associations based on the * base entity is enough. */ - static buildSimpleProcessor, D extends AssociationDto>( + static buildSimpleProcessor>( dtoClass: Type, associationGetter: (entity: EntityModel, entityLaravelType: string) => Promise ) { diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index f8383b5a..c7a7cc1e 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -5,7 +5,7 @@ import { EntitiesService, ProcessableEntity } from "../entities.service"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { BadRequestException, Type } from "@nestjs/common"; import { EntityDto } from "../dto/entity.dto"; -import { EntityClass, EntityModel } from "@terramatch-microservices/database/constants/entities"; +import { EntityModel } from "@terramatch-microservices/database/constants/entities"; import { Action } from "@terramatch-microservices/database/entities/action.entity"; import { EntityUpdateData } from "../dto/entity-update.dto"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; @@ -107,7 +107,7 @@ export abstract class EntityProcessor< } async delete(model: ModelType) { - await Action.targetable((model.constructor as EntityClass).LARAVEL_TYPE, model.id).destroy(); + await Action.for(model).destroy(); await model.destroy(); } diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts new file mode 100644 index 00000000..cc8faaea --- /dev/null +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -0,0 +1,128 @@ +import { FeedbackModel, LaravelModel, laravelType, StatusModel } from "@terramatch-microservices/database/types/util"; +import { EventProcessor } from "./event.processor"; +import { TMLogger } from "../util/tm-logger"; +import { + ENTITY_MODELS, + EntityModel, + EntityType, + getOrganisationId, + getProjectId, + isReport +} from "@terramatch-microservices/database/constants/entities"; +import { StatusUpdateData } from "../email/email.processor"; +import { EventService } from "./event.service"; +import { Action, AuditStatus, FormQuestion } from "@terramatch-microservices/database/entities"; +import { get, isEmpty, map } from "lodash"; +import { Op } from "sequelize"; +import { STATUS_DISPLAY_STRINGS } from "@terramatch-microservices/database/constants/status"; + +export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; + +export class EntityStatusUpdate extends EventProcessor { + private readonly logger = new TMLogger(EntityStatusUpdate.name); + + constructor(eventService: EventService, private readonly model: StatusUpdateModel) { + super(eventService); + } + + async handle() { + this.logger.log( + `Received model status update [${JSON.stringify({ + type: this.model.constructor.name, + id: this.model.id, + status: this.model.status + })}]` + ); + + const entityType = Object.entries(ENTITY_MODELS).find( + ([, entityClass]) => this.model instanceof entityClass + )?.[0] as EntityType | undefined; + + if (entityType != null) { + await this.sendStatusUpdateEmail(entityType); + await this.updateActions(); + } + await this.createAuditStatus(); + } + + private async sendStatusUpdateEmail(type: EntityType) { + this.logger.log(`Sending status update to email queue [${JSON.stringify({ type, id: this.model.id })}]`); + await this.eventService.emailQueue.add("statusUpdate", { type, id: this.model.id } as StatusUpdateData); + } + + private async updateActions() { + this.logger.log(`Updating actions [${JSON.stringify({ model: this.model.constructor.name, id: this.model.id })}]`); + const entity = this.model as EntityModel; + await Action.for(entity).destroy({ where: { type: "notification" } }); + + if (entity.status !== "awaiting-approval") { + const action = new Action(); + action.status = "pending"; + action.targetableType = laravelType(entity); + action.targetableId = entity.id; + action.type = "notification"; + action.projectId = (await getProjectId(entity)) ?? null; + action.organisationId = (await getOrganisationId(entity)) ?? null; + + if (!isReport(entity)) { + action.title = get(entity, "name") ?? ""; + action.text = STATUS_DISPLAY_STRINGS[entity.status]; + } + + await action.save(); + } + } + + private async createAuditStatus() { + this.logger.log( + `Creating auditStatus [${JSON.stringify({ model: this.model.constructor.name, id: this.model.id })}]` + ); + + const user = await this.getAuthenticatedUser(); + const auditStatus = new AuditStatus(); + auditStatus.auditableType = laravelType(this.model); + auditStatus.auditableId = this.model.id; + auditStatus.createdBy = user?.emailAddress ?? null; + auditStatus.firstName = user?.firstName ?? null; + auditStatus.lastName = user?.lastName ?? null; + + // TODO: the update is different for UpdateRequest awaiting approval + if (this.model.status === "approved") { + auditStatus.comment = `Approved: ${this.model.feedback}`; + } else if (this.model.status === "restoration-in-progress") { + auditStatus.comment = "Restoration In Progress"; + } else if (this.model.status === "needs-more-information") { + auditStatus.type = "change-request"; + auditStatus.comment = await this.getNeedsMoreInfoComment(); + } else if (this.model.status === "awaiting-approval") { + // NOOP, but avoid the short circuit warning below. + } else { + this.logger.warn( + `Skipping audit status for entity status [${JSON.stringify({ + model: this.model.constructor.name, + id: this.model.id, + status: this.model.status + })}]` + ); + return; + } + + await auditStatus.save(); + } + + private async getNeedsMoreInfoComment() { + const { feedback, feedbackFields } = this.model; + const labels = map( + isEmpty(feedbackFields) + ? [] + : await FormQuestion.findAll({ + where: { uuid: { [Op.in]: this.model.feedbackFields } }, + attributes: ["label"] + }), + "label" + ); + return `Request More Information on the following fields: ${labels.join(", ")}. Feedback: ${ + feedback ?? "(No feedback)" + }`; + } +} diff --git a/libs/common/src/lib/events/event.processor.ts b/libs/common/src/lib/events/event.processor.ts new file mode 100644 index 00000000..8a04f4f5 --- /dev/null +++ b/libs/common/src/lib/events/event.processor.ts @@ -0,0 +1,22 @@ +import { User } from "@terramatch-microservices/database/entities"; +import { RequestContext } from "nestjs-request-context"; +import { EventService } from "./event.service"; + +export abstract class EventProcessor { + constructor(protected readonly eventService: EventService) {} + + private _authenticatedUser?: User | null; + async getAuthenticatedUser() { + if (this._authenticatedUser != null) return this._authenticatedUser; + + const userId = RequestContext.currentContext.req.authenticatedUserId as number | null; + if (userId == null) return null; + + return (this._authenticatedUser = await User.findOne({ + where: { id: userId }, + attributes: ["id", "emailAddress", "firstName", "lastName"] + })); + } + + abstract handle(): Promise; +} diff --git a/libs/common/src/lib/events/event.service.ts b/libs/common/src/lib/events/event.service.ts index d7421569..751a41d3 100644 --- a/libs/common/src/lib/events/event.service.ts +++ b/libs/common/src/lib/events/event.service.ts @@ -1,11 +1,8 @@ import { Injectable } from "@nestjs/common"; import { OnEvent } from "@nestjs/event-emitter"; -import { Model } from "sequelize-typescript"; import { InjectQueue } from "@nestjs/bullmq"; -import { TMLogger } from "../util/tm-logger"; import { Queue } from "bullmq"; -import { ENTITY_MODELS } from "@terramatch-microservices/database/constants/entities"; -import { StatusUpdateData } from "../email/email.processor"; +import { EntityStatusUpdate, StatusUpdateModel } from "./entity-status-update.event-processor"; /** * A service to handle general events that are emitted in the common or database libraries, and @@ -13,25 +10,10 @@ import { StatusUpdateData } from "../email/email.processor"; */ @Injectable() export class EventService { - private readonly logger = new TMLogger(EventService.name); - - constructor(@InjectQueue("email") private readonly emailQueue: Queue) {} + constructor(@InjectQueue("email") readonly emailQueue: Queue) {} @OnEvent("database.statusUpdated") - async handleStatusUpdated(model: Model & { status: string }) { - this.logger.log("Received model status update", { - type: model.constructor.name, - id: model.id, - status: model.status - }); - - const type = Object.entries(ENTITY_MODELS).find(([, entityClass]) => model instanceof entityClass)?.[0]; - if (type == null) { - this.logger.error("Status update not an entity model", { type: model.constructor.name }); - return; - } - - this.logger.log("Sending status update to email queue", { type, id: model.id }); - await this.emailQueue.add("statusUpdate", { type, id: model.id } as StatusUpdateData); + async handleStatusUpdated(model: StatusUpdateModel) { + await new EntityStatusUpdate(this, model).handle(); } } diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 156b32c2..97f92b30 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -45,6 +45,13 @@ export async function getProjectId(entity: EntityModel) { return (await parentClass.findOne({ where: { id: parentId }, attributes: ["projectId"] }))?.projectId; } +export async function getOrganisationId(entity: EntityModel) { + if (entity instanceof Project) return entity.organisationId; + + return (await Project.findOne({ where: { id: await getProjectId(entity) }, attributes: ["organisationId"] })) + ?.organisationId; +} + export function getViewLinkPath(entity: EntityModel) { const prefix = isReport(entity) ? "/reports/" : "/"; return `${prefix}${kebabCase(entity.constructor.name)}/${entity.uuid}`; diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 52339b52..2ac085ab 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -58,5 +58,37 @@ export const FORM_SUBMISSION_STATUSES = [ export type FormSubmissionStatus = (typeof FORM_SUBMISSION_STATUSES)[number]; export const PENDING = "pending"; -export const ORGANISATION_STATUSES = [APPROVED, PENDING, REJECTED, DRAFT]; +export const ORGANISATION_STATUSES = [APPROVED, PENDING, REJECTED, DRAFT] as const; export type OrganisationStatus = (typeof ORGANISATION_STATUSES)[number]; + +type AllStatuses = + | EntityStatus + | SiteStatus + | ReportStatus + | UpdateRequestStatus + | FormSubmissionStatus + | OrganisationStatus; + +/** + * A mapping of all statuses to an English language display string for that status. + * + * Note: Please do not send this value to the client directly. The client should be responsible + * for managing (and translating) these display strings itself. This is used to support some legacy + * systems (like Actions) that require a display string for a status to be embedded in the DB. + * + * Ideally we fix up and remove those needs over time, and eventually git rid of this structure from + * BE code. + */ +export const STATUS_DISPLAY_STRINGS: Record = { + [DRAFT]: "Draft", + [DUE]: "Due", + [PENDING]: "Pending", + [STARTED]: "Started", + [AWAITING_APPROVAL]: "Awaiting approval", + [NEEDS_MORE_INFORMATION]: "Needs more information", + [APPROVED]: "Approved", + [RESTORATION_IN_PROGRESS]: "Restoration in progress", + [REJECTED]: "Rejected", + [NO_UPDATE]: "No update", + [REQUIRES_MORE_INFORMATION]: "Requires more information" +}; diff --git a/libs/database/src/lib/entities/action.entity.ts b/libs/database/src/lib/entities/action.entity.ts index d81afad0..e643c893 100644 --- a/libs/database/src/lib/entities/action.entity.ts +++ b/libs/database/src/lib/entities/action.entity.ts @@ -13,15 +13,14 @@ import { import { BIGINT, STRING, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { Project } from "./project.entity"; -import { Literal } from "sequelize/types/utils"; -import { isNumber } from "lodash"; import { chainScope } from "../util/chain-scope"; +import { LaravelModel, laravelType } from "../types/util"; @Scopes(() => ({ - targetable: (laravelType: string, ids: number[] | Literal) => ({ + targetable: (targetable: LaravelModel) => ({ where: { - targetableType: laravelType, - targetableId: ids + targetableType: laravelType(targetable), + targetableId: targetable.id } }) })) @@ -33,9 +32,8 @@ import { chainScope } from "../util/chain-scope"; indexes: [{ name: "v2_actions_targetable_type_targetable_id_index", fields: ["targetable_type", "targetable_id"] }] }) export class Action extends Model { - static targetable(laravelType: string, ids: number | number[] | Literal) { - if (isNumber(ids)) ids = [ids]; - return chainScope(this, "targetable", laravelType, ids) as typeof Action; + static for(targetable: LaravelModel) { + return chainScope(this, "targetable", targetable) as typeof Action; } @PrimaryKey @@ -57,12 +55,13 @@ export class Action extends Model { @Column(BIGINT.UNSIGNED) targetableId: number; + @AllowNull @ForeignKey(() => Organisation) @Column(BIGINT.UNSIGNED) - organisationId: number; + organisationId: number | null; @BelongsTo(() => Organisation) - organisation?: Organisation; + organisation?: Organisation | null; @AllowNull @ForeignKey(() => Project) diff --git a/libs/database/src/lib/entities/audit-status.entity.ts b/libs/database/src/lib/entities/audit-status.entity.ts index 371cc362..2175d559 100644 --- a/libs/database/src/lib/entities/audit-status.entity.ts +++ b/libs/database/src/lib/entities/audit-status.entity.ts @@ -1,6 +1,8 @@ -import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, ENUM, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; -import { User } from "./user.entity"; +import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, BOOLEAN, DATE, ENUM, NOW, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; + +const TYPES = ["change-request", "status", "submission", "comment", "change-request-updated", "reminder-sent"] as const; +type AuditStatusType = (typeof TYPES)[number]; @Table({ tableName: "audit_statuses", @@ -21,48 +23,64 @@ export class AuditStatus extends Model { @AllowNull @Column(STRING) - status: string; + status: string | null; @AllowNull @Column(TEXT) - comment: string; + comment: string | null; @AllowNull @Column(STRING) - firstName: string; + firstName: string | null; @AllowNull @Column(STRING) - lastName: string; + lastName: string | null; @AllowNull - @Column({ - type: ENUM, - values: ["change-request", "status", "submission", "comment", "change-request-updated", "reminder-sent"] - }) - type: string; + @Column({ type: ENUM, values: TYPES }) + type: AuditStatusType | null; + /** + * @deprecated + * + * All records in the DB have null for this field, so it seems not to be useful. + */ @AllowNull @Column(BOOLEAN) - isSubmitted: boolean; + isSubmitted: boolean | null; + /** + * @deprecated + * + * All records in the DB have true for this field, so it seems not to be useful. + */ @AllowNull - @Column(BOOLEAN) - isActive: boolean; + @Column({ type: BOOLEAN, defaultValue: true }) + isActive: boolean | null; + /** + * @deprecated + * + * All records in the DB have null for this field, so it seems not to be useful. + */ @AllowNull @Column(BOOLEAN) - requestRemoved: boolean; + requestRemoved: boolean | null; - /** @deprecated */ + /** + * @deprecated + * + * Needs an investigation to see if this field is being used anywhere, but it is totally superfluous with the + * automatic createdAt field. + **/ @AllowNull - @Column(DATE) - dateCreated: Date; + @Column({ type: DATE, defaultValue: NOW }) + dateCreated: Date | null; @AllowNull - @ForeignKey(() => User) - @Column(BIGINT.UNSIGNED) - createdBy: number | null; + @Column(STRING) + createdBy: string | null; @Column(STRING) auditableType: string; diff --git a/libs/database/src/lib/entities/form-question.entity.ts b/libs/database/src/lib/entities/form-question.entity.ts new file mode 100644 index 00000000..8c71243a --- /dev/null +++ b/libs/database/src/lib/entities/form-question.entity.ts @@ -0,0 +1,120 @@ +import { AllowNull, AutoIncrement, BelongsTo, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, BOOLEAN, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { I18nItem } from "./i18n-item.entity"; +import { JsonColumn } from "../decorators/json-column.decorator"; + +@Table({ tableName: "form_questions", underscored: true, paranoid: true }) +export class FormQuestion extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column({ type: UUID, defaultValue: UUIDV4 }) + uuid: string; + + // TODO: foreign key for FormSection, when that's been added. Uses cascading delete constraint + @Column(BIGINT.UNSIGNED) + formSectionId: number; + + // TODO: foreign key on Form UUID + @AllowNull + @Column(UUID) + parentId: string; + + @AllowNull + @Column(STRING) + linkedFieldKey: string | null; + + @Column(STRING) + inputType: string; + + @AllowNull + @Column(STRING) + name: string | null; + + @Column(TEXT) + label: string; + + @AllowNull + @Column(INTEGER) + labelId: number | null; + + @BelongsTo(() => I18nItem, { foreignKey: "label_id", constraints: false }) + labelI18nItem: I18nItem | null; + + @AllowNull + @Column(TEXT) + description: string | null; + + @AllowNull + @Column(INTEGER) + descriptionId: number | null; + + @BelongsTo(() => I18nItem, { foreignKey: "description_id", constraints: false }) + descriptionI18nItem: I18nItem | null; + + @AllowNull + @Column(STRING) + placeholder: string | null; + + @AllowNull + @Column(INTEGER) + placeholderId: number | null; + + @BelongsTo(() => I18nItem, { foreignKey: "placeholder_id", constraints: false }) + placeholderI18nItem: I18nItem | null; + + @AllowNull + @Column(STRING) + optionsList: string | null; + + @Column({ type: BOOLEAN, field: "multichoice", defaultValue: false }) + multiChoice: boolean; + + @AllowNull + @Column(STRING) + collection: string | null; + + @Column(TINYINT) + order: number; + + @AllowNull + @JsonColumn() + additionalProps: object | null; + + @AllowNull + @Column(TEXT("tiny")) + additionalText: string | null; + + @AllowNull + @Column(STRING) + additionalUrl: string | null; + + @AllowNull + @Column(BOOLEAN) + showOnParentCondition: boolean | null; + + @AllowNull + @JsonColumn() + validation: object | null; + + @AllowNull + @Column({ type: BOOLEAN, defaultValue: false }) + optionsOther: boolean | null; + + @Column({ type: BOOLEAN, defaultValue: true }) + conditionalDefault: boolean; + + @Column({ type: BOOLEAN, defaultValue: false }) + isParentConditionalDefault: boolean; + + @AllowNull + @Column({ type: INTEGER.UNSIGNED, defaultValue: 0 }) + minCharacterLimit: number | null; + + @AllowNull + @Column({ type: INTEGER.UNSIGNED, defaultValue: 90000 }) + maxCharacterLimit: number | null; +} diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 9aaa1aa5..9f34bc96 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -5,6 +5,7 @@ export * from "./delayed-job.entity"; export * from "./demographic.entity"; export * from "./demographic-entry.entity"; export * from "./form-submission.entity"; +export * from "./form-question.entity"; export * from "./framework.entity"; export * from "./framework-user.entity"; export * from "./funding-programme.entity"; diff --git a/libs/database/src/lib/entities/media.entity.ts b/libs/database/src/lib/entities/media.entity.ts index 13641d1e..f8c256b7 100644 --- a/libs/database/src/lib/entities/media.entity.ts +++ b/libs/database/src/lib/entities/media.entity.ts @@ -14,14 +14,14 @@ import { BIGINT, BOOLEAN, DOUBLE, ENUM, INTEGER, STRING, UUID, UUIDV4 } from "se import { JsonColumn } from "../decorators/json-column.decorator"; import { User } from "./user.entity"; import { chainScope } from "../util/chain-scope"; -import { LaravelModel, LaravelModelCtor } from "../types/util"; +import { LaravelModel, laravelType } from "../types/util"; @DefaultScope(() => ({ order: ["orderColumn"] })) @Scopes(() => ({ collection: (collectionName: string) => ({ where: { collectionName } }), association: (association: LaravelModel) => ({ where: { - modelType: (association.constructor as LaravelModelCtor).LARAVEL_TYPE, + modelType: laravelType(association), modelId: association.id } }) diff --git a/libs/database/src/lib/types/util.ts b/libs/database/src/lib/types/util.ts index 0ea63756..3ec6d35b 100644 --- a/libs/database/src/lib/types/util.ts +++ b/libs/database/src/lib/types/util.ts @@ -4,3 +4,9 @@ export type UuidModel = Model & { uuid: string }; export type LaravelModelCtor = ModelCtor & { LARAVEL_TYPE: string }; export type LaravelModel = InstanceType; + +export const laravelType = (model: LaravelModel) => (model.constructor as LaravelModelCtor).LARAVEL_TYPE; + +export type StatusModel = Model & { status: string }; + +export type FeedbackModel = Model & { feedback?: string; feedbackFields?: string[] }; From f7350e62e4325a3336056f482811a4bac1f57776 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 21:37:00 -0700 Subject: [PATCH 22/98] [TM-1861] Expose the set of supported entities as a const to the FE. --- apps/entity-service/src/entities/dto/entity.dto.ts | 8 ++++++++ apps/entity-service/src/entities/entities.controller.ts | 3 ++- libs/common/src/lib/events/event.processor.ts | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity.dto.ts b/apps/entity-service/src/entities/dto/entity.dto.ts index aa8a898c..43ebbc41 100644 --- a/apps/entity-service/src/entities/dto/entity.dto.ts +++ b/apps/entity-service/src/entities/dto/entity.dto.ts @@ -1,5 +1,7 @@ import { ApiProperty } from "@nestjs/swagger"; import { HybridSupportDto } from "@terramatch-microservices/common/dto/hybrid-support.dto"; +import { JsonApiConstants } from "@terramatch-microservices/common/decorators/json-api-constants.decorator"; +import { ENTITY_TYPES } from "@terramatch-microservices/database/constants/entities"; /** * A utility type for constructing the "extra props" type of a DTO based on what's in the dto, the @@ -7,6 +9,12 @@ import { HybridSupportDto } from "@terramatch-microservices/common/dto/hybrid-su */ export type AdditionalProps = Pick>; +@JsonApiConstants +export class SupportedEntities { + @ApiProperty({ example: ENTITY_TYPES }) + ENTITY_TYPES: string[]; +} + export abstract class EntityDto extends HybridSupportDto { /** * All EntityDtos must include UUID in the attributes for use in the react-admin pagination diff --git a/apps/entity-service/src/entities/entities.controller.ts b/apps/entity-service/src/entities/entities.controller.ts index 7ecb4566..169e3cfd 100644 --- a/apps/entity-service/src/entities/entities.controller.ts +++ b/apps/entity-service/src/entities/entities.controller.ts @@ -28,9 +28,10 @@ import { JsonApiDeletedResponse } from "@terramatch-microservices/common/decorat import { NurseryReportFullDto, NurseryReportLightDto } from "./dto/nursery-report.dto"; import { SiteReportFullDto, SiteReportLightDto } from "./dto/site-report.dto"; import { EntityUpdateBody } from "./dto/entity-update.dto"; +import { SupportedEntities } from "./dto/entity.dto"; @Controller("entities/v3") -@ApiExtraModels(ANRDto, ProjectApplicationDto, MediaDto, EntitySideload) +@ApiExtraModels(ANRDto, ProjectApplicationDto, MediaDto, EntitySideload, SupportedEntities) export class EntitiesController { constructor(private readonly policyService: PolicyService, private readonly entitiesService: EntitiesService) {} diff --git a/libs/common/src/lib/events/event.processor.ts b/libs/common/src/lib/events/event.processor.ts index 8a04f4f5..14489541 100644 --- a/libs/common/src/lib/events/event.processor.ts +++ b/libs/common/src/lib/events/event.processor.ts @@ -3,7 +3,7 @@ import { RequestContext } from "nestjs-request-context"; import { EventService } from "./event.service"; export abstract class EventProcessor { - constructor(protected readonly eventService: EventService) {} + protected constructor(protected readonly eventService: EventService) {} private _authenticatedUser?: User | null; async getAuthenticatedUser() { From d37a778d9025cecfec12dfaae0cf3ddd3ca8c33b Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 21:40:38 -0700 Subject: [PATCH 23/98] [TM-1861] Fix build. --- .../src/airtable/entities/airtable-entity.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts index fd668fc0..529204bc 100644 --- a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts @@ -156,7 +156,7 @@ export abstract class AirtableEntity, Associa try { const records = (await this.MODEL.findAll( this.getDeletePageFindOptions(deletedSince, page) - )) as unknown as UuidModel[]; + )) as unknown as UuidModel[]; // Page had no records, halt processing. if (records.length === 0) return false; @@ -319,7 +319,7 @@ const selectIncludes = , A>(columns: ColumnMapping[]) = return mapping.include.reduce(mergeInclude, includes); }, [] as Include[]); -export const commonEntityColumns = , A = Record>(adminSiteType: string) => +export const commonEntityColumns = >(adminSiteType: string) => [ "uuid", "createdAt", From 55ed532ed3951af62254a9ea4c1fb860726366dc Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 21:56:22 -0700 Subject: [PATCH 24/98] [TM-1861] Update user creation service spec. --- .../src/users/user-creation.service.spec.ts | 128 ++---------------- .../src/users/user-creation.service.ts | 2 +- .../lib/localization/localization.service.ts | 2 +- .../lib/entities/localization-keys.entity.ts | 2 +- .../src/lib/factories/user.factory.ts | 2 +- 5 files changed, 13 insertions(+), 123 deletions(-) diff --git a/apps/user-service/src/users/user-creation.service.spec.ts b/apps/user-service/src/users/user-creation.service.spec.ts index 868064ab..50bc613f 100644 --- a/apps/user-service/src/users/user-creation.service.spec.ts +++ b/apps/user-service/src/users/user-creation.service.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from "@nestjs/testing"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; -import { ModelHasRole, Role, User, Verification } from "@terramatch-microservices/database/entities"; +import { LocalizationKey, ModelHasRole, Role, User, Verification } from "@terramatch-microservices/database/entities"; import { EmailService } from "@terramatch-microservices/common/email/email.service"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { UserCreationService } from "./user-creation.service"; @@ -62,6 +62,10 @@ describe("UserCreationService", () => { return await UserFactory.create(); } + beforeAll(async () => { + await LocalizationKey.truncate(); + }); + beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -110,116 +114,15 @@ describe("UserCreationService", () => { Promise.resolve([localizationBody, localizationSubject, localizationTitle, localizationCta]) ); - emailService.sendEmail.mockReturnValue(Promise.resolve()); + emailService.sendI18nTemplateEmail.mockReturnValue(Promise.resolve()); templateService.render.mockReturnValue("rendered template"); const result = await service.createNewUser(userNewRequest); expect(reloadSpy).toHaveBeenCalled(); - expect(emailService.sendEmail).toHaveBeenCalled(); + expect(emailService.sendI18nTemplateEmail).toHaveBeenCalled(); expect(result).toBeDefined(); }); - it("should return an error because localizations not found", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - const role = RoleFactory.create({ name: userNewRequest.role }); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(role)); - jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); - jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new NotFoundException("Localizations not found") - ); - }); - - it("should return an error because localization body not found", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - const role = RoleFactory.create({ name: userNewRequest.role }); - - const localizationSubject = await getLocalizationSubject(); - - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([localizationSubject])); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(role)); - jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); - jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new NotFoundException("Localization body not found") - ); - }); - - it("should return an error because localization subject not found", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - const role = RoleFactory.create({ name: userNewRequest.role }); - - const localizationBody = await getLocalizationBody(); - - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([localizationBody])); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(role)); - jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); - jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new NotFoundException("Localization subject not found") - ); - }); - - it("should return an error because localization title not found", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - const role = RoleFactory.create({ name: userNewRequest.role }); - - const localizationBody = await getLocalizationBody(); - const localizationSubject = await getLocalizationSubject(); - - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([localizationBody, localizationSubject])); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(role)); - jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); - jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new NotFoundException("Localization title not found") - ); - }); - - it("should return an error because localization CTA not found", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - const role = RoleFactory.create({ name: userNewRequest.role }); - - const localizationBody = await getLocalizationBody(); - const localizationSubject = await getLocalizationSubject(); - const localizationTitle = await getLocalizationTitle(); - - localizationService.getLocalizationKeys.mockReturnValue( - Promise.resolve([localizationBody, localizationSubject, localizationTitle]) - ); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(role)); - jest.spyOn(User, "create").mockImplementation(() => Promise.resolve(user)); - jest.spyOn(ModelHasRole, "findOrCreate").mockResolvedValue(null); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new NotFoundException("Localization CTA not found") - ); - }); - it("should generate a error because user already exist", async () => { const user = await UserFactory.create(); jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); @@ -235,7 +138,7 @@ describe("UserCreationService", () => { ); await expect(service.createNewUser(userNewRequest)).rejects.toThrow( - new UnprocessableEntityException("User already exist") + new UnprocessableEntityException("User already exists") ); }); @@ -326,7 +229,7 @@ describe("UserCreationService", () => { Promise.resolve([localizationBody, localizationSubject, localizationTitle, localizationCta]) ); - emailService.sendEmail.mockRejectedValue(null); + emailService.sendI18nTemplateEmail.mockRejectedValue(null); const result = await service.createNewUser(userNewRequest); expect(result).toBeNull(); @@ -401,17 +304,4 @@ describe("UserCreationService", () => { await expect(service.createNewUser(userNewRequest)).resolves.toBeNull(); }); - - it("should return an error when localizationService.getLocalizationKeys fails", async () => { - const user = await UserFactory.create(); - const userNewRequest = getRequest(user.emailAddress, "project-developer"); - - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(null)); - jest.spyOn(Role, "findOne").mockImplementation(() => Promise.resolve(null)); - jest - .spyOn(localizationService, "getLocalizationKeys") - .mockRejectedValue(new Error("Localization keys retrieval failed")); - - await expect(service.createNewUser(userNewRequest)).rejects.toThrow("Localization keys retrieval failed"); - }); }); diff --git a/apps/user-service/src/users/user-creation.service.ts b/apps/user-service/src/users/user-creation.service.ts index b6599d56..409e7042 100644 --- a/apps/user-service/src/users/user-creation.service.ts +++ b/apps/user-service/src/users/user-creation.service.ts @@ -34,7 +34,7 @@ export class UserCreationService { const userExists = (await User.count({ where: { emailAddress: request.emailAddress } })) !== 0; if (userExists) { - throw new UnprocessableEntityException("User already exist"); + throw new UnprocessableEntityException("User already exists"); } try { diff --git a/libs/common/src/lib/localization/localization.service.ts b/libs/common/src/lib/localization/localization.service.ts index d2d554c2..f537fea5 100644 --- a/libs/common/src/lib/localization/localization.service.ts +++ b/libs/common/src/lib/localization/localization.service.ts @@ -38,7 +38,7 @@ export class LocalizationService { }, {} as Dictionary); } - private async getLocalizationKeys(keys: string[]): Promise { + async getLocalizationKeys(keys: string[]): Promise { return (await LocalizationKey.findAll({ where: { key: { [Op.in]: keys } } })) ?? []; } diff --git a/libs/database/src/lib/entities/localization-keys.entity.ts b/libs/database/src/lib/entities/localization-keys.entity.ts index d8c4410b..92bd37d8 100644 --- a/libs/database/src/lib/entities/localization-keys.entity.ts +++ b/libs/database/src/lib/entities/localization-keys.entity.ts @@ -19,6 +19,6 @@ export class LocalizationKey extends Model { @Column(INTEGER({ length: 11 })) valueId: number; - @BelongsTo(() => I18nItem, { foreignKey: "value_id" }) + @BelongsTo(() => I18nItem, { foreignKey: "value_id", constraints: false }) i18nItem: I18nItem; } diff --git a/libs/database/src/lib/factories/user.factory.ts b/libs/database/src/lib/factories/user.factory.ts index 759d7ef1..b565ed65 100644 --- a/libs/database/src/lib/factories/user.factory.ts +++ b/libs/database/src/lib/factories/user.factory.ts @@ -14,7 +14,7 @@ export const UserFactory = FactoryGirl.define(User, async () => ({ async function generateUniqueEmail() { let emailAddress = faker.internet.email(); - while ((await User.findOne({ where: { emailAddress } })) != null) { + while ((await User.count({ where: { emailAddress } })) !== 0) { emailAddress = faker.internet.email(); } From 7a91bdb24cb73eaf91a1b053319d9f17efd7f9bb Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 10 Apr 2025 22:01:15 -0700 Subject: [PATCH 25/98] [TM-1861] Update reset password service spec. --- .../src/auth/reset-password.service.spec.ts | 87 ++----------------- 1 file changed, 9 insertions(+), 78 deletions(-) diff --git a/apps/user-service/src/auth/reset-password.service.spec.ts b/apps/user-service/src/auth/reset-password.service.spec.ts index 4f09ca46..2122ac9e 100644 --- a/apps/user-service/src/auth/reset-password.service.spec.ts +++ b/apps/user-service/src/auth/reset-password.service.spec.ts @@ -14,8 +14,6 @@ describe("ResetPasswordService", () => { let service: ResetPasswordService; let jwtService: DeepMocked; let emailService: DeepMocked; - let localizationService: DeepMocked; - let templateService: DeepMocked; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -31,11 +29,11 @@ describe("ResetPasswordService", () => { }, { provide: LocalizationService, - useValue: (localizationService = createMock()) + useValue: createMock() }, { provide: TemplateService, - useValue: (templateService = createMock()) + useValue: createMock() } ] }).compile(); @@ -54,100 +52,33 @@ describe("ResetPasswordService", () => { ).rejects.toThrow(NotFoundException); }); - it("should throw when localizations not found", async () => { - const user = await UserFactory.create(); - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([])); - await expect( - service.sendResetPasswordEmail("user@gmail.com", "https://example.com/auth/reset-password") - ).rejects.toThrow(new NotFoundException("Localizations not found")); - }); - - it("should throw when localization body not found", async () => { - const user = await UserFactory.create(); - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); - const localizationSubject = await LocalizationKeyFactory.create({ - key: "reset-password.subject", - value: "Reset Password" - }); - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([localizationSubject])); - await expect( - service.sendResetPasswordEmail("user@gmail.com", "https://example.com/auth/reset-password") - ).rejects.toThrow(new NotFoundException("Localization body not found")); - }); - - it("should throw when localization title not found", async () => { - const user = await UserFactory.create(); - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); - const localizationSubject = await LocalizationKeyFactory.create({ - key: "reset-password.subject", - value: "Reset Password" - }); - const localizationBody = await LocalizationKeyFactory.create({ - key: "reset-password.body", - value: "Reset your password by clicking on the following link: link" - }); - localizationService.getLocalizationKeys.mockReturnValue(Promise.resolve([localizationSubject, localizationBody])); - await expect( - service.sendResetPasswordEmail("user@gmail.com", "https://example.com/auth/reset-password") - ).rejects.toThrow(new NotFoundException("Localization title not found")); - }); - - it("should throw when localization CTA not found", async () => { - const user = await UserFactory.create(); - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); - const localizationSubject = await LocalizationKeyFactory.create({ - key: "reset-password.subject", - value: "Reset Password" - }); - const localizationBody = await LocalizationKeyFactory.create({ - key: "reset-password.body", - value: "Reset your password by clicking on the following link: link" - }); - const localizationTitle = await LocalizationKeyFactory.create({ - key: "reset-password.title", - value: "Reset Password" - }); - localizationService.getLocalizationKeys.mockReturnValue( - Promise.resolve([localizationSubject, localizationBody, localizationTitle]) - ); - await expect( - service.sendResetPasswordEmail("user@gmail.com", "https://example.com/auth/reset-password") - ).rejects.toThrow(new NotFoundException("Localization CTA not found")); - }); - it("should send a reset password email to the user", async () => { const user = await UserFactory.create(); - const localizationBody = await LocalizationKeyFactory.create({ + await LocalizationKeyFactory.create({ key: "reset-password.body", value: "Reset your password by clicking on the following link: link" }); - const localizationSubject = await LocalizationKeyFactory.create({ + await LocalizationKeyFactory.create({ key: "reset-password.subject", value: "Reset Password" }); - const localizationTitle = await LocalizationKeyFactory.create({ + await LocalizationKeyFactory.create({ key: "reset-password.title", value: "Reset Password" }); - const localizationCta = await LocalizationKeyFactory.create({ + await LocalizationKeyFactory.create({ key: "reset-password.cta", value: "Reset Password" }); - jest.spyOn(User, "findOne").mockImplementation(() => Promise.resolve(user)); - localizationService.getLocalizationKeys.mockReturnValue( - Promise.resolve([localizationBody, localizationSubject, localizationTitle, localizationCta]) - ); const token = "fake token"; jwtService.signAsync.mockReturnValue(Promise.resolve(token)); - emailService.sendEmail.mockReturnValue(Promise.resolve()); - templateService.render.mockReturnValue("rendered template"); + emailService.sendI18nTemplateEmail.mockReturnValue(Promise.resolve()); - const result = await service.sendResetPasswordEmail("user@gmail.com", "https://example.com/auth/reset-password"); + const result = await service.sendResetPasswordEmail(user.emailAddress, "https://example.com/auth/reset-password"); expect(jwtService.signAsync).toHaveBeenCalled(); - expect(emailService.sendEmail).toHaveBeenCalled(); + expect(emailService.sendI18nTemplateEmail).toHaveBeenCalled(); expect(result).toStrictEqual({ email: user.emailAddress, uuid: user.uuid }); }); From 8f86726de0ee2939c5e1160ae1c8f4310cade1da Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Wed, 16 Apr 2025 06:39:14 -0600 Subject: [PATCH 26/98] [TM-1711] refactor: simplify job query by removing unnecessary status filter --- apps/job-service/src/jobs/delayed-jobs.controller.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/job-service/src/jobs/delayed-jobs.controller.ts b/apps/job-service/src/jobs/delayed-jobs.controller.ts index f047340d..f283d275 100644 --- a/apps/job-service/src/jobs/delayed-jobs.controller.ts +++ b/apps/job-service/src/jobs/delayed-jobs.controller.ts @@ -90,8 +90,7 @@ export class DelayedJobsController { const jobs = await DelayedJob.findAll({ where: { uuid: { [Op.in]: jobUpdates.map(({ uuid }) => uuid) }, - createdBy: authenticatedUserId, - status: { [Op.ne]: "pending" } + createdBy: authenticatedUserId }, order: [["createdAt", "DESC"]] }); From 9ace600b27aecabd1b32a95e94d52332aa8f06c7 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 16 Apr 2025 19:29:08 -0700 Subject: [PATCH 27/98] [TM-1861] npm audit fix --- package-lock.json | 136 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 132 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 970fd950..a15236cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3531,11 +3531,14 @@ } }, "node_modules/@nestjs/common": { - "version": "11.0.13", - "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.0.13.tgz", - "integrity": "sha512-cXqXJPQTcJIYqT8GtBYqjYY9sklCBqp/rh9z1R40E60gWnsU598YIQWkojSFRI9G7lT/+uF+jqSrg/CMPBk7QQ==", + "version": "11.0.20", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.0.20.tgz", + "integrity": "sha512-/GH8NDCczjn6+6RNEtSNAts/nq/wQE8L1qZ9TRjqjNqEsZNE1vpFuRIhmcO2isQZ0xY5rySnpaRdrOAul3gQ3A==", + "license": "MIT", "dependencies": { + "file-type": "20.4.1", "iterare": "1.2.1", + "load-esm": "1.0.2", "tslib": "2.8.1", "uid": "2.0.2" }, @@ -6623,6 +6626,30 @@ "@swc/counter": "^0.1.3" } }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, "node_modules/@transifex/native": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@transifex/native/-/native-7.1.3.tgz", @@ -10621,6 +10648,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -10657,6 +10690,24 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-type": { + "version": "20.4.1", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.4.1.tgz", + "integrity": "sha512-hw9gNZXUfZ02Jo0uafWLaFVPter5/k2rfcrjFJJHX/77xtSDOfJuEFb6oKlFV86FLP1SuyHMW1PSk0U9M5tKkQ==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.6", + "strtok3": "^10.2.0", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -11706,7 +11757,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -13228,6 +13278,25 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/load-esm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/load-esm/-/load-esm-1.0.2.tgz", + "integrity": "sha512-nVAvWk/jeyrWyXEAs84mpQCYccxRqgKY4OznLuJhJCa0XsPSfdOIr2zvBZEj3IHEHbX97jjscKRRV539bW0Gpw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + }, + { + "type": "buymeacoffee", + "url": "https://buymeacoffee.com/borewit" + } + ], + "license": "MIT", + "engines": { + "node": ">=13.2.0" + } + }, "node_modules/loader-runner": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", @@ -14588,6 +14657,19 @@ "node": ">=8" } }, + "node_modules/peek-readable": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-7.0.0.tgz", + "integrity": "sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/pg-connection-string": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz", @@ -17287,6 +17369,23 @@ } ] }, + "node_modules/strtok3": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.2.tgz", + "integrity": "sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/style-loader": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.4.tgz", @@ -17783,6 +17882,23 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.0.0.tgz", + "integrity": "sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/toposort-class": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", @@ -18066,6 +18182,18 @@ "node": ">=8" } }, + "node_modules/uint8array-extras": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.4.0.tgz", + "integrity": "sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/undici-types": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", From 7d207d51d332dc46e6bbca0fc4fe780440d12b93 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 16 Apr 2025 19:52:32 -0700 Subject: [PATCH 28/98] [TM-1861] Provide a built-in option for getting to the shell instead of the repl in the remote ECS env. --- tm-v3-cli/src/repl/command.ts | 37 ++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tm-v3-cli/src/repl/command.ts b/tm-v3-cli/src/repl/command.ts index e2a24f84..471fa45d 100644 --- a/tm-v3-cli/src/repl/command.ts +++ b/tm-v3-cli/src/repl/command.ts @@ -1,4 +1,4 @@ -import { Argument, Command } from "commander"; +import { Argument, Command, Option, OptionValues } from "commander"; import { rootDebug } from "../utils"; import { ECSClient, ExecuteCommandCommand, ListTasksCommand } from "@aws-sdk/client-ecs"; import { CLUSTER, Environment, ENVIRONMENTS, Service, SERVICES } from "../consts"; @@ -13,6 +13,9 @@ const debugError = rootDebug.extend("remote-repl:error"); const TERM_OPTIONS = { rows: 34, cols: 197 }; +const REMOTE_COMMANDS = ["repl", "sh"] as const; +type RemoteCommand = (typeof REMOTE_COMMANDS)[number]; + const startLocalRepl = async (service: Service) => { debug(`Building REPL for ${service}`); await new Promise(resolve => { @@ -46,17 +49,32 @@ const getTaskId = async (service: Service, env: Environment) => { return undefined; }; -const startRemoteRepl = async (taskId: string, service: Service) => { +const getRemoteCommandString = (service: Service, remoteCommand: RemoteCommand) => { + switch (remoteCommand) { + case "repl": + return `node dist/apps/${service}-repl`; + case "sh": + return "sh"; + + default: + debugError(`Unrecognized command: ${remoteCommand}`); + process.exit(-1); + } +}; + +const startRemoteRepl = async (taskId: string, service: Service, remoteCommand: RemoteCommand) => { const client = new ECSClient(); - const command = new ExecuteCommandCommand({ + const command = getRemoteCommandString(service, remoteCommand); + debug(`Setting command to run after connection: ${command}`); + const executionCommand = new ExecuteCommandCommand({ cluster: CLUSTER, task: taskId, interactive: true, - command: `node dist/apps/${service}-repl` + command }); const { session: { streamUrl, tokenValue } - } = await client.send(command); + } = await client.send(executionCommand); const textDecoder = new TextDecoder(); const textEncoder = new TextEncoder(); @@ -109,7 +127,12 @@ export const replCommand = () => .addArgument( new Argument("environment", "The environment to connect to").choices(ENVIRONMENTS).default("local").argOptional() ) - .action(async (service: Service, environment: Environment) => { + .addOption( + new Option("-c, --command ", "Ignored in local environment. Command to run after connecting.") + .default("repl") + .choices(REMOTE_COMMANDS) + ) + .action(async (service: Service, environment: Environment, options: OptionValues) => { if (environment === "local") { await startLocalRepl(service); } else { @@ -117,6 +140,6 @@ export const replCommand = () => if (taskId == null) return; debug(`Found task id: ${taskId}`); - await startRemoteRepl(taskId, service); + await startRemoteRepl(taskId, service, options.command as RemoteCommand); } }); From f9bb163c6c0195e7e6684b816874f391c1463e97 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 16 Apr 2025 20:57:45 -0700 Subject: [PATCH 29/98] [TM-1861] Email specs updated. --- .../src/lib/email/email.service.spec.ts | 52 ++++++++++++++++++- libs/common/src/lib/email/email.service.ts | 32 ++++++------ .../src/lib/factories/user.factory.ts | 1 + 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/libs/common/src/lib/email/email.service.spec.ts b/libs/common/src/lib/email/email.service.spec.ts index fd47a75c..29455ee7 100644 --- a/libs/common/src/lib/email/email.service.spec.ts +++ b/libs/common/src/lib/email/email.service.spec.ts @@ -3,6 +3,9 @@ import { EmailService } from "./email.service"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { ConfigService } from "@nestjs/config"; import * as nodemailer from "nodemailer"; +import { LocalizationService } from "../localization/localization.service"; +import { TemplateService } from "./template.service"; +import { UserFactory } from "@terramatch-microservices/database/factories"; jest.mock("nodemailer"); @@ -10,13 +13,20 @@ describe("EmailService", () => { let service: EmailService; let configService: DeepMocked; let transporter: nodemailer.Transporter; + let localizationService: DeepMocked; + let templateService: DeepMocked; beforeEach(async () => { // @ts-expect-error mock compiler confusion nodemailer.createTransport.mockReturnValue({ sendMail: jest.fn() }); const module = await Test.createTestingModule({ - providers: [EmailService, { provide: ConfigService, useValue: (configService = createMock()) }] + providers: [ + EmailService, + { provide: ConfigService, useValue: (configService = createMock()) }, + { provide: LocalizationService, useValue: (localizationService = createMock()) }, + { provide: TemplateService, useValue: (templateService = createMock()) } + ] }).compile(); service = module.get(EmailService); @@ -63,4 +73,44 @@ describe("EmailService", () => { }) ); }); + + it("filters email addresses based on ENTITY_UPDATE_DO_NOT_EMAIL in env", async () => { + const u1 = await UserFactory.create(); + const u2 = await UserFactory.create(); + const u3 = await UserFactory.create(); + configService.get.mockImplementation((envName: string) => { + if (envName === "ENTITY_UPDATE_DO_NOT_EMAIL") return `${u1.emailAddress},vader@empire.com`; + return ""; + }); + + const filtered = service.filterEntityEmailRecipients([u1, u2, u3]); + expect(filtered.length).toBe(2); + expect(filtered.map(({ id }) => id)).toEqual([u2, u3].map(({ id }) => id)); + }); + + it("throws if the template subject key is not provided", async () => { + await expect(service.sendI18nTemplateEmail("", "en-GB", {})).rejects.toThrow("Email subject is required"); + }); + + it("translates and renders template, then sends email", async () => { + configService.get.mockImplementation((envName: string) => { + if (envName === "MAIL_FROM_ADDRESS") return "person@terramatch.org"; + if (envName === "MAIL_RECIPIENTS") return ""; + return ""; + }); + + const to = ["pd@wri.org", "monitoring-partner@wri.org"]; + const locale = "es-MX"; + const i18nKeys = { subject: "foo-email.subject", body: "foo-email.body" }; + const i18nReplacements = { "{thing}": "replacement" }; + const additionalValues = { link: "aclu.org" }; + await service.sendI18nTemplateEmail(to, locale, i18nKeys, { + i18nReplacements, + additionalValues + }); + + expect(localizationService.translateKeys).toHaveBeenCalledWith(i18nKeys, locale, i18nReplacements); + expect(templateService.render).toHaveBeenCalled(); + expect(transporter.sendMail).toHaveBeenCalledWith(expect.objectContaining({ from: "person@terramatch.org", to })); + }); }); diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index 3e66d7d3..3d29220d 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -42,22 +42,6 @@ export class EmailService { return recipients.filter(({ emailAddress }) => !doNotEmail.includes(emailAddress)); } - async renderI18nTemplateEmail( - locale: string, - i18nKeys: Dictionary, - { i18nReplacements, additionalValues }: I18nEmailOptions = {} - ) { - if (i18nKeys["subject"] == null) throw new InternalServerErrorException("Email subject is required"); - const { subject, ...translated } = await this.localizationService.translateKeys( - i18nKeys, - locale, - i18nReplacements ?? {} - ); - - const body = this.templateService.render(EMAIL_TEMPLATE, { ...translated, ...(additionalValues ?? {}) }); - return { subject, body }; - } - async sendI18nTemplateEmail( to: string | string[], locale: string, @@ -92,4 +76,20 @@ export class EmailService { await this.transporter.sendMail(mailOptions); } + + private async renderI18nTemplateEmail( + locale: string, + i18nKeys: Dictionary, + { i18nReplacements, additionalValues }: I18nEmailOptions = {} + ) { + if (i18nKeys["subject"] == null) throw new InternalServerErrorException("Email subject is required"); + const { subject, ...translated } = await this.localizationService.translateKeys( + i18nKeys, + locale, + i18nReplacements ?? {} + ); + + const body = this.templateService.render(EMAIL_TEMPLATE, { ...translated, ...(additionalValues ?? {}) }); + return { subject, body }; + } } diff --git a/libs/database/src/lib/factories/user.factory.ts b/libs/database/src/lib/factories/user.factory.ts index b565ed65..c0640208 100644 --- a/libs/database/src/lib/factories/user.factory.ts +++ b/libs/database/src/lib/factories/user.factory.ts @@ -5,6 +5,7 @@ import { User } from "../entities"; // TODO: generate correctly hashed passwords. This will be easily accomplished once user signup // has been implemented in this codebase. export const UserFactory = FactoryGirl.define(User, async () => ({ + locale: "en-US", firstName: faker.person.firstName(), lastName: faker.person.lastName(), emailAddress: await generateUniqueEmail(), From 3db370a3e9936ce6fa4ae2e5996cec267dbfdc96 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 17 Apr 2025 06:49:16 -0600 Subject: [PATCH 30/98] [TM-1711] test: update bulkUpdateJobs test to correctly handle pending job status --- .../src/jobs/delayed-jobs.controller.spec.ts | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/job-service/src/jobs/delayed-jobs.controller.spec.ts b/apps/job-service/src/jobs/delayed-jobs.controller.spec.ts index 62c0d858..2cba28af 100644 --- a/apps/job-service/src/jobs/delayed-jobs.controller.spec.ts +++ b/apps/job-service/src/jobs/delayed-jobs.controller.spec.ts @@ -224,7 +224,7 @@ describe("DelayedJobsController", () => { await expect(controller.bulkUpdateJobs(payload, request)).rejects.toThrow(NotFoundException); }); - it('should not update jobs with status "pending"', async () => { + it('should update jobs with status "pending"', async () => { const authenticatedUserId = 130999; const pendingJob = await DelayedJob.create({ uuid: uuidv4(), @@ -234,18 +234,36 @@ describe("DelayedJobsController", () => { metadata: { entity_name: "TestEntityPending" } // Adding entity_name }); + const pendingJob2 = await DelayedJob.create({ + uuid: uuidv4(), + createdBy: authenticatedUserId, + isAcknowledged: false, + status: "pending", + metadata: { entity_name: "TestEntityPending2" } // Adding entity_name + }); + const payload: DelayedJobBulkUpdateBodyDto = { data: [ { type: "delayedJobs", uuid: pendingJob.uuid, attributes: { isAcknowledged: true } + }, + { + type: "delayedJobs", + uuid: pendingJob2.uuid, + attributes: { isAcknowledged: true } } ] }; const request = { authenticatedUserId }; - await expect(controller.bulkUpdateJobs(payload, request)).rejects.toThrow(NotFoundException); + const result = await controller.bulkUpdateJobs(payload, request); + expect(result.data).toHaveLength(2); + expect(result.data[0].id).toBe(pendingJob.uuid); + expect(result.data[0].attributes.entityName).toBe("TestEntityPending"); + expect(result.data[1].id).toBe(pendingJob2.uuid); + expect(result.data[1].attributes.entityName).toBe("TestEntityPending2"); }); }); From 1094f42538176368f54639593f25ad231e903971 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 17 Apr 2025 13:38:48 -0700 Subject: [PATCH 31/98] [TM-1861] Full coverage for common lib. --- .../src/auth/reset-password.service.spec.ts | 2 +- .../src/users/user-creation.service.spec.ts | 2 +- libs/common/src/lib/common.module.ts | 2 +- .../src/lib/email/email.service.spec.ts | 11 +- libs/common/src/lib/email/email.service.ts | 22 ++- libs/common/src/lib/email/template.service.ts | 42 ------ .../localization/localization.service.spec.ts | 130 ++++++++++++------ .../lib/localization/localization.service.ts | 2 +- .../lib/templates/template.service.spec.ts | 35 +++++ .../src/lib/templates/template.service.ts | 23 ++++ .../lib/entities/i18n-translation.entity.ts | 3 +- libs/database/src/lib/entities/index.ts | 2 +- ...ion-keys.entity.ts => localization-key.ts} | 0 .../src/lib/factories/i18n-item.factory.ts | 9 ++ .../lib/factories/i18n-translation.factory.ts | 11 ++ libs/database/src/lib/factories/index.ts | 1 + .../lib/factories/localization-key.factory.ts | 3 +- 17 files changed, 207 insertions(+), 93 deletions(-) delete mode 100644 libs/common/src/lib/email/template.service.ts create mode 100644 libs/common/src/lib/templates/template.service.spec.ts create mode 100644 libs/common/src/lib/templates/template.service.ts rename libs/database/src/lib/entities/{localization-keys.entity.ts => localization-key.ts} (100%) create mode 100644 libs/database/src/lib/factories/i18n-item.factory.ts create mode 100644 libs/database/src/lib/factories/i18n-translation.factory.ts diff --git a/apps/user-service/src/auth/reset-password.service.spec.ts b/apps/user-service/src/auth/reset-password.service.spec.ts index 2122ac9e..40001c68 100644 --- a/apps/user-service/src/auth/reset-password.service.spec.ts +++ b/apps/user-service/src/auth/reset-password.service.spec.ts @@ -8,7 +8,7 @@ import { ResetPasswordService } from "./reset-password.service"; import { BadRequestException, NotFoundException } from "@nestjs/common"; import { LocalizationKeyFactory } from "@terramatch-microservices/database/factories/localization-key.factory"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; -import { TemplateService } from "@terramatch-microservices/common/email/template.service"; +import { TemplateService } from "@terramatch-microservices/common/templates/template.service"; describe("ResetPasswordService", () => { let service: ResetPasswordService; diff --git a/apps/user-service/src/users/user-creation.service.spec.ts b/apps/user-service/src/users/user-creation.service.spec.ts index 50bc613f..a5711f9e 100644 --- a/apps/user-service/src/users/user-creation.service.spec.ts +++ b/apps/user-service/src/users/user-creation.service.spec.ts @@ -8,7 +8,7 @@ import { UserNewRequest } from "./dto/user-new-request.dto"; import { NotFoundException, UnprocessableEntityException } from "@nestjs/common"; import { RoleFactory, UserFactory } from "@terramatch-microservices/database/factories"; import { LocalizationKeyFactory } from "@terramatch-microservices/database/factories/localization-key.factory"; -import { TemplateService } from "@terramatch-microservices/common/email/template.service"; +import { TemplateService } from "@terramatch-microservices/common/templates/template.service"; describe("UserCreationService", () => { let service: UserCreationService; diff --git a/libs/common/src/lib/common.module.ts b/libs/common/src/lib/common.module.ts index b3f63ca1..9455189b 100644 --- a/libs/common/src/lib/common.module.ts +++ b/libs/common/src/lib/common.module.ts @@ -8,13 +8,13 @@ import { PolicyService } from "./policies/policy.service"; import { LocalizationService } from "./localization/localization.service"; import { EmailService } from "./email/email.service"; import { MediaService } from "./media/media.service"; -import { TemplateService } from "./email/template.service"; import { SlackService } from "./slack/slack.service"; import { DatabaseModule } from "@terramatch-microservices/database"; import { EventEmitterModule } from "@nestjs/event-emitter"; import { EventService } from "./events/event.service"; import { BullModule } from "@nestjs/bullmq"; import { EmailProcessor } from "./email/email.processor"; +import { TemplateService } from "./templates/template.service"; export const QUEUES = ["email"]; diff --git a/libs/common/src/lib/email/email.service.spec.ts b/libs/common/src/lib/email/email.service.spec.ts index 29455ee7..b9933625 100644 --- a/libs/common/src/lib/email/email.service.spec.ts +++ b/libs/common/src/lib/email/email.service.spec.ts @@ -4,8 +4,8 @@ import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { ConfigService } from "@nestjs/config"; import * as nodemailer from "nodemailer"; import { LocalizationService } from "../localization/localization.service"; -import { TemplateService } from "./template.service"; import { UserFactory } from "@terramatch-microservices/database/factories"; +import { TemplateService } from "../templates/template.service"; jest.mock("nodemailer"); @@ -88,6 +88,15 @@ describe("EmailService", () => { expect(filtered.map(({ id }) => id)).toEqual([u2, u3].map(({ id }) => id)); }); + it("returns the list of users when there is no do not email list", async () => { + const users = await UserFactory.createMany(3); + configService.get.mockImplementation((envName: string) => { + if (envName === "ENTITY_UPDATE_DO_NOT_EMAIL") return ""; + return ""; + }); + expect(service.filterEntityEmailRecipients(users)).toBe(users); + }); + it("throws if the template subject key is not provided", async () => { await expect(service.sendI18nTemplateEmail("", "en-GB", {})).rejects.toThrow("Email subject is required"); }); diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index 3d29220d..412d220b 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -2,10 +2,10 @@ import { Injectable, InternalServerErrorException } from "@nestjs/common"; import * as nodemailer from "nodemailer"; import { ConfigService } from "@nestjs/config"; import * as Mail from "nodemailer/lib/mailer"; -import { Dictionary, isEmpty } from "lodash"; +import { Dictionary, isEmpty, isString } from "lodash"; import { LocalizationService } from "../localization/localization.service"; -import { TemplateService } from "./template.service"; import { User } from "@terramatch-microservices/database/entities"; +import { TemplateService } from "../templates/template.service"; type I18nEmailOptions = { i18nReplacements?: Dictionary; @@ -89,7 +89,23 @@ export class EmailService { i18nReplacements ?? {} ); - const body = this.templateService.render(EMAIL_TEMPLATE, { ...translated, ...(additionalValues ?? {}) }); + const data: Dictionary = { + backendUrl: this.configService.get("EMAIL_IMAGE_BASE_URL"), + banner: null, + invite: null, + monitoring: null, + transactional: null, + year: new Date().getFullYear(), + ...translated, + ...(additionalValues ?? {}) + }; + if (isString(data["link"]) && data["link"].substring(0, 4).toLowerCase() !== "http") { + // If we're given a link that's pointing to a raw path, assume it should be prepended with + // the configured app FE base URL. + data["link"] = `${this.configService.get("APP_FRONT_END")}${data["link"]}`; + } + + const body = this.templateService.render(EMAIL_TEMPLATE, data); return { subject, body }; } } diff --git a/libs/common/src/lib/email/template.service.ts b/libs/common/src/lib/email/template.service.ts deleted file mode 100644 index 1ed15948..00000000 --- a/libs/common/src/lib/email/template.service.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Injectable } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; -import * as Handlebars from "handlebars"; -import * as fs from "fs"; -import * as path from "path"; -import { Dictionary } from "factory-girl-ts"; -import { isString } from "lodash"; - -@Injectable() -export class TemplateService { - private templates: Dictionary = {}; - - constructor(private readonly configService: ConfigService) {} - - private getCompiledTemplate(template: string) { - if (this.templates[template] != null) return this.templates[template]; - - // reference the template path from the NX root. - const templatePath = path.join(__dirname, "../../..", template); - const templateSource = fs.readFileSync(templatePath, "utf-8"); - return (this.templates[template] = Handlebars.compile(templateSource)); - } - - render(templatePath: string, data: Dictionary): string { - const params: Dictionary = { - backendUrl: this.configService.get("EMAIL_IMAGE_BASE_URL"), - banner: null, - invite: null, - monitoring: null, - transactional: null, - year: new Date().getFullYear(), - ...data - }; - - if (isString(params["link"]) && params["link"].substring(0, 4).toLowerCase() !== "http") { - // If we're given a link that's pointing to a raw path, assume it should be prepended with - // the configured app FE base URL. - params["link"] = `${this.configService.get("APP_FRONT_END")}${params["link"]}`; - } - return this.getCompiledTemplate(templatePath)(params); - } -} diff --git a/libs/common/src/lib/localization/localization.service.spec.ts b/libs/common/src/lib/localization/localization.service.spec.ts index 63ea964e..01675652 100644 --- a/libs/common/src/lib/localization/localization.service.spec.ts +++ b/libs/common/src/lib/localization/localization.service.spec.ts @@ -1,9 +1,14 @@ import { Test, TestingModule } from "@nestjs/testing"; import { LocalizationService } from "./localization.service"; -import { I18nItem, I18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; +import { I18nTranslation, LocalizationKey } from "@terramatch-microservices/database/entities"; +import { faker } from "@faker-js/faker"; import { ConfigService } from "@nestjs/config"; -import { tx, t } from "@transifex/native"; +import { normalizeLocale, t, tx } from "@transifex/native"; import { createMock } from "@golevelup/ts-jest"; +import { I18nTranslationFactory } from "@terramatch-microservices/database/factories/i18n-translation.factory"; +import { LocalizationKeyFactory } from "@terramatch-microservices/database/factories/localization-key.factory"; +import { Dictionary } from "lodash"; +import { NotFoundException } from "@nestjs/common"; jest.mock("@transifex/native", () => ({ tx: { @@ -29,51 +34,98 @@ describe("LocalizationService", () => { jest.restoreAllMocks(); }); - it("should return an empty array when no localization keys are found", async () => { - await expect(service.getLocalizationKeys(["foo", "abc"])).resolves.toStrictEqual([]); - }); + describe("getLocalizationKeys", () => { + it("should return an empty array when no localization keys are found", async () => { + await expect(service.getLocalizationKeys(["foo", "abc"])).resolves.toStrictEqual([]); + }); - it("should return one localization key when a matching key is found", async () => { - const localization = [new LocalizationKey()]; - jest.spyOn(LocalizationKey, "findAll").mockImplementation(() => Promise.resolve(localization)); - const result = await service.getLocalizationKeys(["foo"]); - expect(result.length).toBe(1); - }); + it("should return one localization key when a matching key is found", async () => { + const localization = [new LocalizationKey()]; + jest.spyOn(LocalizationKey, "findAll").mockImplementation(() => Promise.resolve(localization)); + const result = await service.getLocalizationKeys(["foo"]); + expect(result.length).toBe(1); + }); - it("should return the translated text when a matching i18n item and translation record are found", async () => { - const i18Record = new I18nTranslation(); - i18Record.shortValue = "contenido traducido"; - const i18Item = new I18nItem(); - jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); - jest.spyOn(I18nTranslation, "findOne").mockImplementation(() => Promise.resolve(i18Record)); - const result = await service.translate("content translate", "es-MX"); - expect(result).toBe(i18Record.shortValue); + it("should return an empty array when none of the keys are found", async () => { + await LocalizationKey.truncate(); + expect(await service.getLocalizationKeys(["foo", "bar"])).toStrictEqual([]); + }); }); - it("should return the original text when no matching i18n item is found", async () => { - jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(null)); - const result = await service.translate("content translate", "es-MX"); - expect(result).toBe("content translate"); - }); + describe("localizeText", () => { + it("should normalize the locale and return the translated text", async () => { + const text = "Hello"; + const locale = "es_MX"; + const translatedText = "Hola"; - it("should return the original text when no translation record is found for the given locale", async () => { - const i18Item = new I18nItem(); - jest.spyOn(I18nItem, "findOne").mockImplementation(() => Promise.resolve(i18Item)); - jest.spyOn(I18nTranslation, "findOne").mockImplementation(() => Promise.resolve(null)); - const result = await service.translate("content translate", "es-MX"); - expect(result).toBe("content translate"); + (normalizeLocale as jest.Mock).mockReturnValue(locale); + (tx.setCurrentLocale as jest.Mock).mockResolvedValue(undefined); + (t as jest.Mock).mockReturnValue(translatedText); + + const result = await service.localizeText(text, "es_ES"); + expect(normalizeLocale).toHaveBeenCalledWith("es_ES"); + expect(tx.setCurrentLocale).toHaveBeenCalledWith(locale); + expect(t).toHaveBeenCalledWith(text, undefined); + expect(result).toBe(translatedText); + }); }); - it("should translate text correctly", async () => { - const text = "Hello"; - const locale = "es_MX"; - const translatedText = "Hola"; + describe("translateKeys", () => { + it("should translate", async () => { + const enTranslations = await I18nTranslationFactory.createMany(5); + const esTranslations = await Promise.all( + enTranslations.map(({ i18nItemId }) => I18nTranslationFactory.create({ i18nItemId, language: "es-MX" })) + ); + const localizationKeys = await Promise.all( + enTranslations.map(({ i18nItemId }) => LocalizationKeyFactory.create({ valueId: i18nItemId })) + ); + + const keyMap = localizationKeys.reduce( + (keyMap, { key }) => ({ ...keyMap, [faker.lorem.slug()]: key as string }), + {} as Dictionary + ); + + const reduceTranslations = (translations: I18nTranslation[]) => + Object.entries(keyMap).reduce((i18nMap, [prop, key]) => { + const itemId = localizationKeys.find(({ key: lKey }) => lKey === key)?.valueId; + if (itemId == null) return i18nMap; + + const translation = translations.find(({ i18nItemId }) => i18nItemId === itemId); + return { + ...i18nMap, + [prop]: translation?.longValue ?? translation?.shortValue ?? "missing-translation" + }; + }, {} as Dictionary); + + expect(await service.translateKeys(keyMap, "en-US")).toMatchObject(reduceTranslations(enTranslations)); + expect(await service.translateKeys(keyMap, "es-MX")).toMatchObject(reduceTranslations(esTranslations)); + }); + + it("uses replacements", async () => { + const esTranslation = await I18nTranslationFactory.create({ + language: "es-MX", + longValue: null, + shortValue: "Este edificio parece uno de {city}, y me encanta {city} y todos los ciudades de {country}." + }); + const { key } = await LocalizationKeyFactory.create({ valueId: esTranslation.i18nItemId }); + expect( + ( + await service.translateKeys({ body: key as string }, "es-MX", { + ["{city}"]: "Barcelona", + ["{country}"]: "España" + }) + )["body"] + ).toBe("Este edificio parece uno de Barcelona, y me encanta Barcelona y todos los ciudades de España."); + }); - (tx.setCurrentLocale as jest.Mock).mockResolvedValue(undefined); - (t as jest.Mock).mockReturnValue(translatedText); + it("should throw if a localization key is missing", async () => { + await LocalizationKey.truncate(); + await expect(service.translateKeys({ foo: "bar" }, "es-MX")).rejects.toThrow(NotFoundException); + }); - const result = await service.localizeText(text, locale); - expect(t).toHaveBeenCalledWith(text, undefined); - expect(result).toBe(translatedText); + it("should use the value from LocalizationKey if there is no translation", async () => { + const { key, value } = await LocalizationKeyFactory.create(); + expect((await service.translateKeys({ foo: key as string }, "es-MX"))["foo"]).toEqual(value); + }); }); }); diff --git a/libs/common/src/lib/localization/localization.service.ts b/libs/common/src/lib/localization/localization.service.ts index ff43723f..3782fcfa 100644 --- a/libs/common/src/lib/localization/localization.service.ts +++ b/libs/common/src/lib/localization/localization.service.ts @@ -39,7 +39,7 @@ export class LocalizationService { } async getLocalizationKeys(keys: string[]): Promise { - return (await LocalizationKey.findAll({ where: { key: { [Op.in]: keys } } })) ?? []; + return await LocalizationKey.findAll({ where: { key: { [Op.in]: keys } } }); } /** diff --git a/libs/common/src/lib/templates/template.service.spec.ts b/libs/common/src/lib/templates/template.service.spec.ts new file mode 100644 index 00000000..356b4d58 --- /dev/null +++ b/libs/common/src/lib/templates/template.service.spec.ts @@ -0,0 +1,35 @@ +import * as fs from "fs"; +import { Test } from "@nestjs/testing"; +import { TemplateService } from "./template.service"; + +describe("TemplateService", () => { + let service: TemplateService; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [TemplateService] + }).compile(); + service = module.get(TemplateService); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("should render the template with the given data", async () => { + const template = "A sample {{type}} template. Useful for {{entity}} updates."; + jest.spyOn(fs, "readFileSync").mockImplementation(() => template); + const result = service.render("template", { type: "email", unused: "foo" }); + expect(result).toEqual("A sample email template. Useful for updates."); + }); + + it("should cache the compiled template", async () => { + const template = "A sample {{type}} template. Useful for {{entity}} updates."; + const spy = jest.spyOn(fs, "readFileSync").mockImplementation(() => template); + let result = service.render("template", { type: "email", entity: "foo" }); + expect(result).toEqual("A sample email template. Useful for foo updates."); + result = service.render("template", { type: "text", entity: "bar" }); + expect(result).toEqual("A sample text template. Useful for bar updates."); + expect(spy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/libs/common/src/lib/templates/template.service.ts b/libs/common/src/lib/templates/template.service.ts new file mode 100644 index 00000000..f898d57b --- /dev/null +++ b/libs/common/src/lib/templates/template.service.ts @@ -0,0 +1,23 @@ +import { Injectable } from "@nestjs/common"; +import * as Handlebars from "handlebars"; +import * as fs from "fs"; +import * as path from "path"; +import { Dictionary } from "factory-girl-ts"; + +@Injectable() +export class TemplateService { + private templates: Dictionary = {}; + + private getCompiledTemplate(template: string) { + if (this.templates[template] != null) return this.templates[template]; + + // reference the template path from the NX root. + const templatePath = path.join(__dirname, "../../..", template); + const templateSource = fs.readFileSync(templatePath, "utf-8"); + return (this.templates[template] = Handlebars.compile(templateSource)); + } + + render(templatePath: string, data: Dictionary): string { + return this.getCompiledTemplate(templatePath)(data); + } +} diff --git a/libs/database/src/lib/entities/i18n-translation.entity.ts b/libs/database/src/lib/entities/i18n-translation.entity.ts index 4df85647..41fc9e20 100644 --- a/libs/database/src/lib/entities/i18n-translation.entity.ts +++ b/libs/database/src/lib/entities/i18n-translation.entity.ts @@ -1,4 +1,4 @@ -import { AllowNull, AutoIncrement, Column, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { AllowNull, AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, INTEGER, STRING, TEXT } from "sequelize"; @Table({ tableName: "i18n_translations", underscored: true }) @@ -9,7 +9,6 @@ export class I18nTranslation extends Model { override id: number; @AllowNull - @Unique @Column(INTEGER({ length: 11 })) i18nItemId: number; diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 9f34bc96..c2fe5b7d 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -18,7 +18,7 @@ export * from "./indicator-output-tree-count.entity"; export * from "./indicator-output-tree-cover.entity"; export * from "./indicator-output-tree-cover-loss.entity"; export * from "./landscape-geometry.entity"; -export * from "./localization-keys.entity"; +export * from "./localization-key"; export * from "./media.entity"; export * from "./model-has-role.entity"; export * from "./nursery.entity"; diff --git a/libs/database/src/lib/entities/localization-keys.entity.ts b/libs/database/src/lib/entities/localization-key.ts similarity index 100% rename from libs/database/src/lib/entities/localization-keys.entity.ts rename to libs/database/src/lib/entities/localization-key.ts diff --git a/libs/database/src/lib/factories/i18n-item.factory.ts b/libs/database/src/lib/factories/i18n-item.factory.ts new file mode 100644 index 00000000..a182fc5e --- /dev/null +++ b/libs/database/src/lib/factories/i18n-item.factory.ts @@ -0,0 +1,9 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { I18nItem } from "../entities"; +import { faker } from "@faker-js/faker"; + +export const I18nItemFactory = FactoryGirl.define(I18nItem, async () => ({ + status: "translated", + type: "short", + shortValue: faker.lorem.word(4) +})); diff --git a/libs/database/src/lib/factories/i18n-translation.factory.ts b/libs/database/src/lib/factories/i18n-translation.factory.ts new file mode 100644 index 00000000..b36d4665 --- /dev/null +++ b/libs/database/src/lib/factories/i18n-translation.factory.ts @@ -0,0 +1,11 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { I18nTranslation } from "../entities"; +import { faker } from "@faker-js/faker"; +import { I18nItemFactory } from "./i18n-item.factory"; + +export const I18nTranslationFactory = FactoryGirl.define(I18nTranslation, async () => ({ + i18nItemId: I18nItemFactory.associate("id"), + language: "en-US", + shortValue: faker.lorem.word(4), + longValue: faker.lorem.sentence() +})); diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts index 96f82d0b..fbf5dea0 100644 --- a/libs/database/src/lib/factories/index.ts +++ b/libs/database/src/lib/factories/index.ts @@ -4,6 +4,7 @@ export * from "./demographic.factory"; export * from "./demographic-entry.factory"; export * from "./form-submission.factory"; export * from "./funding-programme.factory"; +export * from "./i18n-item.factory"; export * from "./indicator-output-field-monitoring.factory"; export * from "./indicator-output-hectares.factory"; export * from "./indicator-output-msu-carbon.factory"; diff --git a/libs/database/src/lib/factories/localization-key.factory.ts b/libs/database/src/lib/factories/localization-key.factory.ts index 2bb45d6b..73fdacad 100644 --- a/libs/database/src/lib/factories/localization-key.factory.ts +++ b/libs/database/src/lib/factories/localization-key.factory.ts @@ -1,9 +1,10 @@ import { FactoryGirl } from "factory-girl-ts"; import { faker } from "@faker-js/faker"; import { LocalizationKey } from "../entities"; +import { I18nItemFactory } from "./i18n-item.factory"; export const LocalizationKeyFactory = FactoryGirl.define(LocalizationKey, async () => ({ key: faker.lorem.word(), value: faker.lorem.sentence(), - valueId: faker.number.int({ min: 1, max: 1000 }) + valueId: I18nItemFactory.associate("id") })); From dec37df2e6554c127635e946808a5032b4e53d99 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 17 Apr 2025 15:32:34 -0700 Subject: [PATCH 32/98] [TM-1861] Cover the new behaviors in entity service. --- .../src/entities/dto/entity-update.dto.ts | 4 +- .../src/entities/entities.controller.spec.ts | 66 ++++++++++++++++++- .../entity-associations.controller.spec.ts | 2 - .../processors/entity.processor.spec.ts | 36 +++++++++- .../processors/project.processor.spec.ts | 26 +++++++- 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index b5b86f6b..8d853d74 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -38,8 +38,6 @@ export class ProjectUpdateAttributes extends EntityUpdateAttributes { isTest?: boolean; } -export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} - export class SiteUpdateAttributes extends EntityUpdateAttributes { @IsOptional() @IsIn(SITE_STATUSES) @@ -51,8 +49,8 @@ export class SiteUpdateAttributes extends EntityUpdateAttributes { status?: string | null; } +export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} - export class NurseryUpdateData extends JsonApiDataDto({ type: "nurseries" }, EntityUpdateAttributes) {} export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, EntityUpdateAttributes) {} export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, EntityUpdateAttributes) {} diff --git a/apps/entity-service/src/entities/entities.controller.spec.ts b/apps/entity-service/src/entities/entities.controller.spec.ts index 10c60734..989c69e8 100644 --- a/apps/entity-service/src/entities/entities.controller.spec.ts +++ b/apps/entity-service/src/entities/entities.controller.spec.ts @@ -7,11 +7,12 @@ import { AdditionalProjectFullProps, ProjectFullDto, ProjectLightDto } from "./d import { Project } from "@terramatch-microservices/database/entities"; import { PolicyService } from "@terramatch-microservices/common"; import { ProjectFactory } from "@terramatch-microservices/database/factories"; -import { NotFoundException, UnauthorizedException } from "@nestjs/common"; +import { BadRequestException, NotFoundException, UnauthorizedException } from "@nestjs/common"; import { EntityQueryDto } from "./dto/entity-query.dto"; import { faker } from "@faker-js/faker"; +import { EntityUpdateData } from "./dto/entity-update.dto"; -class StubProcessor extends EntityProcessor { +class StubProcessor extends EntityProcessor { LIGHT_DTO = ProjectLightDto; FULL_DTO = ProjectFullDto; @@ -22,6 +23,7 @@ class StubProcessor extends EntityProcessor Promise.resolve({ id: faker.string.uuid(), dto: new ProjectLightDto() })); delete = jest.fn(() => Promise.resolve()); + update = jest.fn(() => Promise.resolve()); } describe("EntitiesController", () => { @@ -116,4 +118,64 @@ describe("EntitiesController", () => { expect(result.data).toBeUndefined(); }); }); + + describe("entityUpdate", () => { + it("should throw if the entity payload type does not match the path type", async () => { + await expect( + controller.entityUpdate( + { entity: "sites", uuid: "asdf" }, + { data: { type: "projects", id: "asdf", attributes: {} } } + ) + ).rejects.toThrow(BadRequestException); + }); + + it("should throw if the entity payload id does not match the path uuid", async () => { + await expect( + controller.entityUpdate( + { entity: "projects", uuid: "asdf" }, + { data: { type: "projects", id: "qwerty", attributes: {} } } + ) + ).rejects.toThrow(BadRequestException); + }); + + it("should throw if the model is not found", async () => { + jest.spyOn(processor, "findOne").mockResolvedValue(null); + await expect( + controller.entityUpdate( + { entity: "projects", uuid: "asdf" }, + { data: { type: "projects", id: "asdf", attributes: {} } } + ) + ).rejects.toThrow(NotFoundException); + }); + + it("authorizes access to the model", async () => { + const project = await ProjectFactory.create(); + processor.findOne.mockResolvedValue(project); + const { id, uuid } = project; + policyService.authorize.mockRejectedValueOnce(new UnauthorizedException()); + await expect( + controller.entityUpdate({ entity: "projects", uuid }, { data: { type: "projects", id: uuid, attributes: {} } }) + ).rejects.toThrow(UnauthorizedException); + + policyService.authorize.mockReset(); + policyService.authorize.mockResolvedValueOnce(undefined); + await controller.entityUpdate( + { entity: "projects", uuid }, + { data: { type: "projects", id: uuid, attributes: {} } } + ); + expect(policyService.authorize).toHaveBeenCalledWith("update", expect.objectContaining({ id, uuid })); + }); + + it("calls update on the processor and creates the DTO", async () => { + const project = await ProjectFactory.create(); + processor.findOne.mockResolvedValue(project); + const { uuid } = project; + policyService.authorize.mockResolvedValueOnce(undefined); + const attributes = { status: "approved", feedback: "foo" }; + await controller.entityUpdate({ entity: "projects", uuid }, { data: { type: "projects", id: uuid, attributes } }); + + expect(processor.update).toHaveBeenCalledWith(project, attributes); + expect(processor.getFullDto).toHaveBeenCalledWith(project); + }); + }); }); diff --git a/apps/entity-service/src/entities/entity-associations.controller.spec.ts b/apps/entity-service/src/entities/entity-associations.controller.spec.ts index 89665613..5db280af 100644 --- a/apps/entity-service/src/entities/entity-associations.controller.spec.ts +++ b/apps/entity-service/src/entities/entity-associations.controller.spec.ts @@ -35,7 +35,6 @@ describe("EntityAssociationsController", () => { }).compile(); controller = module.get(EntityAssociationsController); - // @ts-expect-error union type complexity entitiesService.createAssociationProcessor.mockImplementation((entity, uuid) => { return new StubProcessor(entity, uuid, ProjectReport); }); @@ -50,7 +49,6 @@ describe("EntityAssociationsController", () => { policyService.getPermissions.mockResolvedValue(["view-dashboard"]); const pr = await ProjectReportFactory.create(); const processor = new StubProcessor("projectReports", pr.uuid, ProjectReport); - // @ts-expect-error union type complexity entitiesService.createAssociationProcessor.mockImplementation(() => processor); const spy = jest.spyOn(processor, "getBaseEntity"); await controller.associationIndex({ diff --git a/apps/entity-service/src/entities/processors/entity.processor.spec.ts b/apps/entity-service/src/entities/processors/entity.processor.spec.ts index 5319d748..c93e73a0 100644 --- a/apps/entity-service/src/entities/processors/entity.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/entity.processor.spec.ts @@ -1,21 +1,23 @@ import { ProjectProcessor } from "./project.processor"; import { Test } from "@nestjs/testing"; import { MediaService } from "@terramatch-microservices/common/media/media.service"; -import { createMock } from "@golevelup/ts-jest"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { EntitiesService } from "../entities.service"; import { ProjectFactory } from "@terramatch-microservices/database/factories"; import { ActionFactory } from "@terramatch-microservices/database/factories/action.factory"; import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; +import { UnauthorizedException } from "@nestjs/common"; describe("EntityProcessor", () => { let processor: ProjectProcessor; + let policyService: DeepMocked; beforeEach(async () => { const module = await Test.createTestingModule({ providers: [ { provide: MediaService, useValue: createMock() }, - { provide: PolicyService, useValue: createMock() }, + { provide: PolicyService, useValue: (policyService = createMock()) }, { provide: LocalizationService, useValue: createMock() }, EntitiesService ] @@ -46,4 +48,34 @@ describe("EntityProcessor", () => { } }); }); + + describe("update", () => { + it("calls model.save", async () => { + const project = await ProjectFactory.create(); + const spy = jest.spyOn(project, "save"); + await processor.update(project, {}); + expect(spy).toHaveBeenCalled(); + }); + + it("authorizes for approval when appropriate", async () => { + const project = await ProjectFactory.create({ status: "started", feedback: null, feedbackFields: null }); + + policyService.authorize.mockResolvedValueOnce(undefined); + await processor.update(project, { status: "awaiting-approval", feedback: "foo", feedbackFields: ["bar"] }); + expect(policyService.authorize).not.toHaveBeenCalled(); + // These two should be ignored for non approval statuses + expect(project.feedback).toBeNull(); + expect(project.feedbackFields).toBeNull(); + policyService.authorize.mockReset(); + + policyService.authorize.mockRejectedValueOnce(new UnauthorizedException()); + await expect(processor.update(project, { status: "approved" })).rejects.toThrow(UnauthorizedException); + + policyService.authorize.mockResolvedValueOnce(undefined); + await processor.update(project, { status: "approved", feedback: "foo", feedbackFields: ["bar"] }); + expect(project.status).toEqual("approved"); + expect(project.feedback).toEqual("foo"); + expect(project.feedbackFields).toEqual(["bar"]); + }); + }); }); diff --git a/apps/entity-service/src/entities/processors/project.processor.spec.ts b/apps/entity-service/src/entities/processors/project.processor.spec.ts index 22324fbc..6650e3e2 100644 --- a/apps/entity-service/src/entities/processors/project.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/project.processor.spec.ts @@ -26,12 +26,13 @@ import { flatten, reverse, sortBy, sum, sumBy } from "lodash"; import { DateTime } from "luxon"; import { faker } from "@faker-js/faker"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; -import { BadRequestException } from "@nestjs/common"; +import { BadRequestException, UnauthorizedException } from "@nestjs/common"; import { FULL_TIME, PART_TIME } from "@terramatch-microservices/database/constants/demographic-collections"; import { PolicyService } from "@terramatch-microservices/common"; import { ProjectLightDto } from "../dto/project.dto"; import { buildJsonApi } from "@terramatch-microservices/common/util"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; +import { EntityProcessor } from "./entity-processor"; describe("ProjectProcessor", () => { let processor: ProjectProcessor; @@ -250,6 +251,29 @@ describe("ProjectProcessor", () => { }); }); + describe("update", () => { + it("can update the isTest flag", async () => { + const project = await ProjectFactory.create({ isTest: false }); + policyService.getPermissions.mockResolvedValue(["projects-read"]); + await expect(processor.update(project, { isTest: false })).rejects.toThrow(UnauthorizedException); + expect(project.isTest).toBe(false); + await processor.update(project, {}); + expect(project.isTest).toBe(false); + + policyService.getPermissions.mockResolvedValue([`framework-${project.frameworkKey}`]); + await processor.update(project, { isTest: true }); + expect(project.isTest).toBe(true); + }); + + it("should call super.update", async () => { + const project = await ProjectFactory.create(); + const spy = jest.spyOn(EntityProcessor.prototype, "update"); + const update = { feedback: "foo" }; + await processor.update(project, update); + expect(spy).toHaveBeenCalledWith(project, update); + }); + }); + describe("DTOs", () => { it("includes calculated fields in ProjectLightDto", async () => { const org = await OrganisationFactory.create(); From a032071ae5224bf600011288a27dde9f8b121e6a Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 18 Apr 2025 10:35:53 -0700 Subject: [PATCH 33/98] [TM-1861] Specs for ModelColumnStateMachine --- .github/workflows/pull-request.yml | 2 +- .../util/model-column-state-machine.spec.ts | 112 ++ .../lib/util/model-column-state-machine.ts | 12 +- package-lock.json | 1232 ++++++++++++++++- package.json | 3 +- 5 files changed, 1348 insertions(+), 13 deletions(-) create mode 100644 libs/database/src/lib/util/model-column-state-machine.spec.ts diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index bfe0f591..45a352ba 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -46,7 +46,7 @@ jobs: # First run just the small database test to get the test database synced to the current schema # in a clean way. - name: Sync DB Schema - run: npx nx test database --no-cloud --skip-nx-cache + run: npx nx test database --no-cloud --skip-nx-cache libs/database/src/lib/database.module.spec.ts # All tests are run in band so that wipes of database tables from one test doesn't interfere # with other tests. diff --git a/libs/database/src/lib/util/model-column-state-machine.spec.ts b/libs/database/src/lib/util/model-column-state-machine.spec.ts new file mode 100644 index 00000000..4ae0806c --- /dev/null +++ b/libs/database/src/lib/util/model-column-state-machine.spec.ts @@ -0,0 +1,112 @@ +import { Model, Sequelize, Table } from "sequelize-typescript"; +import { StateMachineColumn, StateMachineException, States } from "./model-column-state-machine"; +import { SEQUELIZE_GLOBAL_HOOKS } from "../sequelize-config.service"; + +const hook = jest.fn(() => undefined); +const transitionValid = jest.fn(() => true); + +type StubStatus = "first" | "second" | "third" | "final"; +const StubStates: States = { + default: "first", + + transitions: { + first: ["second", "final"], + second: ["third", "final"], + third: ["final"] + }, + + transitionValidForModel: transitionValid, + + afterTransitionHooks: { + final: hook + } +}; + +@Table({}) +class StubModel extends Model { + @StateMachineColumn(StubStates) + status: StubStatus; +} + +const sequelize = new Sequelize({ + dialect: "sqlite", + database: "test_db", + storage: ":memory:", + logging: false, + models: [StubModel], + hooks: SEQUELIZE_GLOBAL_HOOKS +}); + +async function createModel(status?: StubStatus) { + const model = new StubModel(); + if (status != null) model.status = status; + await model.save(); + return model; +} + +describe("ModelColumnStateMachine", () => { + beforeAll(async () => { + await sequelize.sync(); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + transitionValid.mockClear(); + hook.mockClear(); + }); + + it("should return the current status string from the model attribute getter", async () => { + expect((await createModel()).status).toBe("first"); + expect((await createModel("second")).status).toBe("second"); + }); + + it("should allow a new model to start with any state", async () => { + expect(() => (new StubModel().status = "third")).not.toThrow(StateMachineException); + }); + + it("should throw when the transition is not defined", async () => { + const model = new StubModel(); + model.isNewRecord = false; + expect(() => (model.status = "third")).toThrow(StateMachineException); + }); + + it("should throw if the current state is not defined in the SM", async () => { + const model = await createModel("foo" as StubStatus); + expect(() => (model.status = "first")).toThrow(StateMachineException); + expect(() => (model.status = "second")).toThrow(StateMachineException); + expect(() => (model.status = "third")).toThrow(StateMachineException); + expect(() => (model.status = "final")).toThrow(StateMachineException); + }); + + it("should serialize to the string value of the state", async () => { + let result = JSON.parse(JSON.stringify(await createModel())); + expect(result.status).toBe("first"); + result = JSON.parse(JSON.stringify(await createModel("final"))); + expect(result.status).toBe("final"); + }); + + it("should allow a transition to the current status", async () => { + const model = await createModel("second"); + expect(() => (model.status = "second")).not.toThrow(StateMachineException); + }); + + it("should check validations with defined validator", async () => { + const model = await createModel(); + model.status = "second"; + expect(transitionValid).toHaveBeenCalledWith("first", "second", model); + + transitionValid.mockClear(); + transitionValid.mockReturnValueOnce(false); + expect(() => (model.status = "third")).toThrow(StateMachineException); + expect(transitionValid).toHaveBeenCalledWith("second", "third", model); + }); + + it("should process after save hooks", async () => { + const model = await createModel("second"); + await model.update({ status: "third" }); + expect(hook).not.toHaveBeenCalled(); + + await model.update({ status: "final" }); + expect(hook).toHaveBeenCalledTimes(1); + }); +}); diff --git a/libs/database/src/lib/util/model-column-state-machine.ts b/libs/database/src/lib/util/model-column-state-machine.ts index afe7db32..bfad12a5 100644 --- a/libs/database/src/lib/util/model-column-state-machine.ts +++ b/libs/database/src/lib/util/model-column-state-machine.ts @@ -100,19 +100,13 @@ export class ModelColumnStateMachine { return this.model.getDataValue(this.column) as S; } - /** - * Allows this object to serialize to the string value of its DB column when included in an API DTO. - */ - toJSON() { - return this.current; - } - canBe(from: S, to: S) { - if (!Object.keys(this.states.transitions).includes(from)) { + const toStates = this.states.transitions[from]; + if (toStates == null) { throw new StateMachineException(`Current state is not defined [${from}]`); } - return from === to || this.states.transitions[from]?.includes(to) === true; + return from === to || toStates.includes(to) === true; } transitionTo(to: S) { diff --git a/package-lock.json b/package-lock.json index a15236cf..0d5a7d4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,7 @@ "jest-environment-node": "^29.7.0", "nx": "20.6.1", "prettier": "^2.6.2", + "sqlite3": "^5.1.7", "supertest": "^7.0.0", "ts-jest": "^29.2.6", "ts-node": "10.9.2", @@ -2792,6 +2793,13 @@ "tslib": "2" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "dev": true, + "optional": true + }, "node_modules/@golevelup/ts-jest": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/@golevelup/ts-jest/-/ts-jest-0.6.2.tgz", @@ -3882,6 +3890,45 @@ "node": ">= 8" } }, + "node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "dev": true, + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "dev": true, + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@npmcli/move-file/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "optional": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@nuxt/opencollective": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@nuxt/opencollective/-/opencollective-0.4.1.tgz", @@ -6650,6 +6697,16 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "license": "MIT" }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/@transifex/native": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@transifex/native/-/native-7.1.3.tgz", @@ -7650,6 +7707,13 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true, + "optional": true + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -7727,6 +7791,46 @@ "node": ">= 10.0.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "dev": true, + "optional": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dev": true, + "optional": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/airtable": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/airtable/-/airtable-0.12.2.tgz", @@ -7900,6 +8004,43 @@ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==" }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "dev": true, + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", + "dev": true, + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/are-we-there-yet/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -8307,6 +8448,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -8566,6 +8716,82 @@ "node": ">= 0.8" } }, + "node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "dev": true, + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/cacache/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cacache/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacache/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "optional": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cacache/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -8698,6 +8924,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/chrome-trace-event": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", @@ -8742,6 +8977,16 @@ "validator": "^13.9.0" } }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/cli-boxes": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", @@ -8854,6 +9099,16 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "dev": true, + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/colord": { "version": "2.9.3", "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", @@ -9031,6 +9286,13 @@ "node": "^14.18.0 || >=16.10.0" } }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "dev": true, + "optional": true + }, "node_modules/content-disposition": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", @@ -9520,6 +9782,21 @@ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.5.0.tgz", "integrity": "sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw==" }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dedent": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", @@ -9534,6 +9811,15 @@ } } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -9606,6 +9892,13 @@ "node": ">=0.4.0" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "dev": true, + "optional": true + }, "node_modules/denque": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", @@ -9907,6 +10200,27 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -9953,6 +10267,16 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/envinfo": { "version": "7.14.0", "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.14.0.tgz", @@ -9965,6 +10289,13 @@ "node": ">=4" } }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "dev": true, + "optional": true + }, "node_modules/errno": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", @@ -10400,6 +10731,15 @@ "node": ">= 0.8.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/expect": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", @@ -10708,6 +11048,12 @@ "url": "https://github.com/sindresorhus/file-type?sponsor=1" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -11164,6 +11510,36 @@ "node": ">=12" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs-minipass/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/fs-monkey": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.6.tgz", @@ -11197,6 +11573,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", + "dev": true, + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/generate-function": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", @@ -11287,6 +11684,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -11497,6 +11900,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "dev": true, + "optional": true + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -11587,6 +11997,13 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "node_modules/http-cache-semantics": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", + "dev": true, + "optional": true + }, "node_modules/http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -11628,6 +12045,21 @@ "node": ">=8.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dev": true, + "optional": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/http-proxy-middleware": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", @@ -11685,6 +12117,20 @@ "node": ">=12" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "optional": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -11694,6 +12140,16 @@ "node": ">=10.17.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/husky": { "version": "9.1.7", "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", @@ -11876,6 +12332,23 @@ "node": ">=0.8.19" } }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "dev": true, + "optional": true + }, "node_modules/inflection": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", @@ -11899,6 +12372,12 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true + }, "node_modules/interpret": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", @@ -11942,6 +12421,27 @@ "url": "https://opencollective.com/ioredis" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dev": true, + "optional": true, + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "optional": true + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -12087,6 +12587,13 @@ "node": ">=8" } }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "dev": true, + "optional": true + }, "node_modules/is-network-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", @@ -12953,6 +13460,13 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "dev": true, + "optional": true + }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -13488,6 +14002,77 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "dev": true, + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/make-fetch-happen/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-fetch-happen/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/make-fetch-happen/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/make-fetch-happen/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, "node_modules/makeerror": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", @@ -13671,6 +14256,18 @@ "node": ">=6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mini-css-extract-plugin": { "version": "2.4.7", "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.4.7.tgz", @@ -13728,6 +14325,207 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-collect/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-collect/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, + "node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/minipass-fetch/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-fetch/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -13739,6 +14537,12 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/module-details-from-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", @@ -13933,6 +14737,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "dev": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -14010,8 +14820,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "dev": true, - "optional": true + "dev": true }, "node_modules/node-fetch": { "version": "2.7.0", @@ -14041,6 +14850,31 @@ "node": ">= 6.13.0" } }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "dev": true, + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, "node_modules/node-gyp-build-optional-packages": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", @@ -14081,6 +14915,22 @@ "node": ">=6.0.0" } }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dev": true, + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -14126,6 +14976,23 @@ "node": ">=8" } }, + "node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", + "dev": true, + "optional": true, + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/nth-check": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", @@ -14470,6 +15337,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "optional": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-queue": { "version": "6.6.2", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", @@ -15471,6 +16354,32 @@ "node": ">=0.10.0" } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "dev": true, + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -15535,6 +16444,37 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "dev": true, + "optional": true + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "dev": true, + "optional": true, + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/promise-retry/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -15572,6 +16512,16 @@ "dev": true, "optional": true }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -15673,6 +16623,30 @@ "node": ">=0.10.0" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", @@ -16912,6 +17886,13 @@ "node": ">= 18" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true, + "optional": true + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -17041,6 +18022,51 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -17056,6 +18082,17 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/sockjs": { "version": "0.3.24", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", @@ -17076,6 +18113,36 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/socks": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.4.tgz", + "integrity": "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==", + "dev": true, + "optional": true, + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "dev": true, + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/source-map": { "version": "0.7.4", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", @@ -17195,6 +18262,30 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/sqlite3": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.7.tgz", + "integrity": "sha512-GGIyOiFaG+TUra3JIfkI/zGP8yZYLPQ0pl1bH+ODjiX57sPhrLU5sQJn1y9bDKZUFYkX1crlrPfSYt0BKKdkog==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, "node_modules/sqlstring": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", @@ -17208,6 +18299,39 @@ "resolved": "https://registry.npmjs.org/ssm-session/-/ssm-session-1.0.6.tgz", "integrity": "sha512-1qUuh1Pw5dR6l6ldZWeNsafHSDoqU8UltyQfJUGDrl7xCsWMlVW+HcygJ1MCMYprxw1vjmhph16QXdxRi56Dig==" }, + "node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "dev": true, + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ssri/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ssri/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "optional": true + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -17635,6 +18759,41 @@ "node": ">=6" } }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "dev": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar-fs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", + "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "dev": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -17665,6 +18824,33 @@ "node": ">= 6" } }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/terser": { "version": "5.39.0", "resolved": "https://registry.npmjs.org/terser/-/terser-5.39.0.tgz", @@ -18090,6 +19276,18 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -18251,6 +19449,26 @@ "node": ">= 0.8.0" } }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "dev": true, + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "dev": true, + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -19268,6 +20486,16 @@ "node": ">= 8" } }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "dev": true, + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "node_modules/widest-line": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", diff --git a/package.json b/package.json index fae62a3b..0e4b25e0 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "fe-services": "nx run-many -t serve --parallel=10 --no-cloud --projects user-service job-service entity-service research-service", "all": "nx run-many --parallel=100 -t serve --no-cloud", "prepare": "husky", - "test:ci": "nx run-many -t test --parallel=1 --no-cloud --coverage --runInBand --passWithNoTests --exclude database", + "test:ci": "nx run-many -t test --parallel=1 --no-cloud --coverage --runInBand --passWithNoTests", "lint-build": "nx run-many -t lint build --no-cloud" }, "private": true, @@ -92,6 +92,7 @@ "jest-environment-node": "^29.7.0", "nx": "20.6.1", "prettier": "^2.6.2", + "sqlite3": "^5.1.7", "supertest": "^7.0.0", "ts-jest": "^29.2.6", "ts-node": "10.9.2", From 1234ca098d94e292cb53fe275c7fbcb2f62f5545 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 18 Apr 2025 12:12:22 -0600 Subject: [PATCH 34/98] [TM-1937] change nothingToReport to nullable --- apps/entity-service/src/entities/dto/site-report.dto.ts | 4 ++-- libs/database/src/lib/entities/site-report.entity.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/entity-service/src/entities/dto/site-report.dto.ts b/apps/entity-service/src/entities/dto/site-report.dto.ts index 3e064682..fb1da270 100644 --- a/apps/entity-service/src/entities/dto/site-report.dto.ts +++ b/apps/entity-service/src/entities/dto/site-report.dto.ts @@ -152,8 +152,8 @@ export class SiteReportFullDto extends SiteReportLightDto { @ApiProperty({ nullable: true }) feedbackFields: string[] | null; - @ApiProperty() - nothingToReport: boolean; + @ApiProperty({ nullable: true }) + nothingToReport: boolean | null; @ApiProperty({ nullable: true }) completion: number | null; diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 11f07dcb..45155618 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -305,7 +305,7 @@ export class SiteReport extends Model { @AllowNull @Column(TINYINT) - nothingToReport: boolean; + nothingToReport: boolean | null; @HasMany(() => TreeSpecies, { foreignKey: "speciesableId", From 996fcaa95905b9b8ba063b65b555de05b02072e1 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 18 Apr 2025 12:47:26 -0700 Subject: [PATCH 35/98] [TM-1861] Fix directions for seeing local emails. --- .env.local.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.local.sample b/.env.local.sample index 120ba928..8f4e8f31 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -28,7 +28,7 @@ AIRTABLE_BASE_ID= SLACK_API_KEY= UDB_SLACK_CHANNEL= -# Points to the mail catcher running from the PHP docker-compose. Visit localhost:8000 to see "sent" emails. +# Points to the mail catcher running from the PHP docker-compose. Visit localhost:1080 to see "sent" emails. MAIL_HOST=localhost MAIL_PORT=1025 MAIL_FROM_ADDRESS=local@terramatch.org From 75007a1c1a1323f3d5bffe22288d6cf0c3a1defb Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 18 Apr 2025 14:00:12 -0600 Subject: [PATCH 36/98] [TM-1937] add nothingToReport to LigthDto --- apps/entity-service/src/entities/dto/nursery-report.dto.ts | 3 +++ apps/entity-service/src/entities/dto/site-report.dto.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/apps/entity-service/src/entities/dto/nursery-report.dto.ts b/apps/entity-service/src/entities/dto/nursery-report.dto.ts index 08268970..38ee638d 100644 --- a/apps/entity-service/src/entities/dto/nursery-report.dto.ts +++ b/apps/entity-service/src/entities/dto/nursery-report.dto.ts @@ -89,6 +89,9 @@ export class NurseryReportLightDto extends EntityDto { @ApiProperty() createdAt: Date; + + @ApiProperty({ nullable: true }) + nothingToReport: boolean | null; } export type AdditionalNurseryReportLightProps = Pick; diff --git a/apps/entity-service/src/entities/dto/site-report.dto.ts b/apps/entity-service/src/entities/dto/site-report.dto.ts index fb1da270..b3240da5 100644 --- a/apps/entity-service/src/entities/dto/site-report.dto.ts +++ b/apps/entity-service/src/entities/dto/site-report.dto.ts @@ -87,6 +87,9 @@ export class SiteReportLightDto extends EntityDto { @ApiProperty() createdAt: Date; + + @ApiProperty({ nullable: true }) + nothingToReport: boolean | null; } export type AdditionalSiteReportLightProps = Pick; From 3ae4c9cc070c96f96fe7c352634b99bcdfaebf58 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 18 Apr 2025 13:04:07 -0700 Subject: [PATCH 37/98] [TM-1861] Specs for the entity status update event processor. --- ...tity-status-update.event-processor.spec.ts | 85 +++++++++++++++++++ .../entity-status-update.event-processor.ts | 2 +- .../src/lib/entities/audit-status.entity.ts | 16 +++- .../src/lib/factories/action.factory.ts | 1 + libs/database/src/lib/factories/index.ts | 1 + .../lib/factories/update-request.factory.ts | 15 ++++ libs/database/src/lib/types/util.ts | 2 +- 7 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 libs/common/src/lib/events/entity-status-update.event-processor.spec.ts create mode 100644 libs/database/src/lib/factories/update-request.factory.ts diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts b/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts new file mode 100644 index 00000000..999dc746 --- /dev/null +++ b/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts @@ -0,0 +1,85 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { EventService } from "./event.service"; +import { EntityStatusUpdate } from "./entity-status-update.event-processor"; +import { ProjectFactory, UpdateRequestFactory, UserFactory } from "@terramatch-microservices/database/factories"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { RequestContext } from "nestjs-request-context"; +import { ActionFactory } from "@terramatch-microservices/database/factories/action.factory"; +import { Action, AuditStatus } from "@terramatch-microservices/database/entities"; +import { faker } from "@faker-js/faker"; + +function mockUserId(userId?: number) { + jest + .spyOn(RequestContext, "currentContext", "get") + .mockReturnValue({ req: { authenticatedUserId: userId }, res: {} }); +} + +describe("EntityStatusUpdate EventProcessor", () => { + let eventService: DeepMocked; + + beforeEach(async () => { + eventService = createMock(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("should avoid status email and actions for non-entities", async () => { + mockUserId(); + const processor = new EntityStatusUpdate(eventService, await UpdateRequestFactory.forProject.create()); + const statusUpdateSpy = jest.spyOn(processor as any, "sendStatusUpdateEmail"); + const updateActionsSpy = jest.spyOn(processor as any, "updateActions"); + const createAuditStatusSpy = jest.spyOn(processor as any, "createAuditStatus"); + await processor.handle(); + expect(createAuditStatusSpy).toHaveBeenCalled(); + expect(updateActionsSpy).not.toHaveBeenCalled(); + expect(statusUpdateSpy).not.toHaveBeenCalled(); + }); + + it("should send a status update email", async () => { + mockUserId(); + const project = await ProjectFactory.create(); + await new EntityStatusUpdate(eventService, project).handle(); + expect(eventService.emailQueue.add).toHaveBeenCalledWith( + "statusUpdate", + expect.objectContaining({ type: "projects", id: project.id }) + ); + }); + + it("should update actions", async () => { + mockUserId(); + const project = await ProjectFactory.create({ status: "approved" }); + const action = await ActionFactory.forProject.create({ targetableId: project.id }); + await new EntityStatusUpdate(eventService, project).handle(); + + const actions = await Action.for(project).findAll(); + expect(actions.length).toBe(1); + expect(actions[0].id).not.toBe(action.id); + expect(actions[0]).toMatchObject({ + status: "pending", + type: "notification", + projectId: project.id, + organisationId: project.organisationId, + title: project.name, + text: "Approved" + }); + }); + + it("should create an audit status", async () => { + const user = await UserFactory.create(); + mockUserId(user.id); + + const feedback = faker.lorem.sentence(); + const project = await ProjectFactory.create({ status: "approved", feedback }); + await new EntityStatusUpdate(eventService, project).handle(); + + const auditStatus = await AuditStatus.for(project).findOne({ order: [["createdAt", "DESC"]] }); + expect(auditStatus).toMatchObject({ + createdBy: user.emailAddress, + firstName: user.firstName, + lastName: user.lastName, + comment: `Approved: ${feedback}` + }); + }); +}); diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index cc8faaea..7adb7692 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -116,7 +116,7 @@ export class EntityStatusUpdate extends EventProcessor { isEmpty(feedbackFields) ? [] : await FormQuestion.findAll({ - where: { uuid: { [Op.in]: this.model.feedbackFields } }, + where: { uuid: { [Op.in]: feedbackFields as string[] } }, attributes: ["label"] }), "label" diff --git a/libs/database/src/lib/entities/audit-status.entity.ts b/libs/database/src/lib/entities/audit-status.entity.ts index 2175d559..06494601 100644 --- a/libs/database/src/lib/entities/audit-status.entity.ts +++ b/libs/database/src/lib/entities/audit-status.entity.ts @@ -1,9 +1,19 @@ -import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Scopes, Table } from "sequelize-typescript"; import { BIGINT, BOOLEAN, DATE, ENUM, NOW, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; +import { LaravelModel, laravelType } from "../types/util"; +import { chainScope } from "../util/chain-scope"; const TYPES = ["change-request", "status", "submission", "comment", "change-request-updated", "reminder-sent"] as const; type AuditStatusType = (typeof TYPES)[number]; +@Scopes(() => ({ + auditable: (auditable: LaravelModel) => ({ + where: { + auditableType: laravelType(auditable), + auditableId: auditable.id + } + }) +})) @Table({ tableName: "audit_statuses", underscored: true, @@ -12,6 +22,10 @@ type AuditStatusType = (typeof TYPES)[number]; indexes: [{ name: "audit_statuses_auditable_type_auditable_id_index", fields: ["auditable_type", "auditable_id"] }] }) export class AuditStatus extends Model { + static for(auditable: LaravelModel) { + return chainScope(this, "auditable", auditable) as typeof AuditStatus; + } + @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) diff --git a/libs/database/src/lib/factories/action.factory.ts b/libs/database/src/lib/factories/action.factory.ts index 999660ed..d733404c 100644 --- a/libs/database/src/lib/factories/action.factory.ts +++ b/libs/database/src/lib/factories/action.factory.ts @@ -4,6 +4,7 @@ import { OrganisationFactory } from "./organisation.factory"; import { ProjectFactory } from "./project.factory"; const defaultAttributesFactory = async () => ({ + type: "notification", organisationId: OrganisationFactory.associate("id"), projectId: ProjectFactory.associate("id") }); diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts index fbf5dea0..0837714d 100644 --- a/libs/database/src/lib/factories/index.ts +++ b/libs/database/src/lib/factories/index.ts @@ -28,5 +28,6 @@ export * from "./site-polygon.factory"; export * from "./site-report.factory"; export * from "./tree-species.factory"; export * from "./tree-species-research.factory"; +export * from "./update-request.factory"; export * from "./user.factory"; export * from "./task.factory"; diff --git a/libs/database/src/lib/factories/update-request.factory.ts b/libs/database/src/lib/factories/update-request.factory.ts new file mode 100644 index 00000000..0290d9ba --- /dev/null +++ b/libs/database/src/lib/factories/update-request.factory.ts @@ -0,0 +1,15 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { Project, UpdateRequest } from "../entities"; +import { ProjectFactory } from "./project.factory"; + +const defaultAttributesFactory = async () => ({ + status: "awaiting-approval" +}); + +export const UpdateRequestFactory = { + forProject: FactoryGirl.define(UpdateRequest, async () => ({ + ...(await defaultAttributesFactory()), + updateRequestableType: Project.LARAVEL_TYPE, + updateRequestableId: ProjectFactory.associate("id") + })) +}; diff --git a/libs/database/src/lib/types/util.ts b/libs/database/src/lib/types/util.ts index 3ec6d35b..ff55d54c 100644 --- a/libs/database/src/lib/types/util.ts +++ b/libs/database/src/lib/types/util.ts @@ -9,4 +9,4 @@ export const laravelType = (model: LaravelModel) => (model.constructor as Larave export type StatusModel = Model & { status: string }; -export type FeedbackModel = Model & { feedback?: string; feedbackFields?: string[] }; +export type FeedbackModel = Model & { feedback?: string | null; feedbackFields?: string[] | null }; From 5afae3998e0d79aaca1e7ed77c9379a154bfc5c8 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 18 Apr 2025 14:09:19 -0700 Subject: [PATCH 38/98] [TM-1861] Handle status updates for site and nursery --- .../common/src/lib/policies/nursery.policy.ts | 9 +++---- libs/common/src/lib/policies/site.policy.ts | 9 +++---- libs/database/src/lib/constants/status.ts | 25 +++++++++++++------ .../src/lib/entities/nursery.entity.ts | 5 ++-- libs/database/src/lib/entities/site.entity.ts | 11 ++++++-- .../util/model-column-state-machine.spec.ts | 11 ++++---- .../lib/util/model-column-state-machine.ts | 19 +++++++++++++- 7 files changed, 58 insertions(+), 31 deletions(-) diff --git a/libs/common/src/lib/policies/nursery.policy.ts b/libs/common/src/lib/policies/nursery.policy.ts index 21c10313..d6c4479a 100644 --- a/libs/common/src/lib/policies/nursery.policy.ts +++ b/libs/common/src/lib/policies/nursery.policy.ts @@ -9,8 +9,7 @@ export class NurseryPolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can("read", Nursery, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", Nursery, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], Nursery, { frameworkKey: { $in: this.frameworks } }); } if (this.permissions.includes("manage-own")) { @@ -24,8 +23,7 @@ export class NurseryPolicy extends UserPermissionsPolicy { ...user.projects.map(({ id }) => id) ]; if (projectIds.length > 0) { - this.builder.can("read", Nursery, { projectId: { $in: projectIds } }); - this.builder.can("delete", Nursery, { projectId: { $in: projectIds } }); + this.builder.can(["read", "delete", "update"], Nursery, { projectId: { $in: projectIds } }); } } } @@ -35,8 +33,7 @@ export class NurseryPolicy extends UserPermissionsPolicy { if (user != null) { const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); if (projectIds.length > 0) { - this.builder.can("read", Nursery, { projectId: { $in: projectIds } }); - this.builder.can("delete", Nursery, { projectId: { $in: projectIds } }); + this.builder.can(["read", "delete", "update", "approve"], Nursery, { projectId: { $in: projectIds } }); } } } diff --git a/libs/common/src/lib/policies/site.policy.ts b/libs/common/src/lib/policies/site.policy.ts index 4e3af091..dd0c1c53 100644 --- a/libs/common/src/lib/policies/site.policy.ts +++ b/libs/common/src/lib/policies/site.policy.ts @@ -9,8 +9,7 @@ export class SitePolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can("read", Site, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", Site, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], Site, { frameworkKey: { $in: this.frameworks } }); } if (this.permissions.includes("manage-own")) { @@ -24,8 +23,7 @@ export class SitePolicy extends UserPermissionsPolicy { ...user.projects.map(({ id }) => id) ]; if (projectIds.length > 0) { - this.builder.can("read", Site, { projectId: { $in: projectIds } }); - this.builder.can("delete", Site, { projectId: { $in: projectIds } }); + this.builder.can(["read", "delete", "update"], Site, { projectId: { $in: projectIds } }); } } } @@ -35,8 +33,7 @@ export class SitePolicy extends UserPermissionsPolicy { if (user != null) { const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); if (projectIds.length > 0) { - this.builder.can("read", Site, { projectId: { $in: projectIds } }); - this.builder.can("delete", Site, { projectId: { $in: projectIds } }); + this.builder.can(["read", "delete", "update", "approve"], Site, { projectId: { $in: projectIds } }); } } } diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 2ac085ab..21391c5c 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -1,5 +1,5 @@ -import { States } from "../util/model-column-state-machine"; -import { Nursery, Project } from "../entities"; +import { States, transitions } from "../util/model-column-state-machine"; +import { Nursery, Project, Site } from "../entities"; import { Model } from "sequelize-typescript"; import { DatabaseModule } from "../database.module"; @@ -18,12 +18,11 @@ const emitStatusUpdateHook = (from: string, model: Model) => { export const EntityStatusStates: States = { default: STARTED, - transitions: { - [STARTED]: [AWAITING_APPROVAL], - [AWAITING_APPROVAL]: [APPROVED, NEEDS_MORE_INFORMATION], - [NEEDS_MORE_INFORMATION]: [APPROVED, AWAITING_APPROVAL], - [APPROVED]: [NEEDS_MORE_INFORMATION] - }, + transitions: transitions() + .from(STARTED, () => [AWAITING_APPROVAL]) + .from(AWAITING_APPROVAL, () => [APPROVED, NEEDS_MORE_INFORMATION]) + .from(NEEDS_MORE_INFORMATION, () => [APPROVED, AWAITING_APPROVAL]) + .from(APPROVED, () => [NEEDS_MORE_INFORMATION]).transitions, afterTransitionHooks: { [APPROVED]: emitStatusUpdateHook, @@ -35,6 +34,16 @@ export const EntityStatusStates: States = { export const SITE_STATUSES = [...ENTITY_STATUSES, RESTORATION_IN_PROGRESS] as const; export type SiteStatus = (typeof SITE_STATUSES)[number]; +export const SiteStatusStates: States = { + ...(EntityStatusStates as unknown as States), + + transitions: transitions(EntityStatusStates.transitions) + .from(AWAITING_APPROVAL, to => [...to, RESTORATION_IN_PROGRESS]) + .from(NEEDS_MORE_INFORMATION, to => [...to, RESTORATION_IN_PROGRESS]) + .from(APPROVED, to => [...to, RESTORATION_IN_PROGRESS]) + .from(RESTORATION_IN_PROGRESS, () => [NEEDS_MORE_INFORMATION, APPROVED]).transitions +}; + export const DUE = "due"; export const REPORT_STATUSES = [DUE, ...ENTITY_STATUSES] as const; export type ReportStatus = (typeof REPORT_STATUSES)[number]; diff --git a/libs/database/src/lib/entities/nursery.entity.ts b/libs/database/src/lib/entities/nursery.entity.ts index 1af7613d..dffed98e 100644 --- a/libs/database/src/lib/entities/nursery.entity.ts +++ b/libs/database/src/lib/entities/nursery.entity.ts @@ -15,11 +15,12 @@ import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize import { Project } from "./project.entity"; import { TreeSpecies } from "./tree-species.entity"; import { NurseryReport } from "./nursery-report.entity"; -import { EntityStatus, UpdateRequestStatus } from "../constants/status"; +import { EntityStatus, EntityStatusStates, UpdateRequestStatus } from "../constants/status"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { FrameworkKey } from "../constants/framework"; import { JsonColumn } from "../decorators/json-column.decorator"; +import { StateMachineColumn } from "../util/model-column-state-machine"; // Incomplete stub @Scopes(() => ({ @@ -59,7 +60,7 @@ export class Nursery extends Model { @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; - @Column(STRING) + @StateMachineColumn(EntityStatusStates) status: EntityStatus; @AllowNull diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts index 56452ef3..5a2a2661 100644 --- a/libs/database/src/lib/entities/site.entity.ts +++ b/libs/database/src/lib/entities/site.entity.ts @@ -16,7 +16,13 @@ import { TreeSpecies } from "./tree-species.entity"; import { SiteReport } from "./site-report.entity"; import { Project } from "./project.entity"; import { SitePolygon } from "./site-polygon.entity"; -import { APPROVED, RESTORATION_IN_PROGRESS, SiteStatus, UpdateRequestStatus } from "../constants/status"; +import { + APPROVED, + RESTORATION_IN_PROGRESS, + SiteStatus, + SiteStatusStates, + UpdateRequestStatus +} from "../constants/status"; import { SitingStrategy } from "../constants/entity-selects"; import { Seeding } from "./seeding.entity"; import { FrameworkKey } from "../constants/framework"; @@ -24,6 +30,7 @@ import { Framework } from "./framework.entity"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { JsonColumn } from "../decorators/json-column.decorator"; +import { StateMachineColumn } from "../util/model-column-state-machine"; // Incomplete stub @Scopes(() => ({ @@ -71,7 +78,7 @@ export class Site extends Model { @Column(STRING) name: string; - @Column(STRING) + @StateMachineColumn(SiteStatusStates) status: SiteStatus; @AllowNull diff --git a/libs/database/src/lib/util/model-column-state-machine.spec.ts b/libs/database/src/lib/util/model-column-state-machine.spec.ts index 4ae0806c..e9f19c7b 100644 --- a/libs/database/src/lib/util/model-column-state-machine.spec.ts +++ b/libs/database/src/lib/util/model-column-state-machine.spec.ts @@ -1,5 +1,5 @@ import { Model, Sequelize, Table } from "sequelize-typescript"; -import { StateMachineColumn, StateMachineException, States } from "./model-column-state-machine"; +import { StateMachineColumn, StateMachineException, States, transitions } from "./model-column-state-machine"; import { SEQUELIZE_GLOBAL_HOOKS } from "../sequelize-config.service"; const hook = jest.fn(() => undefined); @@ -9,11 +9,10 @@ type StubStatus = "first" | "second" | "third" | "final"; const StubStates: States = { default: "first", - transitions: { - first: ["second", "final"], - second: ["third", "final"], - third: ["final"] - }, + transitions: transitions() + .from("first", () => ["second", "final"]) + .from("second", () => ["third", "final"]) + .from("third", () => ["final"]).transitions, transitionValidForModel: transitionValid, diff --git a/libs/database/src/lib/util/model-column-state-machine.ts b/libs/database/src/lib/util/model-column-state-machine.ts index bfad12a5..84418b9f 100644 --- a/libs/database/src/lib/util/model-column-state-machine.ts +++ b/libs/database/src/lib/util/model-column-state-machine.ts @@ -2,9 +2,11 @@ import { Column, Model } from "sequelize-typescript"; import { Attributes, STRING } from "sequelize"; import { HttpException, HttpStatus } from "@nestjs/common"; +type Transitions = Partial>; + export type States = { default: S; - transitions: Partial>; + transitions: Transitions; /** * If specified, this function can be used to perform extra validations for a transition within @@ -29,6 +31,21 @@ export type States = { afterTransitionHooks?: Partial void>>; }; +/** + * A simple builder mechanism to make it more readable and concise to extend one set of states for another set of states. + */ +class TransitionBuilder { + constructor(public transitions: Transitions = {}) {} + + from(from: S, to: (current: S[]) => S[]) { + this.transitions[from] = to(this.transitions[from] ?? []); + return this; + } +} + +export const transitions = (transitions: Transitions = {}) => + new TransitionBuilder(transitions); + // Extends HttpException so these errors bubble up to the API consumer export class StateMachineException extends HttpException { constructor(message: string) { From 521775ea9324fa52b76b91c83583880d8f627724 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 21 Apr 2025 10:58:19 -0700 Subject: [PATCH 39/98] [TM-1960] To get a true count, we need to modify with distinct. --- libs/database/src/lib/util/paginated-query.builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/database/src/lib/util/paginated-query.builder.ts b/libs/database/src/lib/util/paginated-query.builder.ts index 85b7dd6a..83bb1caf 100644 --- a/libs/database/src/lib/util/paginated-query.builder.ts +++ b/libs/database/src/lib/util/paginated-query.builder.ts @@ -77,6 +77,6 @@ export class PaginatedQueryBuilder> { } async paginationTotal() { - return await this.MODEL.count(this.findOptions); + return await this.MODEL.count({ distinct: true, ...this.findOptions }); } } From 08d6f8fb2ec88edfab0a8e602c9901917390a168 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 21 Apr 2025 11:50:48 -0700 Subject: [PATCH 40/98] [TM-1955] All columns for Task defined. --- bin/setup-test-database.sh | 4 +- libs/database/src/lib/constants/status.ts | 3 ++ .../src/lib/entities/nursery-report.entity.ts | 6 +-- .../src/lib/entities/project-report.entity.ts | 6 +-- .../src/lib/entities/site-report.entity.ts | 6 +-- libs/database/src/lib/entities/task.entity.ts | 39 ++++++++++++++++--- 6 files changed, 48 insertions(+), 16 deletions(-) diff --git a/bin/setup-test-database.sh b/bin/setup-test-database.sh index 59d6c144..795780f1 100755 --- a/bin/setup-test-database.sh +++ b/bin/setup-test-database.sh @@ -1,7 +1,7 @@ #!/bin/bash pushd ../wri-terramatch-api -cat <<- SQL | docker-compose exec -T mariadb mysql -h localhost -u root -proot +cat <<- SQL | docker compose exec -T mariadb mysql -h localhost -u root -proot drop database if exists terramatch_microservices_test; create database terramatch_microservices_test; grant all on terramatch_microservices_test.* to 'wri'@'%'; @@ -9,5 +9,5 @@ SQL popd # Sync the DB schema -nx test database --no-cloud --skip-nx-cache +nx test database --no-cloud --skip-nx-cache libs/database/src/lib/database.module.spec.ts diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 21391c5c..8d56f547 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -50,6 +50,9 @@ export type ReportStatus = (typeof REPORT_STATUSES)[number]; export const COMPLETE_REPORT_STATUSES = [APPROVED, AWAITING_APPROVAL] as const; +export const TASK_STATUSES = [DUE, NEEDS_MORE_INFORMATION, AWAITING_APPROVAL, APPROVED] as const; +export type TaskStatus = (typeof TASK_STATUSES)[number]; + export const DRAFT = "draft"; export const NO_UPDATE = "no-update"; export const UPDATE_REQUEST_STATUSES = [NO_UPDATE, DRAFT, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index 2bf907cb..5022b886 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -103,9 +103,6 @@ export class NurseryReport extends Model { @BelongsTo(() => User, { foreignKey: "approvedBy", as: "approvedByUser" }) approvedByUser: User | null; - @BelongsTo(() => Task) - task: Task | null; - get projectName() { return this.nursery?.project?.name; } @@ -155,6 +152,9 @@ export class NurseryReport extends Model { @Column(BIGINT.UNSIGNED) taskId: number; + @BelongsTo(() => Task, { constraints: false }) + task: Task | null; + @Column(STRING) status: ReportStatus; diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index a1a87648..ef02dc8a 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -131,9 +131,6 @@ export class ProjectReport extends Model { @BelongsTo(() => User) user: User | null; - @BelongsTo(() => Task) - task: Task | null; - get projectName() { return this.project?.name; } @@ -163,6 +160,9 @@ export class ProjectReport extends Model { @Column(BIGINT.UNSIGNED) taskId: number; + @BelongsTo(() => Task, { constraints: false }) + task: Task | null; + @Column(STRING) status: string; diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 19e03534..a048dcab 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -116,9 +116,6 @@ export class SiteReport extends Model { @BelongsTo(() => User, { foreignKey: "approvedBy", as: "approvedByUser" }) approvedByUser: User | null; - @BelongsTo(() => Task) - task: Task | null; - @ForeignKey(() => User) @Column(BIGINT.UNSIGNED) createdBy: number; @@ -132,6 +129,9 @@ export class SiteReport extends Model { @Column(BIGINT.UNSIGNED) taskId: number; + @BelongsTo(() => Task, { constraints: false }) + task: Task | null; + get projectName() { return this.site?.project?.name; } diff --git a/libs/database/src/lib/entities/task.entity.ts b/libs/database/src/lib/entities/task.entity.ts index feba4a7a..52138f99 100644 --- a/libs/database/src/lib/entities/task.entity.ts +++ b/libs/database/src/lib/entities/task.entity.ts @@ -1,6 +1,8 @@ -import { AutoIncrement, Column, HasOne, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, UUID, UUIDV4 } from "sequelize"; -import { ProjectReport } from "./project-report.entity"; +import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, DATE, STRING, UUID, UUIDV4 } from "sequelize"; +import { Organisation } from "./organisation.entity"; +import { Project } from "./project.entity"; +import { DUE, TaskStatus } from "../constants/status"; @Table({ tableName: "v2_tasks", underscored: true, paranoid: true }) export class Task extends Model { @@ -15,6 +17,33 @@ export class Task extends Model { @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; - @HasOne(() => ProjectReport) - projectReport: ProjectReport | null; + @AllowNull + @ForeignKey(() => Organisation) + @Column(BIGINT.UNSIGNED) + organisationId: number | null; + + @AllowNull + @ForeignKey(() => Project) + @Column(BIGINT.UNSIGNED) + projectId: number | null; + + /** @deprecated this field is null for all rows in the production DB. */ + @AllowNull + @Column(STRING) + title: string | null; + + // Note: this column is marked nullable in the DB, but in fact no rows are null, and we should + // make that a real constraint when the schema is controlled by v3 code. + @Column({ type: STRING, defaultValue: DUE }) + status: TaskStatus; + + // Note: this column is marked nullable in the DB, but in fact no rows are null, and we should + // make that a real constraint when the schema is controlled by v3 code. + @Column(STRING) + periodKey: string; + + // Note: this column is marked nullable in the DB, but in fact no rows are null, and we should + // make that a real constraint when the schema is controlled by v3 code. + @Column(DATE) + dueAt: Date; } From 3c4d34e413548a64c4bc4aad8318ca53fdf75f16 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 21 Apr 2025 12:25:13 -0700 Subject: [PATCH 41/98] [TM-1955] Define the state machines for tasks and reports. --- libs/database/src/lib/constants/status.ts | 33 ++++++++++++++++++- .../src/lib/entities/nursery-report.entity.ts | 5 +-- .../src/lib/entities/project-report.entity.ts | 7 ++-- .../src/lib/entities/site-report.entity.ts | 7 ++-- libs/database/src/lib/entities/task.entity.ts | 7 ++-- 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 8d56f547..a1abe88c 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -1,7 +1,8 @@ import { States, transitions } from "../util/model-column-state-machine"; -import { Nursery, Project, Site } from "../entities"; +import { Nursery, Project, ProjectReport, Site, Task } from "../entities"; import { Model } from "sequelize-typescript"; import { DatabaseModule } from "../database.module"; +import { ReportModel } from "./entities"; export const STARTED = "started"; export const AWAITING_APPROVAL = "awaiting-approval"; @@ -48,11 +49,41 @@ export const DUE = "due"; export const REPORT_STATUSES = [DUE, ...ENTITY_STATUSES] as const; export type ReportStatus = (typeof REPORT_STATUSES)[number]; +export const ReportStatusStates: States = { + ...(EntityStatusStates as unknown as States), + + default: DUE, + + transitions: transitions(EntityStatusStates.transitions) + .from(DUE, () => [STARTED, AWAITING_APPROVAL]) + // reports can go from awaiting approval to started in the nothing_to_report case (see validation below) + .from(AWAITING_APPROVAL, to => [...to, STARTED]).transitions, + + transitionValidForModel: (from: ReportStatus, to: ReportStatus, report: ReportModel) => { + if ((from === DUE && to === AWAITING_APPROVAL) || (from === AWAITING_APPROVAL && to === STARTED)) { + // these two transitions are only allowed for site / nursery reports when the nothingToReport flag is true; + return !(report instanceof ProjectReport) && report.nothingToReport; + } + + return true; + } +}; + export const COMPLETE_REPORT_STATUSES = [APPROVED, AWAITING_APPROVAL] as const; export const TASK_STATUSES = [DUE, NEEDS_MORE_INFORMATION, AWAITING_APPROVAL, APPROVED] as const; export type TaskStatus = (typeof TASK_STATUSES)[number]; +export const TaskStatusStates: States = { + default: DUE, + + transitions: transitions() + .from(DUE, () => [AWAITING_APPROVAL]) + .from(AWAITING_APPROVAL, () => [NEEDS_MORE_INFORMATION, APPROVED]) + .from(NEEDS_MORE_INFORMATION, () => [AWAITING_APPROVAL, APPROVED]) + .from(APPROVED, () => [AWAITING_APPROVAL, NEEDS_MORE_INFORMATION]).transitions +}; + export const DRAFT = "draft"; export const NO_UPDATE = "no-update"; export const UPDATE_REQUEST_STATUSES = [NO_UPDATE, DRAFT, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index 5022b886..260315f1 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -14,7 +14,7 @@ import { import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; import { Nursery } from "./nursery.entity"; import { TreeSpecies } from "./tree-species.entity"; -import { COMPLETE_REPORT_STATUSES, ReportStatus, UpdateRequestStatus } from "../constants/status"; +import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates, UpdateRequestStatus } from "../constants/status"; import { FrameworkKey } from "../constants/framework"; import { Literal } from "sequelize/types/utils"; import { chainScope } from "../util/chain-scope"; @@ -22,6 +22,7 @@ import { Subquery } from "../util/subquery.builder"; import { User } from "./user.entity"; import { JsonColumn } from "../decorators/json-column.decorator"; import { Task } from "./task.entity"; +import { StateMachineColumn } from "../util/model-column-state-machine"; // Incomplete stub @Scopes(() => ({ @@ -155,7 +156,7 @@ export class NurseryReport extends Model { @BelongsTo(() => Task, { constraints: false }) task: Task | null; - @Column(STRING) + @StateMachineColumn(ReportStatusStates) status: ReportStatus; @AllowNull diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index ef02dc8a..6d1773e2 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -15,7 +15,7 @@ import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 import { TreeSpecies } from "./tree-species.entity"; import { Project } from "./project.entity"; import { FrameworkKey } from "../constants/framework"; -import { COMPLETE_REPORT_STATUSES } from "../constants/status"; +import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates } from "../constants/status"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { Framework } from "./framework.entity"; @@ -23,6 +23,7 @@ import { SiteReport } from "./site-report.entity"; import { Literal } from "sequelize/types/utils"; import { User } from "./user.entity"; import { Task } from "./task.entity"; +import { StateMachineColumn } from "../util/model-column-state-machine"; type ApprovedIdsSubqueryOptions = { dueAfter?: string | Date; @@ -163,8 +164,8 @@ export class ProjectReport extends Model { @BelongsTo(() => Task, { constraints: false }) task: Task | null; - @Column(STRING) - status: string; + @StateMachineColumn(ReportStatusStates) + status: ReportStatus; @AllowNull @Column(STRING) diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index a048dcab..ca0cc123 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -17,12 +17,13 @@ import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; import { FrameworkKey } from "../constants/framework"; import { Literal } from "sequelize/types/utils"; -import { COMPLETE_REPORT_STATUSES } from "../constants/status"; +import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates } from "../constants/status"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { Task } from "./task.entity"; import { User } from "./user.entity"; import { JsonColumn } from "../decorators/json-column.decorator"; +import { StateMachineColumn } from "../util/model-column-state-machine"; type ApprovedIdsSubqueryOptions = { dueAfter?: string | Date; @@ -176,8 +177,8 @@ export class SiteReport extends Model { return this.approvedByUser?.lastName; } - @Column(STRING) - status: string; + @StateMachineColumn(ReportStatusStates) + status: ReportStatus; @AllowNull @Column(STRING) diff --git a/libs/database/src/lib/entities/task.entity.ts b/libs/database/src/lib/entities/task.entity.ts index 52138f99..f8cae190 100644 --- a/libs/database/src/lib/entities/task.entity.ts +++ b/libs/database/src/lib/entities/task.entity.ts @@ -2,7 +2,8 @@ import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, import { BIGINT, DATE, STRING, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { Project } from "./project.entity"; -import { DUE, TaskStatus } from "../constants/status"; +import { TaskStatus, TaskStatusStates } from "../constants/status"; +import { StateMachineColumn } from "../util/model-column-state-machine"; @Table({ tableName: "v2_tasks", underscored: true, paranoid: true }) export class Task extends Model { @@ -32,9 +33,7 @@ export class Task extends Model { @Column(STRING) title: string | null; - // Note: this column is marked nullable in the DB, but in fact no rows are null, and we should - // make that a real constraint when the schema is controlled by v3 code. - @Column({ type: STRING, defaultValue: DUE }) + @StateMachineColumn(TaskStatusStates) status: TaskStatus; // Note: this column is marked nullable in the DB, but in fact no rows are null, and we should From 41dc299cd7b2d5752c44f89b4c288138e6b323fb Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 21 Apr 2025 15:11:24 -0700 Subject: [PATCH 42/98] [TM-1955] Implement task status update when a report is updated. --- .../entities/processors/entity-processor.ts | 2 +- .../entity-status-update.event-processor.ts | 81 +++++++++++++++++-- libs/common/src/lib/events/event.service.ts | 3 +- .../src/lib/entities/nursery-report.entity.ts | 2 +- .../src/lib/entities/project-report.entity.ts | 11 ++- .../src/lib/entities/site-report.entity.ts | 6 +- libs/database/src/lib/entities/task.entity.ts | 25 +++++- libs/database/src/lib/types/util.ts | 2 + 8 files changed, 115 insertions(+), 17 deletions(-) diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index c7a7cc1e..e2cdc702 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -128,7 +128,7 @@ export abstract class EntityProcessor< model.feedbackFields = update.feedbackFields; } - model.status = update.status; + model.status = update.status as ModelType["status"]; } await model.save(); diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index 7adb7692..6e2eddcd 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -1,4 +1,4 @@ -import { FeedbackModel, LaravelModel, laravelType, StatusModel } from "@terramatch-microservices/database/types/util"; +import { laravelType, StatusUpdateModel } from "@terramatch-microservices/database/types/util"; import { EventProcessor } from "./event.processor"; import { TMLogger } from "../util/tm-logger"; import { @@ -7,16 +7,23 @@ import { EntityType, getOrganisationId, getProjectId, - isReport + isReport, + ReportModel } from "@terramatch-microservices/database/constants/entities"; import { StatusUpdateData } from "../email/email.processor"; import { EventService } from "./event.service"; -import { Action, AuditStatus, FormQuestion } from "@terramatch-microservices/database/entities"; -import { get, isEmpty, map } from "lodash"; +import { Action, AuditStatus, FormQuestion, Task } from "@terramatch-microservices/database/entities"; +import { flatten, get, isEmpty, map, uniq } from "lodash"; import { Op } from "sequelize"; -import { STATUS_DISPLAY_STRINGS } from "@terramatch-microservices/database/constants/status"; - -export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; +import { + APPROVED, + AWAITING_APPROVAL, + DUE, + NEEDS_MORE_INFORMATION, + STARTED, + STATUS_DISPLAY_STRINGS +} from "@terramatch-microservices/database/constants/status"; +import { InternalServerErrorException } from "@nestjs/common"; export class EntityStatusUpdate extends EventProcessor { private readonly logger = new TMLogger(EntityStatusUpdate.name); @@ -43,6 +50,10 @@ export class EntityStatusUpdate extends EventProcessor { await this.updateActions(); } await this.createAuditStatus(); + + if (entityType != null && isReport(this.model as EntityModel)) { + await this.checkTaskStatus(); + } } private async sendStatusUpdateEmail(type: EntityType) { @@ -125,4 +136,60 @@ export class EntityStatusUpdate extends EventProcessor { feedback ?? "(No feedback)" }`; } + + async checkTaskStatus() { + const { taskId } = this.model as ReportModel; + if (taskId == null) { + this.logger.warn(`No task found for status changed report [${this.model.constructor.name}, ${this.model.id}]`); + return; + } + + const attributes = ["id", "status", "updateRequestStatus"]; + const task = await Task.findOne({ + where: { id: taskId }, + include: [ + { association: "projectReport", attributes }, + { association: "siteReports", attributes }, + { association: "nurseryReports", attributes } + ] + }); + if (task == null) { + this.logger.error(`No task found for task id [${taskId}]`); + return; + } + + if (task.status === DUE) { + // No further processing needed; nothing automatic happens until the task has been submitted. + return; + } + + const reports = flatten([task.projectReport, task.siteReports, task.nurseryReports]).filter( + report => report != null + ); + + const reportStatuses = uniq(reports.map(({ status }) => status)); + if (reportStatuses.length === 1 && reportStatuses[0] === APPROVED) { + await task.update({ status: "approved" }); + return; + } + + if (reportStatuses.includes(DUE) || reportStatuses.includes(STARTED)) { + throw new InternalServerErrorException(`Task has unsubmitted reports [${taskId}]`); + } + + const moreInfoReport = reports.find( + ({ status, updateRequestStatus }) => + (status === NEEDS_MORE_INFORMATION && updateRequestStatus !== AWAITING_APPROVAL) || + updateRequestStatus === NEEDS_MORE_INFORMATION + ); + if (moreInfoReport != null) { + // A report in needs-more-information causes the task to go to needs-more-information + await task.update({ status: "needs-more-information" }); + return; + } + + // If there are no reports or update requests in needs-more-information, the only option left is that + // something is in awaiting-approval. + await task.update({ status: "awaiting-approval" }); + } } diff --git a/libs/common/src/lib/events/event.service.ts b/libs/common/src/lib/events/event.service.ts index 751a41d3..688ca38d 100644 --- a/libs/common/src/lib/events/event.service.ts +++ b/libs/common/src/lib/events/event.service.ts @@ -2,7 +2,8 @@ import { Injectable } from "@nestjs/common"; import { OnEvent } from "@nestjs/event-emitter"; import { InjectQueue } from "@nestjs/bullmq"; import { Queue } from "bullmq"; -import { EntityStatusUpdate, StatusUpdateModel } from "./entity-status-update.event-processor"; +import { EntityStatusUpdate } from "./entity-status-update.event-processor"; +import { StatusUpdateModel } from "@terramatch-microservices/database/types/util"; /** * A service to handle general events that are emitted in the common or database libraries, and diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index 260315f1..f7358cd8 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -29,7 +29,7 @@ import { StateMachineColumn } from "../util/model-column-state-machine"; incomplete: { where: { status: { [Op.notIn]: COMPLETE_REPORT_STATUSES } } }, nurseries: (ids: number[] | Literal) => ({ where: { nurseryId: { [Op.in]: ids } } }), approved: { where: { status: { [Op.in]: NurseryReport.APPROVED_STATUSES } } }, - task: (taskId: number) => ({ where: { taskId: taskId } }) + task: (taskId: number) => ({ where: { taskId } }) })) @Table({ tableName: "v2_nursery_reports", underscored: true, paranoid: true }) export class NurseryReport extends Model { diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index 6d1773e2..26a7c477 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -15,7 +15,7 @@ import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID, UUIDV4 import { TreeSpecies } from "./tree-species.entity"; import { Project } from "./project.entity"; import { FrameworkKey } from "../constants/framework"; -import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates } from "../constants/status"; +import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates, UpdateRequestStatus } from "../constants/status"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { Framework } from "./framework.entity"; @@ -35,7 +35,8 @@ type ApprovedIdsSubqueryOptions = { incomplete: { where: { status: { [Op.notIn]: COMPLETE_REPORT_STATUSES } } }, approved: { where: { status: { [Op.in]: ProjectReport.APPROVED_STATUSES } } }, project: (id: number) => ({ where: { projectId: id } }), - dueBefore: (date: Date | string) => ({ where: { dueAt: { [Op.lt]: date } } }) + dueBefore: (date: Date | string) => ({ where: { dueAt: { [Op.lt]: date } } }), + task: (taskId: number) => ({ where: { taskId } }) })) @Table({ tableName: "v2_project_reports", underscored: true, paranoid: true }) export class ProjectReport extends Model { @@ -98,6 +99,10 @@ export class ProjectReport extends Model { return builder.literal; } + static task(taskId: number) { + return chainScope(this, "task", taskId) as typeof ProjectReport; + } + @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) @@ -169,7 +174,7 @@ export class ProjectReport extends Model { @AllowNull @Column(STRING) - updateRequestStatus: string; + updateRequestStatus: UpdateRequestStatus | null; @AllowNull @Column(TEXT) diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index ca0cc123..f94de712 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -17,7 +17,7 @@ import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; import { FrameworkKey } from "../constants/framework"; import { Literal } from "sequelize/types/utils"; -import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates } from "../constants/status"; +import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates, UpdateRequestStatus } from "../constants/status"; import { chainScope } from "../util/chain-scope"; import { Subquery } from "../util/subquery.builder"; import { Task } from "./task.entity"; @@ -36,7 +36,7 @@ type ApprovedIdsSubqueryOptions = { sites: (ids: number[] | Literal) => ({ where: { siteId: { [Op.in]: ids } } }), approved: { where: { status: { [Op.in]: SiteReport.APPROVED_STATUSES } } }, dueBefore: (date: Date | string) => ({ where: { dueAt: { [Op.lt]: date } } }), - task: (taskId: number) => ({ where: { taskId: taskId } }) + task: (taskId: number) => ({ where: { taskId } }) })) @Table({ tableName: "v2_site_reports", underscored: true, paranoid: true }) export class SiteReport extends Model { @@ -182,7 +182,7 @@ export class SiteReport extends Model { @AllowNull @Column(STRING) - updateRequestStatus: string; + updateRequestStatus: UpdateRequestStatus | null; @AllowNull @Column(DATE) diff --git a/libs/database/src/lib/entities/task.entity.ts b/libs/database/src/lib/entities/task.entity.ts index f8cae190..84d720c5 100644 --- a/libs/database/src/lib/entities/task.entity.ts +++ b/libs/database/src/lib/entities/task.entity.ts @@ -1,9 +1,23 @@ -import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { + AllowNull, + AutoIncrement, + Column, + ForeignKey, + HasMany, + HasOne, + Index, + Model, + PrimaryKey, + Table +} from "sequelize-typescript"; import { BIGINT, DATE, STRING, UUID, UUIDV4 } from "sequelize"; import { Organisation } from "./organisation.entity"; import { Project } from "./project.entity"; import { TaskStatus, TaskStatusStates } from "../constants/status"; import { StateMachineColumn } from "../util/model-column-state-machine"; +import { ProjectReport } from "./project-report.entity"; +import { SiteReport } from "./site-report.entity"; +import { NurseryReport } from "./nursery-report.entity"; @Table({ tableName: "v2_tasks", underscored: true, paranoid: true }) export class Task extends Model { @@ -45,4 +59,13 @@ export class Task extends Model { // make that a real constraint when the schema is controlled by v3 code. @Column(DATE) dueAt: Date; + + @HasOne(() => ProjectReport) + projectReport: ProjectReport | null; + + @HasMany(() => SiteReport) + siteReports: SiteReport[] | null; + + @HasMany(() => NurseryReport) + nurseryReports: NurseryReport[] | null; } diff --git a/libs/database/src/lib/types/util.ts b/libs/database/src/lib/types/util.ts index ff55d54c..5f982e60 100644 --- a/libs/database/src/lib/types/util.ts +++ b/libs/database/src/lib/types/util.ts @@ -10,3 +10,5 @@ export const laravelType = (model: LaravelModel) => (model.constructor as Larave export type StatusModel = Model & { status: string }; export type FeedbackModel = Model & { feedback?: string | null; feedbackFields?: string[] | null }; + +export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; From df98225cc312792bfe54b924cccf4d45bd7b7424 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 11:09:57 -0700 Subject: [PATCH 43/98] [TM-1955] Support update of all six entity types. --- .../src/entities/dto/entity-update.dto.ts | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index 8d853d74..f8ad5d7e 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from "@nestjs/swagger"; -import { ENTITY_STATUSES, SITE_STATUSES } from "@terramatch-microservices/database/constants/status"; +import { ENTITY_STATUSES, REPORT_STATUSES, SITE_STATUSES } from "@terramatch-microservices/database/constants/status"; import { IsArray, IsBoolean, IsIn, IsOptional, IsString } from "class-validator"; import { JsonApiDataDto, JsonApiMultiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; import { Type } from "class-transformer"; @@ -42,21 +42,36 @@ export class SiteUpdateAttributes extends EntityUpdateAttributes { @IsOptional() @IsIn(SITE_STATUSES) @ApiProperty({ - description: "Request to change to the status of the given entity", + description: "Request to change to the status of the given site", nullable: true, enum: SITE_STATUSES }) status?: string | null; } +export class ReportUpdateAttributes extends EntityUpdateAttributes { + @IsOptional() + @IsIn(REPORT_STATUSES) + @ApiProperty({ + description: "Request to change to the status of the given report", + nullable: true, + enum: REPORT_STATUSES + }) + status?: string | null; +} + export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {} export class NurseryUpdateData extends JsonApiDataDto({ type: "nurseries" }, EntityUpdateAttributes) {} -export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, EntityUpdateAttributes) {} -export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, EntityUpdateAttributes) {} -export class NurseryReportUpdateData extends JsonApiDataDto({ type: "nurseryReports" }, EntityUpdateAttributes) {} +export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, ReportUpdateAttributes) {} +export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, ReportUpdateAttributes) {} +export class NurseryReportUpdateData extends JsonApiDataDto({ type: "nurseryReports" }, ReportUpdateAttributes) {} -export type EntityUpdateData = ProjectUpdateAttributes | SiteUpdateAttributes | EntityUpdateAttributes; +export type EntityUpdateData = + | ProjectUpdateAttributes + | SiteUpdateAttributes + | ReportUpdateAttributes + | EntityUpdateAttributes; export class EntityUpdateBody extends JsonApiMultiBodyDto([ ProjectUpdateData, SiteUpdateData, From 6227aec3c646ded2e7177661641338876921a805 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 14:32:05 -0700 Subject: [PATCH 44/98] [TM-1955] Support for nothing to report updates on site / nursery reports. --- .../src/entities/dto/entity-update.dto.ts | 5 ++ .../entities/processors/entity-processor.ts | 50 +++++++++++++++++-- .../processors/nursery-report.processor.ts | 8 +-- .../processors/project-report.processor.ts | 8 +-- .../processors/site-report.processor.ts | 8 +-- .../entity-status-update.event-processor.ts | 8 ++- 6 files changed, 70 insertions(+), 17 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-update.dto.ts b/apps/entity-service/src/entities/dto/entity-update.dto.ts index f8ad5d7e..6dc40ffc 100644 --- a/apps/entity-service/src/entities/dto/entity-update.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-update.dto.ts @@ -58,6 +58,11 @@ export class ReportUpdateAttributes extends EntityUpdateAttributes { enum: REPORT_STATUSES }) status?: string | null; + + @IsOptional() + @IsBoolean() + @ApiProperty({ description: "Update the nothingToReport flag.", nullable: true }) + nothingToReport?: boolean; } export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {} diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index e2cdc702..3d91ffb5 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -5,10 +5,15 @@ import { EntitiesService, ProcessableEntity } from "../entities.service"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { BadRequestException, Type } from "@nestjs/common"; import { EntityDto } from "../dto/entity.dto"; -import { EntityModel } from "@terramatch-microservices/database/constants/entities"; +import { EntityModel, ReportModel } from "@terramatch-microservices/database/constants/entities"; import { Action } from "@terramatch-microservices/database/entities/action.entity"; -import { EntityUpdateData } from "../dto/entity-update.dto"; -import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; +import { EntityUpdateData, ReportUpdateAttributes } from "../dto/entity-update.dto"; +import { + APPROVED, + NEEDS_MORE_INFORMATION, + RESTORATION_IN_PROGRESS +} from "@terramatch-microservices/database/constants/status"; +import { ProjectReport } from "@terramatch-microservices/database/entities"; export type Aggregate> = { func: string; @@ -58,7 +63,7 @@ export abstract class EntityProcessor< abstract readonly LIGHT_DTO: Type; abstract readonly FULL_DTO: Type; - readonly APPROVAL_STATUSES = [APPROVED, NEEDS_MORE_INFORMATION]; + readonly APPROVAL_STATUSES = [APPROVED, NEEDS_MORE_INFORMATION, RESTORATION_IN_PROGRESS]; constructor(protected readonly entitiesService: EntitiesService, protected readonly resource: ProcessableEntity) {} @@ -134,3 +139,40 @@ export abstract class EntityProcessor< await model.save(); } } + +export abstract class ReportProcessor< + ModelType extends ReportModel, + LightDto extends EntityDto, + FullDto extends EntityDto, + UpdateDto extends ReportUpdateAttributes +> extends EntityProcessor { + async update(model: ModelType, update: UpdateDto) { + if (update.nothingToReport != null) { + if (model instanceof ProjectReport) { + throw new BadRequestException("ProjectReport does not support nothingToReport"); + } + + if (update.nothingToReport !== model.nothingToReport) { + model.nothingToReport = update.nothingToReport; + + if (model.nothingToReport) { + const statusChanged = update.status != null && update.status !== model.status; + if (statusChanged && update.status !== "awaiting-approval") { + throw new BadRequestException( + "Cannot set status to anything other than 'awaiting-approval' with nothingToReport: true" + ); + } + + if (model.submittedAt == null) { + model.completion = 100; + model.submittedAt = new Date(); + } + + model.status = "awaiting-approval"; + } + } + } + + await super.update(model, update); + } +} diff --git a/apps/entity-service/src/entities/processors/nursery-report.processor.ts b/apps/entity-service/src/entities/processors/nursery-report.processor.ts index 34f4b3e7..43b39764 100644 --- a/apps/entity-service/src/entities/processors/nursery-report.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery-report.processor.ts @@ -1,5 +1,5 @@ import { Media, Nursery, NurseryReport, ProjectReport, ProjectUser } from "@terramatch-microservices/database/entities"; -import { EntityProcessor } from "./entity-processor"; +import { ReportProcessor } from "./entity-processor"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { Includeable, Op } from "sequelize"; import { BadRequestException } from "@nestjs/common"; @@ -10,13 +10,13 @@ import { NurseryReportLightDto, NurseryReportMedia } from "../dto/nursery-report.dto"; -import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { ReportUpdateAttributes } from "../dto/entity-update.dto"; -export class NurseryReportProcessor extends EntityProcessor< +export class NurseryReportProcessor extends ReportProcessor< NurseryReport, NurseryReportLightDto, NurseryReportFullDto, - EntityUpdateAttributes + ReportUpdateAttributes > { readonly LIGHT_DTO = NurseryReportLightDto; readonly FULL_DTO = NurseryReportFullDto; diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index 46b69802..712e386a 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -1,5 +1,5 @@ import { ProjectReport } from "@terramatch-microservices/database/entities/project-report.entity"; -import { EntityProcessor } from "./entity-processor"; +import { ReportProcessor } from "./entity-processor"; import { AdditionalProjectReportFullProps, ProjectReportFullDto, @@ -20,13 +20,13 @@ import { TreeSpecies } from "@terramatch-microservices/database/entities"; import { sumBy } from "lodash"; -import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { ReportUpdateAttributes } from "../dto/entity-update.dto"; -export class ProjectReportProcessor extends EntityProcessor< +export class ProjectReportProcessor extends ReportProcessor< ProjectReport, ProjectReportLightDto, ProjectReportFullDto, - EntityUpdateAttributes + ReportUpdateAttributes > { readonly LIGHT_DTO = ProjectReportLightDto; readonly FULL_DTO = ProjectReportFullDto; diff --git a/apps/entity-service/src/entities/processors/site-report.processor.ts b/apps/entity-service/src/entities/processors/site-report.processor.ts index 619aa49f..863e42a6 100644 --- a/apps/entity-service/src/entities/processors/site-report.processor.ts +++ b/apps/entity-service/src/entities/processors/site-report.processor.ts @@ -7,7 +7,7 @@ import { SiteReport, TreeSpecies } from "@terramatch-microservices/database/entities"; -import { EntityProcessor } from "./entity-processor"; +import { ReportProcessor } from "./entity-processor"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { Includeable, Op } from "sequelize"; import { BadRequestException } from "@nestjs/common"; @@ -18,13 +18,13 @@ import { SiteReportLightDto, SiteReportMedia } from "../dto/site-report.dto"; -import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { ReportUpdateAttributes } from "../dto/entity-update.dto"; -export class SiteReportProcessor extends EntityProcessor< +export class SiteReportProcessor extends ReportProcessor< SiteReport, SiteReportLightDto, SiteReportFullDto, - EntityUpdateAttributes + ReportUpdateAttributes > { readonly LIGHT_DTO = SiteReportLightDto; readonly FULL_DTO = SiteReportFullDto; diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index 6e2eddcd..749211a7 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -25,6 +25,8 @@ import { } from "@terramatch-microservices/database/constants/status"; import { InternalServerErrorException } from "@nestjs/common"; +const TASK_UPDATE_REPORT_STATUSES = [APPROVED, NEEDS_MORE_INFORMATION, AWAITING_APPROVAL]; + export class EntityStatusUpdate extends EventProcessor { private readonly logger = new TMLogger(EntityStatusUpdate.name); @@ -51,7 +53,11 @@ export class EntityStatusUpdate extends EventProcessor { } await this.createAuditStatus(); - if (entityType != null && isReport(this.model as EntityModel)) { + if ( + entityType != null && + isReport(this.model as EntityModel) && + TASK_UPDATE_REPORT_STATUSES.includes(this.model.status) + ) { await this.checkTaskStatus(); } } From 1a347a09653cf694279cf4e9e2634960154eb6cc Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 15:37:42 -0700 Subject: [PATCH 45/98] [TM-1955] Get coverage back up in entity service. --- .../src/entities/entities.service.ts | 6 +- .../processors/entity.processor.spec.ts | 70 ++++++++++++++++--- .../lib/factories/nursery-report.factory.ts | 3 +- .../src/lib/factories/nursery.factory.ts | 3 +- .../lib/factories/project-report.factory.ts | 3 +- .../src/lib/factories/project.factory.ts | 3 +- .../src/lib/factories/site-report.factory.ts | 3 +- .../src/lib/factories/site.factory.ts | 3 +- 8 files changed, 70 insertions(+), 24 deletions(-) diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index 48ffb368..81c34939 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -28,14 +28,14 @@ import { LocalizationService } from "@terramatch-microservices/common/localizati import { ITranslateParams } from "@transifex/native"; // The keys of this array must match the type in the resulting DTO. -const ENTITY_PROCESSORS = { +export const ENTITY_PROCESSORS = { projects: ProjectProcessor, sites: SiteProcessor, nurseries: NurseryProcessor, projectReports: ProjectReportProcessor, nurseryReports: NurseryReportProcessor, siteReports: SiteReportProcessor -}; +} as const; export type ProcessableEntity = keyof typeof ENTITY_PROCESSORS; export const PROCESSABLE_ENTITIES = Object.keys(ENTITY_PROCESSORS) as ProcessableEntity[]; @@ -46,7 +46,9 @@ export const POLYGON_STATUSES_FILTERS = [ "needs-more-information", "draft" ] as const; + export type PolygonStatusFilter = (typeof POLYGON_STATUSES_FILTERS)[number]; + const ASSOCIATION_PROCESSORS = { demographics: AssociationProcessor.buildSimpleProcessor( DemographicDto, diff --git a/apps/entity-service/src/entities/processors/entity.processor.spec.ts b/apps/entity-service/src/entities/processors/entity.processor.spec.ts index c93e73a0..50a29ccd 100644 --- a/apps/entity-service/src/entities/processors/entity.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/entity.processor.spec.ts @@ -1,18 +1,24 @@ -import { ProjectProcessor } from "./project.processor"; import { Test } from "@nestjs/testing"; import { MediaService } from "@terramatch-microservices/common/media/media.service"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; -import { EntitiesService } from "../entities.service"; -import { ProjectFactory } from "@terramatch-microservices/database/factories"; +import { EntitiesService, ProcessableEntity } from "../entities.service"; +import { + NurseryReportFactory, + ProjectFactory, + ProjectReportFactory, + SiteReportFactory +} from "@terramatch-microservices/database/factories"; import { ActionFactory } from "@terramatch-microservices/database/factories/action.factory"; import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; -import { UnauthorizedException } from "@nestjs/common"; +import { BadRequestException, UnauthorizedException } from "@nestjs/common"; describe("EntityProcessor", () => { - let processor: ProjectProcessor; + let service: EntitiesService; let policyService: DeepMocked; + const createProcessor = (entity: ProcessableEntity = "projects") => service.createEntityProcessor(entity); + beforeEach(async () => { const module = await Test.createTestingModule({ providers: [ @@ -23,7 +29,7 @@ describe("EntityProcessor", () => { ] }).compile(); - processor = module.get(EntitiesService).createEntityProcessor("projects") as ProjectProcessor; + service = module.get(EntitiesService); }); afterEach(async () => { @@ -33,7 +39,7 @@ describe("EntityProcessor", () => { describe("delete", () => { it("deletes the requested model", async () => { const project = await ProjectFactory.create(); - await processor.delete(project); + await createProcessor().delete(project); await project.reload({ paranoid: false }); expect(project.deletedAt).not.toBeNull(); }); @@ -41,7 +47,7 @@ describe("EntityProcessor", () => { it("deletes associated actions", async () => { const project = await ProjectFactory.create(); const actions = await ActionFactory.forProject.createMany(2, { targetableId: project.id }); - await processor.delete(project); + await createProcessor().delete(project); for (const action of actions) { await action.reload({ paranoid: false }); expect(action.deletedAt).not.toBeNull(); @@ -53,7 +59,7 @@ describe("EntityProcessor", () => { it("calls model.save", async () => { const project = await ProjectFactory.create(); const spy = jest.spyOn(project, "save"); - await processor.update(project, {}); + await createProcessor().update(project, {}); expect(spy).toHaveBeenCalled(); }); @@ -61,7 +67,12 @@ describe("EntityProcessor", () => { const project = await ProjectFactory.create({ status: "started", feedback: null, feedbackFields: null }); policyService.authorize.mockResolvedValueOnce(undefined); - await processor.update(project, { status: "awaiting-approval", feedback: "foo", feedbackFields: ["bar"] }); + const processor = createProcessor(); + await processor.update(project, { + status: "awaiting-approval", + feedback: "foo", + feedbackFields: ["bar"] + }); expect(policyService.authorize).not.toHaveBeenCalled(); // These two should be ignored for non approval statuses expect(project.feedback).toBeNull(); @@ -77,5 +88,44 @@ describe("EntityProcessor", () => { expect(project.feedback).toEqual("foo"); expect(project.feedbackFields).toEqual(["bar"]); }); + + describe("nothingToReport", () => { + it("throws when the property is present on a project report update", async () => { + const projectReport = await ProjectReportFactory.create(); + const processor = createProcessor("projectReports"); + await expect(processor.update(projectReport, { nothingToReport: true })).rejects.toThrow(BadRequestException); + await expect(processor.update(projectReport, { nothingToReport: false })).rejects.toThrow(BadRequestException); + }); + + it("throws if the request attempts to set status", async () => { + let siteReport = await SiteReportFactory.create({ submittedAt: null }); + const processor = createProcessor("siteReports"); + await expect(processor.update(siteReport, { nothingToReport: true, status: "started" })).rejects.toThrow( + BadRequestException + ); + siteReport = await SiteReportFactory.create({ submittedAt: null }); + await expect( + processor.update(siteReport, { nothingToReport: false, status: "started" }) + ).resolves.not.toThrow(); + siteReport = await SiteReportFactory.create({ submittedAt: null }); + await expect( + processor.update(siteReport, { nothingToReport: true, status: "awaiting-approval" }) + ).resolves.not.toThrow(); + }); + + it("Sets completion and submission date", async () => { + const report = await NurseryReportFactory.create({ + completion: null, + submittedAt: null, + nothingToReport: null + }); + const processor = createProcessor("nurseryReports"); + await processor.update(report, { nothingToReport: true }); + expect(report.nothingToReport).toBe(true); + expect(report.status).toBe("awaiting-approval"); + expect(report.completion).toBe(100); + expect(report.submittedAt).not.toBeNull(); + }); + }); }); }); diff --git a/libs/database/src/lib/factories/nursery-report.factory.ts b/libs/database/src/lib/factories/nursery-report.factory.ts index 9e66bf40..b8ddc39e 100644 --- a/libs/database/src/lib/factories/nursery-report.factory.ts +++ b/libs/database/src/lib/factories/nursery-report.factory.ts @@ -3,7 +3,7 @@ import { NurseryReport } from "../entities"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; import { NurseryFactory } from "./nursery.factory"; -import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { TaskFactory } from "./task.factory"; export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () => { @@ -14,7 +14,6 @@ export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () = taskId: TaskFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - status: faker.helpers.arrayElement(REPORT_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) }; }); diff --git a/libs/database/src/lib/factories/nursery.factory.ts b/libs/database/src/lib/factories/nursery.factory.ts index 6f64ea48..6eb426a9 100644 --- a/libs/database/src/lib/factories/nursery.factory.ts +++ b/libs/database/src/lib/factories/nursery.factory.ts @@ -2,11 +2,10 @@ import { FactoryGirl } from "factory-girl-ts"; import { Nursery } from "../entities"; import { ProjectFactory } from "./project.factory"; import { faker } from "@faker-js/faker"; -import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; export const NurseryFactory = FactoryGirl.define(Nursery, async () => ({ projectId: ProjectFactory.associate("id"), name: faker.animal.petName(), - status: faker.helpers.arrayElement(ENTITY_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) })); diff --git a/libs/database/src/lib/factories/project-report.factory.ts b/libs/database/src/lib/factories/project-report.factory.ts index c4e0e5db..549874b0 100644 --- a/libs/database/src/lib/factories/project-report.factory.ts +++ b/libs/database/src/lib/factories/project-report.factory.ts @@ -3,7 +3,7 @@ import { ProjectReport } from "../entities"; import { faker } from "@faker-js/faker"; import { ProjectFactory } from "./project.factory"; import { DateTime } from "luxon"; -import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { FRAMEWORK_KEYS } from "../constants/framework"; import { TaskFactory } from "./task.factory"; @@ -16,7 +16,6 @@ export const ProjectReportFactory = FactoryGirl.define(ProjectReport, async () = frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - status: faker.helpers.arrayElement(REPORT_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) }; }); diff --git a/libs/database/src/lib/factories/project.factory.ts b/libs/database/src/lib/factories/project.factory.ts index 93b3cf77..094bc8a7 100644 --- a/libs/database/src/lib/factories/project.factory.ts +++ b/libs/database/src/lib/factories/project.factory.ts @@ -1,7 +1,7 @@ import { FactoryGirl } from "factory-girl-ts"; import { Project } from "../entities"; import { faker } from "@faker-js/faker"; -import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { ApplicationFactory } from "./application.factory"; import { OrganisationFactory } from "./organisation.factory"; import { FRAMEWORK_KEYS } from "../constants/framework"; @@ -11,7 +11,6 @@ const CONTINENTS = ["africa", "australia", "south-america", "asia", "north-ameri export const ProjectFactory = FactoryGirl.define(Project, async () => ({ name: faker.animal.petName(), frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), - status: faker.helpers.arrayElement(ENTITY_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), applicationId: ApplicationFactory.associate("id"), organisationId: OrganisationFactory.associate("id"), diff --git a/libs/database/src/lib/factories/site-report.factory.ts b/libs/database/src/lib/factories/site-report.factory.ts index 0f3320ef..9fce9146 100644 --- a/libs/database/src/lib/factories/site-report.factory.ts +++ b/libs/database/src/lib/factories/site-report.factory.ts @@ -3,7 +3,7 @@ import { FactoryGirl } from "factory-girl-ts"; import { SiteFactory } from "./site.factory"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; -import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { TaskFactory } from "./task.factory"; export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { @@ -14,7 +14,6 @@ export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { taskId: TaskFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - status: faker.helpers.arrayElement(REPORT_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), numTreesRegenerating: faker.number.int({ min: 10, max: 500 }), workdaysPaid: faker.number.int({ min: 10, max: 50 }), diff --git a/libs/database/src/lib/factories/site.factory.ts b/libs/database/src/lib/factories/site.factory.ts index 8766a2a7..3036035b 100644 --- a/libs/database/src/lib/factories/site.factory.ts +++ b/libs/database/src/lib/factories/site.factory.ts @@ -2,13 +2,12 @@ import { Site } from "../entities"; import { FactoryGirl } from "factory-girl-ts"; import { ProjectFactory } from "./project.factory"; import { faker } from "@faker-js/faker"; -import { SITE_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { SITING_STRATEGIES } from "../constants/entity-selects"; export const SiteFactory = FactoryGirl.define(Site, async () => ({ projectId: ProjectFactory.associate("id"), name: faker.animal.petName(), - status: faker.helpers.arrayElement(SITE_STATUSES), updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), sitingStrategy: faker.helpers.arrayElement(SITING_STRATEGIES), descriptionSitingStrategy: faker.lorem.paragraph() From 4595e803aa85cbbf89bad1a8bf7e53f71280b8ea Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 16:49:59 -0700 Subject: [PATCH 46/98] [TM-1955] Coverage for checkTaskStatus. --- ...tity-status-update.event-processor.spec.ts | 149 +++++++++++++++++- .../entity-status-update.event-processor.ts | 8 +- .../src/lib/entities/project-report.entity.ts | 2 +- 3 files changed, 150 insertions(+), 9 deletions(-) diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts b/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts index 999dc746..f268c367 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.spec.ts @@ -1,12 +1,33 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ +import * as FakeTimers from "@sinonjs/fake-timers"; import { EventService } from "./event.service"; import { EntityStatusUpdate } from "./entity-status-update.event-processor"; -import { ProjectFactory, UpdateRequestFactory, UserFactory } from "@terramatch-microservices/database/factories"; +import { + NurseryReportFactory, + ProjectFactory, + ProjectReportFactory, + SiteReportFactory, + TaskFactory, + UpdateRequestFactory, + UserFactory +} from "@terramatch-microservices/database/factories"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { RequestContext } from "nestjs-request-context"; import { ActionFactory } from "@terramatch-microservices/database/factories/action.factory"; -import { Action, AuditStatus } from "@terramatch-microservices/database/entities"; +import { Action, AuditStatus, Task } from "@terramatch-microservices/database/entities"; import { faker } from "@faker-js/faker"; +import { DateTime } from "luxon"; +import { ReportModel } from "@terramatch-microservices/database/constants/entities"; +import { Attributes } from "sequelize"; +import { + APPROVED, + AWAITING_APPROVAL, + DUE, + NEEDS_MORE_INFORMATION, + ReportStatus, + STARTED +} from "@terramatch-microservices/database/constants/status"; +import { InternalServerErrorException } from "@nestjs/common"; function mockUserId(userId?: number) { jest @@ -49,7 +70,7 @@ describe("EntityStatusUpdate EventProcessor", () => { it("should update actions", async () => { mockUserId(); - const project = await ProjectFactory.create({ status: "approved" }); + const project = await ProjectFactory.create({ status: APPROVED }); const action = await ActionFactory.forProject.create({ targetableId: project.id }); await new EntityStatusUpdate(eventService, project).handle(); @@ -71,7 +92,7 @@ describe("EntityStatusUpdate EventProcessor", () => { mockUserId(user.id); const feedback = faker.lorem.sentence(); - const project = await ProjectFactory.create({ status: "approved", feedback }); + const project = await ProjectFactory.create({ status: APPROVED, feedback }); await new EntityStatusUpdate(eventService, project).handle(); const auditStatus = await AuditStatus.for(project).findOne({ order: [["createdAt", "DESC"]] }); @@ -82,4 +103,124 @@ describe("EntityStatusUpdate EventProcessor", () => { comment: `Approved: ${feedback}` }); }); + + describe("checkTaskStatus", () => { + function createHandler(report: ReportModel) { + const handler = new EntityStatusUpdate(eventService, report); + jest.spyOn(handler as any, "sendStatusUpdateEmail").mockResolvedValue(undefined); + jest.spyOn(handler as any, "updateActions").mockResolvedValue(undefined); + jest.spyOn(handler as any, "createAuditStatus").mockResolvedValue(undefined); + return handler; + } + + async function createOldTask(props?: Partial>) { + const creationTime = DateTime.fromJSDate(new Date()).minus({ minutes: 5 }).set({ millisecond: 0 }).toJSDate(); + const clock = FakeTimers.install({ shouldAdvanceTime: true }); + try { + clock.setSystemTime(creationTime); + return await TaskFactory.create(props); + } finally { + clock.uninstall(); + } + } + + it("should only be called for valid task update statuses", async () => { + const report = await ProjectReportFactory.create(); + const handler = createHandler(report); + const spy = jest.spyOn(handler as any, "checkTaskStatus").mockResolvedValue(undefined); + await handler.handle(); + expect(spy).not.toHaveBeenCalled(); + + async function expectCall(status: ReportStatus, expectCall: boolean) { + spy.mockClear(); + report.status = status; + await handler.handle(); + if (expectCall) expect(spy).toHaveBeenCalledTimes(1); + else expect(spy).not.toHaveBeenCalled(); + } + + await expectCall(STARTED, false); + await expectCall(AWAITING_APPROVAL, true); + await expectCall(NEEDS_MORE_INFORMATION, true); + await expectCall(APPROVED, true); + }); + + it("should log a warning if no task ID is found", async () => { + const projectReport = await ProjectReportFactory.create({ taskId: null, status: AWAITING_APPROVAL }); + const handler = createHandler(projectReport); + const logSpy = jest.spyOn((handler as any).logger, "warn"); + await handler.handle(); + expect(logSpy).toHaveBeenCalledWith(expect.stringMatching("No task found for status changed report")); + }); + + it("should log a warning if the task for the ID is not found", async () => { + const projectReport = await ProjectReportFactory.create({ status: AWAITING_APPROVAL }); + await Task.destroy({ where: { id: projectReport.taskId as number } }); + const handler = createHandler(projectReport); + const logSpy = jest.spyOn((handler as any).logger, "error"); + await handler.handle(); + expect(logSpy).toHaveBeenCalledWith(expect.stringMatching("No task found for task id")); + }); + + it("should NOOP if the task is DUE", async () => { + const task = await createOldTask(); + const projectReport = await ProjectReportFactory.create({ taskId: task.id, status: AWAITING_APPROVAL }); + await createHandler(projectReport).handle(); + await task.reload(); + expect(task.updatedAt).toEqual(task.createdAt); + }); + + it("should move task to approved if all reports are approved", async () => { + const task = await createOldTask({ status: AWAITING_APPROVAL }); + const projectReport = await ProjectReportFactory.create({ taskId: task.id, status: APPROVED }); + await SiteReportFactory.create({ taskId: task.id, status: APPROVED }); + await NurseryReportFactory.create({ taskId: task.id, status: APPROVED }); + await createHandler(projectReport).handle(); + await task.reload(); + expect(task.updatedAt).not.toEqual(task.createdAt); + expect(task.status).toBe(APPROVED); + }); + + it("should throw if there is a report in a bad status", async () => { + const task = await createOldTask({ status: AWAITING_APPROVAL }); + const projectReport = await ProjectReportFactory.create({ taskId: task.id, status: APPROVED }); + const siteReport = await SiteReportFactory.create({ taskId: task.id, status: DUE }); + await expect(createHandler(projectReport).handle()).rejects.toThrow(InternalServerErrorException); + + await siteReport.update({ status: STARTED }); + await expect(createHandler(projectReport).handle()).rejects.toThrow(InternalServerErrorException); + }); + + it("should move to needs-more-information if a report is in that status", async () => { + const task = await createOldTask({ status: AWAITING_APPROVAL }); + const projectReport = await ProjectReportFactory.create({ taskId: task.id, status: APPROVED }); + const siteReport = await SiteReportFactory.create({ taskId: task.id, status: NEEDS_MORE_INFORMATION }); + await createHandler(projectReport).handle(); + await task.reload(); + expect(task.updatedAt).not.toEqual(task.createdAt); + expect(task.status).toBe(NEEDS_MORE_INFORMATION); + + await task.update({ status: AWAITING_APPROVAL }); + await siteReport.update({ status: AWAITING_APPROVAL, updateRequestStatus: NEEDS_MORE_INFORMATION }); + await createHandler(projectReport).handle(); + await task.reload(); + expect(task.updatedAt).not.toEqual(task.createdAt); + expect(task.status).toBe(NEEDS_MORE_INFORMATION); + }); + + it("should move the task to awaiting-approval when all reports are in awaiting-approval", async () => { + const task = await createOldTask({ status: NEEDS_MORE_INFORMATION }); + const projectReport = await ProjectReportFactory.create({ taskId: task.id, status: AWAITING_APPROVAL }); + await SiteReportFactory.create({ taskId: task.id, status: AWAITING_APPROVAL }); + await NurseryReportFactory.create({ + taskId: task.id, + status: NEEDS_MORE_INFORMATION, + updateRequestStatus: AWAITING_APPROVAL + }); + await createHandler(projectReport).handle(); + await task.reload(); + expect(task.updatedAt).not.toEqual(task.createdAt); + expect(task.status).toBe(AWAITING_APPROVAL); + }); + }); }); diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index 749211a7..9530dee8 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -143,7 +143,7 @@ export class EntityStatusUpdate extends EventProcessor { }`; } - async checkTaskStatus() { + private async checkTaskStatus() { const { taskId } = this.model as ReportModel; if (taskId == null) { this.logger.warn(`No task found for status changed report [${this.model.constructor.name}, ${this.model.id}]`); @@ -175,7 +175,7 @@ export class EntityStatusUpdate extends EventProcessor { const reportStatuses = uniq(reports.map(({ status }) => status)); if (reportStatuses.length === 1 && reportStatuses[0] === APPROVED) { - await task.update({ status: "approved" }); + await task.update({ status: APPROVED }); return; } @@ -190,12 +190,12 @@ export class EntityStatusUpdate extends EventProcessor { ); if (moreInfoReport != null) { // A report in needs-more-information causes the task to go to needs-more-information - await task.update({ status: "needs-more-information" }); + await task.update({ status: NEEDS_MORE_INFORMATION }); return; } // If there are no reports or update requests in needs-more-information, the only option left is that // something is in awaiting-approval. - await task.update({ status: "awaiting-approval" }); + await task.update({ status: AWAITING_APPROVAL }); } } diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index 26a7c477..d94289de 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -164,7 +164,7 @@ export class ProjectReport extends Model { @ForeignKey(() => Task) @AllowNull @Column(BIGINT.UNSIGNED) - taskId: number; + taskId: number | null; @BelongsTo(() => Task, { constraints: false }) task: Task | null; From bc1fa9d4194fa5b2f2144760f49705bc7092f8ec Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 17:01:29 -0700 Subject: [PATCH 47/98] [TM-1955] Update report policies. --- libs/common/src/lib/policies/nursery-report.policy.ts | 10 +++++----- libs/common/src/lib/policies/project-report.policy.ts | 10 +++++----- libs/common/src/lib/policies/site-report.policy.ts | 8 +++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/libs/common/src/lib/policies/nursery-report.policy.ts b/libs/common/src/lib/policies/nursery-report.policy.ts index 3e23b3b8..0744b0e2 100644 --- a/libs/common/src/lib/policies/nursery-report.policy.ts +++ b/libs/common/src/lib/policies/nursery-report.policy.ts @@ -10,8 +10,7 @@ export class NurseryReportPolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can("read", NurseryReport, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", NurseryReport, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete"], NurseryReport, { frameworkKey: { $in: this.frameworks } }); } if (this.permissions.includes("manage-own")) { @@ -28,7 +27,7 @@ export class NurseryReportPolicy extends UserPermissionsPolicy { }) ).map(({ id }) => id); if (nurseryIds.length > 0) { - this.builder.can("read", NurseryReport, { nurseryId: { $in: nurseryIds } }); + this.builder.can(["read", "update"], NurseryReport, { nurseryId: { $in: nurseryIds } }); } } } @@ -42,8 +41,9 @@ export class NurseryReportPolicy extends UserPermissionsPolicy { await Nursery.findAll({ where: { projectId: { [Op.in]: projectIds } }, attributes: ["id"] }) ).map(({ id }) => id); if (nurseryIds.length > 0) { - this.builder.can("read", NurseryReport, { nurseryId: { $in: nurseryIds } }); - this.builder.can("delete", NurseryReport, { nurseryId: { $in: nurseryIds } }); + this.builder.can(["read", "delete", "update", "approve"], NurseryReport, { + nurseryId: { $in: nurseryIds } + }); } } } diff --git a/libs/common/src/lib/policies/project-report.policy.ts b/libs/common/src/lib/policies/project-report.policy.ts index 1f6bad43..4c945731 100644 --- a/libs/common/src/lib/policies/project-report.policy.ts +++ b/libs/common/src/lib/policies/project-report.policy.ts @@ -9,8 +9,9 @@ export class ProjectReportPolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can("read", ProjectReport, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", ProjectReport, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], ProjectReport, { + frameworkKey: { $in: this.frameworks } + }); } if (this.permissions.includes("manage-own")) { @@ -24,7 +25,7 @@ export class ProjectReportPolicy extends UserPermissionsPolicy { ...user.projects.map(({ id }) => id) ]; if (projectIds.length > 0) { - this.builder.can("read", ProjectReport, { projectId: { $in: projectIds } }); + this.builder.can(["read", "update"], ProjectReport, { projectId: { $in: projectIds } }); } } } @@ -34,8 +35,7 @@ export class ProjectReportPolicy extends UserPermissionsPolicy { if (user != null) { const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); if (projectIds.length > 0) { - this.builder.can("read", ProjectReport, { projectId: { $in: projectIds } }); - this.builder.can("delete", ProjectReport, { projectId: { $in: projectIds } }); + this.builder.can(["read", "delete", "update", "approve"], ProjectReport, { projectId: { $in: projectIds } }); } } } diff --git a/libs/common/src/lib/policies/site-report.policy.ts b/libs/common/src/lib/policies/site-report.policy.ts index 92a23830..e043e914 100644 --- a/libs/common/src/lib/policies/site-report.policy.ts +++ b/libs/common/src/lib/policies/site-report.policy.ts @@ -10,8 +10,7 @@ export class SiteReportPolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can("read", SiteReport, { frameworkKey: { $in: this.frameworks } }); - this.builder.can("delete", SiteReport, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], SiteReport, { frameworkKey: { $in: this.frameworks } }); } if (this.permissions.includes("manage-own")) { @@ -28,7 +27,7 @@ export class SiteReportPolicy extends UserPermissionsPolicy { }) ).map(({ id }) => id); if (siteIds.length > 0) { - this.builder.can("read", SiteReport, { siteId: { $in: siteIds } }); + this.builder.can(["read", "update"], SiteReport, { siteId: { $in: siteIds } }); } } } @@ -42,8 +41,7 @@ export class SiteReportPolicy extends UserPermissionsPolicy { await Site.findAll({ where: { projectId: { [Op.in]: projectIds } }, attributes: ["id"] }) ).map(({ id }) => id); if (siteIds.length > 0) { - this.builder.can("read", SiteReport, { siteId: { $in: siteIds } }); - this.builder.can("delete", SiteReport, { siteId: { $in: siteIds } }); + this.builder.can(["read", "delete", "update", "approve"], SiteReport, { siteId: { $in: siteIds } }); } } } From db27bcc55a6c14d4fb2cc828b843be2fb3320823 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 17:16:35 -0700 Subject: [PATCH 48/98] [TM-1955] updateRequestStatus needs to be predictable to avoid flaky tests now. --- libs/database/src/lib/factories/nursery-report.factory.ts | 4 ++-- libs/database/src/lib/factories/project-report.factory.ts | 4 ++-- libs/database/src/lib/factories/site-report.factory.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/database/src/lib/factories/nursery-report.factory.ts b/libs/database/src/lib/factories/nursery-report.factory.ts index b8ddc39e..061cca36 100644 --- a/libs/database/src/lib/factories/nursery-report.factory.ts +++ b/libs/database/src/lib/factories/nursery-report.factory.ts @@ -3,8 +3,8 @@ import { NurseryReport } from "../entities"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; import { NurseryFactory } from "./nursery.factory"; -import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { TaskFactory } from "./task.factory"; +import { NO_UPDATE } from "../constants/status"; export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () => { const dueAt = faker.date.past({ years: 2 }); @@ -14,6 +14,6 @@ export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () = taskId: TaskFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) + updateRequestStatus: NO_UPDATE }; }); diff --git a/libs/database/src/lib/factories/project-report.factory.ts b/libs/database/src/lib/factories/project-report.factory.ts index 549874b0..b7c4c43d 100644 --- a/libs/database/src/lib/factories/project-report.factory.ts +++ b/libs/database/src/lib/factories/project-report.factory.ts @@ -3,7 +3,7 @@ import { ProjectReport } from "../entities"; import { faker } from "@faker-js/faker"; import { ProjectFactory } from "./project.factory"; import { DateTime } from "luxon"; -import { UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { NO_UPDATE } from "../constants/status"; import { FRAMEWORK_KEYS } from "../constants/framework"; import { TaskFactory } from "./task.factory"; @@ -16,6 +16,6 @@ export const ProjectReportFactory = FactoryGirl.define(ProjectReport, async () = frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) + updateRequestStatus: NO_UPDATE }; }); diff --git a/libs/database/src/lib/factories/site-report.factory.ts b/libs/database/src/lib/factories/site-report.factory.ts index 9fce9146..1822dbb3 100644 --- a/libs/database/src/lib/factories/site-report.factory.ts +++ b/libs/database/src/lib/factories/site-report.factory.ts @@ -3,7 +3,7 @@ import { FactoryGirl } from "factory-girl-ts"; import { SiteFactory } from "./site.factory"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; -import { UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { NO_UPDATE } from "../constants/status"; import { TaskFactory } from "./task.factory"; export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { @@ -14,7 +14,7 @@ export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { taskId: TaskFactory.associate("id"), dueAt, submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), - updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), + updateRequestStatus: NO_UPDATE, numTreesRegenerating: faker.number.int({ min: 10, max: 500 }), workdaysPaid: faker.number.int({ min: 10, max: 50 }), workdaysVolunteer: faker.number.int({ min: 10, max: 50 }) From efe628b5cce13b0c1e663e34610dddd748ea71f4 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 22:15:12 -0700 Subject: [PATCH 49/98] [TM-1955] Test coverage for update / approve on entities. --- .../policies/nursery-report.policy.spec.ts | 45 ++++++++++--------- .../src/lib/policies/nursery-report.policy.ts | 4 +- .../src/lib/policies/nursery.policy.spec.ts | 44 +++++++++--------- .../src/lib/policies/policy.service.spec.ts | 25 +++++++++-- .../policies/project-report.policy.spec.ts | 45 ++++++++++--------- .../src/lib/policies/project.policy.spec.ts | 45 ++++++++++--------- .../lib/policies/site-report.policy.spec.ts | 45 ++++++++++--------- .../src/lib/policies/site.policy.spec.ts | 43 +++++++++--------- 8 files changed, 171 insertions(+), 125 deletions(-) diff --git a/libs/common/src/lib/policies/nursery-report.policy.spec.ts b/libs/common/src/lib/policies/nursery-report.policy.spec.ts index 6191009c..a34141dc 100644 --- a/libs/common/src/lib/policies/nursery-report.policy.spec.ts +++ b/libs/common/src/lib/policies/nursery-report.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { NurseryReport } from "@terramatch-microservices/database/entities"; import { NurseryFactory, @@ -33,18 +33,18 @@ describe("NurseryReportPolicy", () => { await expectCannot(service, "delete", new NurseryReport()); }); - it("allows reading nursery reports in your framework", async () => { + it("allows managing nursery reports in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await NurseryReportFactory.create({ frameworkKey: "ppc" }); const tf = await NurseryReportFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCannot(service, "read", tf); - await expectCan(service, "delete", ppc); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading nursery reports for own projects", async () => { + it("allows managing nursery reports for own projects", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -67,17 +67,22 @@ describe("NurseryReportPolicy", () => { const nr3 = await NurseryReportFactory.create({ nurseryId: n3.id }); const nr4 = await NurseryReportFactory.create({ nurseryId: n4.id }); - await expectCan(service, "read", nr1); - await expectCannot(service, "read", nr2); - await expectCan(service, "read", nr3); - await expectCan(service, "read", nr4); - await expectCannot(service, "delete", nr1); - await expectCannot(service, "delete", nr2); - await expectCannot(service, "delete", nr3); - await expectCannot(service, "delete", nr4); + await expectAuthority(service, { + can: [ + [["read", "update"], nr1], + [["read", "update"], nr3], + [["read", "update"], nr4] + ], + cannot: [ + [["delete", "approve"], nr1], + [["read", "delete"], nr2], + ["delete", nr3], + ["delete", nr4] + ] + }); }); - it("allows reading nursery reports for managed projects", async () => { + it("allows managing nursery reports for managed projects", async () => { mockPermissions("projects-manage"); const user = await UserFactory.create(); mockUserId(user.id); @@ -92,9 +97,9 @@ describe("NurseryReportPolicy", () => { const nr1 = await NurseryReportFactory.create({ nurseryId: n1.id }); const nr2 = await NurseryReportFactory.create({ nurseryId: n2.id }); - await expectCan(service, "read", nr1); - await expectCannot(service, "read", nr2); - await expectCan(service, "delete", nr1); - await expectCannot(service, "delete", nr2); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], nr1]], + cannot: [[["read", "delete", "update", "approve"], nr2]] + }); }); }); diff --git a/libs/common/src/lib/policies/nursery-report.policy.ts b/libs/common/src/lib/policies/nursery-report.policy.ts index 0744b0e2..500b0954 100644 --- a/libs/common/src/lib/policies/nursery-report.policy.ts +++ b/libs/common/src/lib/policies/nursery-report.policy.ts @@ -10,7 +10,9 @@ export class NurseryReportPolicy extends UserPermissionsPolicy { } if (this.frameworks.length > 0) { - this.builder.can(["read", "delete"], NurseryReport, { frameworkKey: { $in: this.frameworks } }); + this.builder.can(["read", "delete", "update", "approve"], NurseryReport, { + frameworkKey: { $in: this.frameworks } + }); } if (this.permissions.includes("manage-own")) { diff --git a/libs/common/src/lib/policies/nursery.policy.spec.ts b/libs/common/src/lib/policies/nursery.policy.spec.ts index 472d0a85..24e358f8 100644 --- a/libs/common/src/lib/policies/nursery.policy.spec.ts +++ b/libs/common/src/lib/policies/nursery.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { Nursery } from "@terramatch-microservices/database/entities"; import { NurseryFactory, @@ -38,18 +38,18 @@ describe("NurseryPolicy", () => { await expectCan(service, "read", new Nursery()); }); - it("allows reading nurseries in your framework", async () => { + it("allows managing nurseries in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await NurseryFactory.create({ frameworkKey: "ppc" }); const tf = await NurseryFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCannot(service, "read", tf); - await expectCan(service, "delete", ppc); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading own nurseries", async () => { + it("allows managing own nurseries", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -68,17 +68,21 @@ describe("NurseryPolicy", () => { const s2 = await NurseryFactory.create({ projectId: p2.id }); const s3 = await NurseryFactory.create({ projectId: p3.id }); const s4 = await NurseryFactory.create({ projectId: p4.id }); - await expectCan(service, "read", s1); - await expectCannot(service, "read", s2); - await expectCan(service, "read", s3); - await expectCan(service, "read", s4); - await expectCan(service, "delete", s1); - await expectCannot(service, "delete", s2); - await expectCan(service, "delete", s3); - await expectCan(service, "delete", s4); + + await expectAuthority(service, { + can: [ + [["read", "delete", "update"], s1], + [["read", "delete", "update"], s3], + [["read", "delete", "update"], s4] + ], + cannot: [ + ["approve", s1], + [["read", "delete", "update"], s2] + ] + }); }); - it("allows reading managed nurseries", async () => { + it("allows managing managed nurseries", async () => { mockPermissions("projects-manage"); const user = await UserFactory.create(); const project = await ProjectFactory.create(); @@ -86,9 +90,9 @@ describe("NurseryPolicy", () => { mockUserId(user.id); const s1 = await NurseryFactory.create({ projectId: project.id }); const s2 = await NurseryFactory.create(); - await expectCan(service, "read", s1); - await expectCannot(service, "read", s2); - await expectCan(service, "delete", s1); - await expectCannot(service, "delete", s2); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], s1]], + cannot: [[["read", "delete", "update", "approve"], s2]] + }); }); }); diff --git a/libs/common/src/lib/policies/policy.service.spec.ts b/libs/common/src/lib/policies/policy.service.spec.ts index 45a97e5d..84974ffa 100644 --- a/libs/common/src/lib/policies/policy.service.spec.ts +++ b/libs/common/src/lib/policies/policy.service.spec.ts @@ -3,6 +3,7 @@ import { PolicyService } from "./policy.service"; import { UnauthorizedException } from "@nestjs/common"; import { ModelHasRole, Permission, User } from "@terramatch-microservices/database/entities"; import { RequestContext } from "nestjs-request-context"; +import { isArray } from "lodash"; export function mockUserId(userId?: number) { jest @@ -15,12 +16,28 @@ export function mockPermissions(...permissions: string[]) { } type Subject = Parameters[1]; -export async function expectCan(service: PolicyService, action: string, subject: Subject) { - await expect(service.authorize(action, subject)).resolves.toBeUndefined(); +export async function expectCan(service: PolicyService, action: string | string[], subject: Subject) { + const actions = isArray(action) ? action : [action]; + for (const action of actions) { + await expect(service.authorize(action, subject)).resolves.toBeUndefined(); + } } -export async function expectCannot(service: PolicyService, action: string, subject: Subject) { - await expect(service.authorize(action, subject)).rejects.toThrow(UnauthorizedException); +export async function expectCannot(service: PolicyService, action: string | string[], subject: Subject) { + const actions = isArray(action) ? action : [action]; + for (const action of actions) { + await expect(service.authorize(action, subject)).rejects.toThrow(UnauthorizedException); + } +} + +type AuthorityTest = [string | string[], Subject]; +type AuthorityTests = { + can?: AuthorityTest[]; + cannot?: AuthorityTest[]; +}; +export async function expectAuthority(service: PolicyService, tests: AuthorityTests) { + await Promise.all((tests.can ?? []).map(([action, subject]) => expectCan(service, action, subject))); + await Promise.all((tests.cannot ?? []).map(([action, subject]) => expectCannot(service, action, subject))); } describe("PolicyService", () => { diff --git a/libs/common/src/lib/policies/project-report.policy.spec.ts b/libs/common/src/lib/policies/project-report.policy.spec.ts index d8caac5d..79c27477 100644 --- a/libs/common/src/lib/policies/project-report.policy.spec.ts +++ b/libs/common/src/lib/policies/project-report.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { ProjectReport } from "@terramatch-microservices/database/entities"; import { OrganisationFactory, @@ -32,18 +32,18 @@ describe("ProjectReportPolicy", () => { await expectCannot(service, "delete", new ProjectReport()); }); - it("allows reading project reports in your framework", async () => { + it("allows managing project reports in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await ProjectReportFactory.create({ frameworkKey: "ppc" }); const tf = await ProjectReportFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCannot(service, "read", tf); - await expectCan(service, "delete", ppc); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading project reports for own projects", async () => { + it("allows managing project reports for own projects", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -61,17 +61,22 @@ describe("ProjectReportPolicy", () => { const pr3 = await ProjectReportFactory.create({ projectId: p3.id }); const pr4 = await ProjectReportFactory.create({ projectId: p4.id }); - await expectCan(service, "read", pr1); - await expectCannot(service, "read", pr2); - await expectCan(service, "read", pr3); - await expectCan(service, "read", pr4); - await expectCannot(service, "delete", pr1); - await expectCannot(service, "delete", pr2); - await expectCannot(service, "delete", pr3); - await expectCannot(service, "delete", pr4); + await expectAuthority(service, { + can: [ + [["read", "update"], pr1], + [["read", "update"], pr3], + [["read", "update"], pr4] + ], + cannot: [ + [["delete", "approve"], pr1], + [["read", "delete"], pr2], + ["delete", pr3], + ["delete", pr4] + ] + }); }); - it("allows reading project reports for managed projects", async () => { + it("allows managing project reports for managed projects", async () => { mockPermissions("projects-manage"); const user = await UserFactory.create(); mockUserId(user.id); @@ -83,9 +88,9 @@ describe("ProjectReportPolicy", () => { const pr1 = await ProjectReportFactory.create({ projectId: p1.id }); const pr2 = await ProjectReportFactory.create({ projectId: p2.id }); - await expectCan(service, "read", pr1); - await expectCannot(service, "read", pr2); - await expectCan(service, "delete", pr1); - await expectCannot(service, "delete", pr2); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], pr1]], + cannot: [[["read", "delete", "update", "approve"], pr2]] + }); }); }); diff --git a/libs/common/src/lib/policies/project.policy.spec.ts b/libs/common/src/lib/policies/project.policy.spec.ts index 173b93c1..ac100e42 100644 --- a/libs/common/src/lib/policies/project.policy.spec.ts +++ b/libs/common/src/lib/policies/project.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { Project } from "@terramatch-microservices/database/entities"; import { OrganisationFactory, @@ -38,18 +38,18 @@ describe("ProjectPolicy", () => { await expectCannot(service, "delete", new Project()); }); - it("allows reading and deleting projects in your framework", async () => { + it("allows managing projects in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await ProjectFactory.create({ frameworkKey: "ppc" }); const tf = await ProjectFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCan(service, "delete", ppc); - await expectCannot(service, "read", tf); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading and deleting own projects", async () => { + it("allows managing own projects", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -61,15 +61,20 @@ describe("ProjectPolicy", () => { const p4 = await ProjectFactory.create({ status: "awaiting-approval" }); await ProjectUserFactory.create({ userId: user.id, projectId: p3.id }); await ProjectUserFactory.create({ userId: user.id, projectId: p4.id, isMonitoring: false, isManaging: true }); - await expectCan(service, "read", p1); - await expectCan(service, "delete", p1); - await expectCannot(service, "read", p2); - await expectCannot(service, "delete", p2); - await expectCan(service, "read", p3); - await expectCan(service, "delete", p3); - await expectCan(service, "read", p4); - // This project is not in the "started" state - await expectCannot(service, "delete", p4); + await expectAuthority(service, { + can: [ + [["read", "update", "delete"], p1], + [["read", "update", "delete"], p3], + [["read", "update"], p4] + ], + cannot: [ + // manage-own does not give permission to approve. + ["approve", p1], + [["read", "update", "delete"], p2], + // This project is not in the "started" state + ["delete", p4] + ] + }); }); it("allows reading and deleting managed projects", async () => { @@ -79,9 +84,9 @@ describe("ProjectPolicy", () => { const p1 = await ProjectFactory.create(); const p2 = await ProjectFactory.create(); await ProjectUserFactory.create({ userId: user.id, projectId: p1.id, isMonitoring: false, isManaging: true }); - await expectCan(service, "read", p1); - await expectCan(service, "delete", p1); - await expectCannot(service, "read", p2); - await expectCannot(service, "delete", p2); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], p1]], + cannot: [[["read", "delete", "update", "approve"], p2]] + }); }); }); diff --git a/libs/common/src/lib/policies/site-report.policy.spec.ts b/libs/common/src/lib/policies/site-report.policy.spec.ts index 9cbf91cc..e88a2d1a 100644 --- a/libs/common/src/lib/policies/site-report.policy.spec.ts +++ b/libs/common/src/lib/policies/site-report.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { SiteReport } from "@terramatch-microservices/database/entities"; import { OrganisationFactory, @@ -33,18 +33,18 @@ describe("SiteReportPolicy", () => { await expectCannot(service, "delete", new SiteReport()); }); - it("allows reading site reports in your framework", async () => { + it("allows managing site reports in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await SiteReportFactory.create({ frameworkKey: "ppc" }); const tf = await SiteReportFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCannot(service, "read", tf); - await expectCan(service, "delete", ppc); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading site reports for own projects", async () => { + it("allows managing site reports for own projects", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -67,17 +67,22 @@ describe("SiteReportPolicy", () => { const sr3 = await SiteReportFactory.create({ siteId: s3.id }); const sr4 = await SiteReportFactory.create({ siteId: s4.id }); - await expectCan(service, "read", sr1); - await expectCannot(service, "read", sr2); - await expectCan(service, "read", sr3); - await expectCan(service, "read", sr4); - await expectCannot(service, "delete", sr1); - await expectCannot(service, "delete", sr2); - await expectCannot(service, "delete", sr3); - await expectCannot(service, "delete", sr4); + await expectAuthority(service, { + can: [ + [["read", "update"], sr1], + [["read", "update"], sr3], + [["read", "update"], sr4] + ], + cannot: [ + [["delete", "approve"], sr1], + [["read", "delete"], sr2], + ["delete", sr3], + ["delete", sr4] + ] + }); }); - it("allows reading site reports for managed projects", async () => { + it("allows managing site reports for managed projects", async () => { mockPermissions("projects-manage"); const user = await UserFactory.create(); mockUserId(user.id); @@ -92,9 +97,9 @@ describe("SiteReportPolicy", () => { const sr1 = await SiteReportFactory.create({ siteId: s1.id }); const sr2 = await SiteReportFactory.create({ siteId: s2.id }); - await expectCan(service, "read", sr1); - await expectCannot(service, "read", sr2); - await expectCan(service, "delete", sr1); - await expectCannot(service, "delete", sr2); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], sr1]], + cannot: [[["read", "delete", "update", "approve"], sr2]] + }); }); }); diff --git a/libs/common/src/lib/policies/site.policy.spec.ts b/libs/common/src/lib/policies/site.policy.spec.ts index d6410a2b..035c2fe5 100644 --- a/libs/common/src/lib/policies/site.policy.spec.ts +++ b/libs/common/src/lib/policies/site.policy.spec.ts @@ -1,6 +1,6 @@ import { PolicyService } from "./policy.service"; import { Test } from "@nestjs/testing"; -import { expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; +import { expectAuthority, expectCan, expectCannot, mockPermissions, mockUserId } from "./policy.service.spec"; import { Site } from "@terramatch-microservices/database/entities"; import { OrganisationFactory, @@ -38,18 +38,18 @@ describe("SitePolicy", () => { await expectCan(service, "read", new Site()); }); - it("allows reading sites in your framework", async () => { + it("allows managing sites in your framework", async () => { mockUserId(123); mockPermissions("framework-ppc"); const ppc = await SiteFactory.create({ frameworkKey: "ppc" }); const tf = await SiteFactory.create({ frameworkKey: "terrafund" }); - await expectCan(service, "read", ppc); - await expectCannot(service, "read", tf); - await expectCan(service, "delete", ppc); - await expectCannot(service, "delete", tf); + await expectAuthority(service, { + can: [[["read", "delete", "update", "approve"], ppc]], + cannot: [[["read", "delete", "update", "approve"], tf]] + }); }); - it("allows reading own sites", async () => { + it("allows managing own sites", async () => { mockPermissions("manage-own"); const org = await OrganisationFactory.create(); const user = await UserFactory.create({ organisationId: org.id }); @@ -68,17 +68,20 @@ describe("SitePolicy", () => { const s2 = await SiteFactory.create({ projectId: p2.id }); const s3 = await SiteFactory.create({ projectId: p3.id }); const s4 = await SiteFactory.create({ projectId: p4.id }); - await expectCan(service, "read", s1); - await expectCannot(service, "read", s2); - await expectCan(service, "read", s3); - await expectCan(service, "read", s4); - await expectCan(service, "delete", s1); - await expectCannot(service, "delete", s2); - await expectCan(service, "delete", s3); - await expectCan(service, "delete", s4); + await expectAuthority(service, { + can: [ + [["read", "delete", "update"], s1], + [["read", "delete", "update"], s3], + [["read", "delete", "update"], s4] + ], + cannot: [ + ["approve", s1], + [["read", "delete"], s2] + ] + }); }); - it("allows reading managed sites", async () => { + it("allows managing managed sites", async () => { mockPermissions("projects-manage"); const user = await UserFactory.create(); const project = await ProjectFactory.create(); @@ -86,9 +89,9 @@ describe("SitePolicy", () => { mockUserId(user.id); const s1 = await SiteFactory.create({ projectId: project.id }); const s2 = await SiteFactory.create(); - await expectCan(service, "read", s1); - await expectCannot(service, "read", s2); - await expectCan(service, "delete", s1); - await expectCannot(service, "delete", s2); + await expectAuthority(service, { + can: [[["read", "delete", "update"], s1]], + cannot: [[["read", "delete", "update"], s2]] + }); }); }); From d58b6f6c2485744db6ec4831c66f33b689e298e9 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 22:43:22 -0700 Subject: [PATCH 50/98] [TM-1955] Fix column definition of feedback fields. --- libs/database/src/lib/entities/project-report.entity.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index d94289de..8ce02efb 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -24,6 +24,7 @@ import { Literal } from "sequelize/types/utils"; import { User } from "./user.entity"; import { Task } from "./task.entity"; import { StateMachineColumn } from "../util/model-column-state-machine"; +import { JsonColumn } from "../decorators/json-column.decorator"; type ApprovedIdsSubqueryOptions = { dueAfter?: string | Date; @@ -181,7 +182,7 @@ export class ProjectReport extends Model { feedback: string | null; @AllowNull - @Column(TEXT) + @JsonColumn() feedbackFields: string[] | null; @AllowNull From 3c382b1c0dbdc36a7a5dc3802b1344e9fd415432 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 22:48:35 -0700 Subject: [PATCH 51/98] [TM-1955] Fix replacement of parent entity name for reports. --- libs/common/src/lib/email/entity-status-update.email.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index ef0f8504..e88ae5c9 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -80,7 +80,7 @@ export class EntityStatusUpdateEmail extends EmailSender { "{entityName}": (isReport(entity) ? "" : entity.name) ?? "", "{feedback}": (await this.getFeedback(entity)) ?? "(No feedback)" }; - if (isReport(entity)) i18nReplacements["{parentEntityName"] = this.getParentName(entity) ?? ""; + if (isReport(entity)) i18nReplacements["{parentEntityName}"] = this.getParentName(entity) ?? ""; const additionalValues = { link: getViewLinkPath(entity), @@ -127,9 +127,11 @@ export class EntityStatusUpdateEmail extends EmailSender { throw new InternalServerErrorException(`Entity model class not found for entity type [${this.type}]`); } - const attributes = ["id", "uuid", "status", "updateRequestStatus", "name", "feedback"]; const include = []; const attributeKeys = Object.keys(entityClass.getAttributes()); + const attributes = ["id", "uuid", "status", "updateRequestStatus", "name", "feedback"].filter(field => + attributeKeys.includes(field) + ); for (const parentId of ["projectId", "siteId", "nurseryId"]) { if (attributeKeys.includes(parentId)) { attributes.push(parentId); From a90fb63d8d1fb2202e7c62e847d46b818ffd819b Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 22 Apr 2025 22:53:11 -0700 Subject: [PATCH 52/98] [TM-1955] Fix empty feedback detection. --- libs/common/src/lib/email/entity-status-update.email.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index e88ae5c9..0d9e8785 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -74,11 +74,12 @@ export class EntityStatusUpdateEmail extends EmailSender { } const entityTypeName = isReport(entity) ? "Report" : entity.constructor.name; + const feedback = await this.getFeedback(entity); const i18nReplacements: Dictionary = { "{entityTypeName}": entityTypeName, "{lowerEntityTypeName}": entityTypeName.toLowerCase(), "{entityName}": (isReport(entity) ? "" : entity.name) ?? "", - "{feedback}": (await this.getFeedback(entity)) ?? "(No feedback)" + "{feedback}": feedback == null || feedback === "" ? "(No feedback)" : feedback }; if (isReport(entity)) i18nReplacements["{parentEntityName}"] = this.getParentName(entity) ?? ""; From c853c741394f465445d1dcfe5f0abc236488555f Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Wed, 23 Apr 2025 00:13:08 -0600 Subject: [PATCH 53/98] [TM-1959] feat: add Disturbance entity support in entity processing and status updates --- .../src/entities/entities.service.ts | 9 +- .../processors/disturbances.processor.ts | 159 ++++++++++++++++++ .../entities/processors/entity-processor.ts | 4 + .../lib/email/entity-status-update.email.ts | 3 + .../entity-status-update.event-processor.ts | 4 + libs/database/src/lib/constants/entities.ts | 11 +- .../src/lib/entities/disturbance.entity.ts | 32 ++++ 7 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 apps/entity-service/src/entities/processors/disturbances.processor.ts create mode 100644 libs/database/src/lib/entities/disturbance.entity.ts diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index 48ffb368..b0b1a3c4 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -26,6 +26,8 @@ import { PolicyService } from "@terramatch-microservices/common"; import { EntityUpdateData } from "./dto/entity-update.dto"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { ITranslateParams } from "@transifex/native"; +import { DisturbancesProcessor } from "./processors/disturbances.processor"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; // The keys of this array must match the type in the resulting DTO. const ENTITY_PROCESSORS = { @@ -34,7 +36,8 @@ const ENTITY_PROCESSORS = { nurseries: NurseryProcessor, projectReports: ProjectReportProcessor, nurseryReports: NurseryReportProcessor, - siteReports: SiteReportProcessor + siteReports: SiteReportProcessor, + disturbances: DisturbancesProcessor }; export type ProcessableEntity = keyof typeof ENTITY_PROCESSORS; @@ -94,7 +97,9 @@ export class EntitiesService { await this.policyService.authorize(action, subject); } - async isFrameworkAdmin({ frameworkKey }: T) { + async isFrameworkAdmin(T: EntityModel) { + if (T instanceof Disturbance) return; + const { frameworkKey } = T; return (await this.getPermissions()).includes(`framework-${frameworkKey}`); } diff --git a/apps/entity-service/src/entities/processors/disturbances.processor.ts b/apps/entity-service/src/entities/processors/disturbances.processor.ts new file mode 100644 index 00000000..9329a4a0 --- /dev/null +++ b/apps/entity-service/src/entities/processors/disturbances.processor.ts @@ -0,0 +1,159 @@ +import { Media, Nursery, NurseryReport, Project, ProjectUser } from "@terramatch-microservices/database/entities"; +import { AdditionalNurseryFullProps, NurseryFullDto, NurseryLightDto, NurseryMedia } from "../dto/nursery.dto"; +import { EntityProcessor } from "./entity-processor"; +import { EntityQueryDto } from "../dto/entity-query.dto"; +import { col, fn, Includeable, Op } from "sequelize"; +import { BadRequestException, NotAcceptableException } from "@nestjs/common"; +import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; +import { EntityUpdateAttributes } from "../dto/entity-update.dto"; + +export class DisturbancesProcessor extends EntityProcessor< + Nursery, + NurseryLightDto, + NurseryFullDto, + EntityUpdateAttributes +> { + readonly LIGHT_DTO = NurseryLightDto; + readonly FULL_DTO = NurseryFullDto; + + async findOne(uuid: string) { + return await Nursery.findOne({ + where: { uuid }, + include: [ + { + association: "project", + attributes: ["uuid", "name"], + include: [{ association: "organisation", attributes: ["name"] }] + } + ] + }); + } + + async findMany(query: EntityQueryDto) { + const projectAssociation: Includeable = { + association: "project", + attributes: ["uuid", "name"], + include: [{ association: "organisation", attributes: ["name"] }] + }; + + const builder = await this.entitiesService.buildQuery(Nursery, query, [projectAssociation]); + if (query.sort != null) { + if (["name", "startDate", "status", "updateRequestStatus", "createdAt"].includes(query.sort.field)) { + builder.order([query.sort.field, query.sort.direction ?? "ASC"]); + } else if (query.sort.field === "organisationName") { + builder.order(["project", "organisation", "name", query.sort.direction ?? "ASC"]); + } else if (query.sort.field === "projectName") { + builder.order(["project", "name", query.sort.direction ?? "ASC"]); + } else if (query.sort.field !== "id") { + throw new BadRequestException(`Invalid sort field: ${query.sort.field}`); + } + } + + const permissions = await this.entitiesService.getPermissions(); + const frameworkPermissions = permissions + ?.filter(name => name.startsWith("framework-")) + .map(name => name.substring("framework-".length) as FrameworkKey); + if (frameworkPermissions?.length > 0) { + builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); + } else if (permissions?.includes("manage-own")) { + builder.where({ projectId: { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } }); + } else if (permissions?.includes("projects-manage")) { + builder.where({ projectId: { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } }); + } + + const associationFieldMap = { + organisationUuid: "$project.organisation.uuid$", + country: "$project.country$", + projectUuid: "$project.uuid$" + }; + + for (const term of [ + "status", + "updateRequestStatus", + "frameworkKey", + "organisationUuid", + "country", + "projectUuid" + ]) { + if (query[term] != null) { + const field = associationFieldMap[term] ?? term; + builder.where({ [field]: query[term] }); + } + } + + if (query.search != null || query.searchFilter != null) { + builder.where({ + [Op.or]: [ + { name: { [Op.like]: `%${query.search ?? query.searchFilter}%` } }, + { "$project.name$": { [Op.like]: `%${query.search}%` } }, + { "$project.organisation.name$": { [Op.like]: `%${query.search}%` } } + ] + }); + } + + if (query.projectUuid != null) { + const project = await Project.findOne({ where: { uuid: query.projectUuid }, attributes: ["id"] }); + if (project == null) { + throw new BadRequestException(`Project with uuid ${query.projectUuid} not found`); + } + builder.where({ projectId: project.id }); + } + + return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; + } + + async getFullDto(nursery: Nursery) { + const nurseryId = nursery.id; + + const nurseryReportsTotal = await NurseryReport.nurseries([nurseryId]).count(); + const seedlingsGrownCount = await this.getSeedlingsGrownCount(nurseryId); + const overdueNurseryReportsTotal = await this.getTotalOverdueReports(nurseryId); + const props: AdditionalNurseryFullProps = { + seedlingsGrownCount, + nurseryReportsTotal, + overdueNurseryReportsTotal, + + ...(this.entitiesService.mapMediaCollection(await Media.for(nursery).findAll(), Nursery.MEDIA) as NurseryMedia) + }; + + return { id: nursery.uuid, dto: new NurseryFullDto(nursery, props) }; + } + + async getLightDto(nursery: Nursery) { + const nurseryId = nursery.id; + + const seedlingsGrownCount = await this.getSeedlingsGrownCount(nurseryId); + return { id: nursery.uuid, dto: new NurseryLightDto(nursery, { seedlingsGrownCount }) }; + } + + protected async getTotalOverdueReports(nurseryId: number) { + const countOpts = { where: { dueAt: { [Op.lt]: new Date() } } }; + return await NurseryReport.incomplete().nurseries([nurseryId]).count(countOpts); + } + + private async getSeedlingsGrownCount(nurseryId: number) { + return ( + ( + await NurseryReport.nurseries([nurseryId]) + .approved() + .findAll({ + raw: true, + attributes: [[fn("SUM", col("seedlings_young_trees")), "seedlingsYoungTrees"]] + }) + )[0].seedlingsYoungTrees ?? 0 + ); + } + + async delete(nursery: Nursery) { + const permissions = await this.entitiesService.getPermissions(); + const managesOwn = permissions.includes("manage-own") && !permissions.includes(`framework-${nursery.frameworkKey}`); + if (managesOwn) { + const reportCount = await NurseryReport.count({ where: { nurseryId: nursery.id } }); + if (reportCount > 0) { + throw new NotAcceptableException("You can only delete nurseries that do not have reports"); + } + } + + await super.delete(nursery); + } +} diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index c7a7cc1e..26a0aa71 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -9,6 +9,8 @@ import { EntityModel } from "@terramatch-microservices/database/constants/entiti import { Action } from "@terramatch-microservices/database/entities/action.entity"; import { EntityUpdateData } from "../dto/entity-update.dto"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; +import { Project } from "@terramatch-microservices/database/entities"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; export type Aggregate> = { func: string; @@ -117,6 +119,8 @@ export abstract class EntityProcessor< * and set the appropriate fields and then call super.update() */ async update(model: ModelType, update: UpdateDto) { + if (model instanceof Disturbance) return; + if (update.status != null) { if (this.APPROVAL_STATUSES.includes(update.status)) { await this.entitiesService.authorize("approve", model); diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index ef0f8504..2141981e 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -22,6 +22,7 @@ import { Op } from "sequelize"; import { TMLogger } from "../util/tm-logger"; import { InternalServerErrorException } from "@nestjs/common"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; export class EntityStatusUpdateEmail extends EmailSender { private readonly logger = new TMLogger(EntityStatusUpdateEmail.name); @@ -37,6 +38,7 @@ export class EntityStatusUpdateEmail extends EmailSender { async send(emailService: EmailService) { const entity = await this.getEntity(); + if (entity instanceof Disturbance) return; const status = entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION ? NEEDS_MORE_INFORMATION @@ -110,6 +112,7 @@ export class EntityStatusUpdateEmail extends EmailSender { } private async getFeedback(entity: EntityModel) { + if (entity instanceof Disturbance) return; if (![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.updateRequestStatus ?? "")) { return entity.feedback; } diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index 7adb7692..12d87b78 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -15,6 +15,7 @@ import { Action, AuditStatus, FormQuestion } from "@terramatch-microservices/dat import { get, isEmpty, map } from "lodash"; import { Op } from "sequelize"; import { STATUS_DISPLAY_STRINGS } from "@terramatch-microservices/database/constants/status"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; @@ -53,6 +54,9 @@ export class EntityStatusUpdate extends EventProcessor { private async updateActions() { this.logger.log(`Updating actions [${JSON.stringify({ model: this.model.constructor.name, id: this.model.id })}]`); const entity = this.model as EntityModel; + + if (entity instanceof Disturbance) return; + await Action.for(entity).destroy({ where: { type: "notification" } }); if (entity.status !== "awaiting-approval") { diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 97f92b30..eb2d1c82 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -2,6 +2,7 @@ import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport } from import { ModelCtor } from "sequelize-typescript"; import { ModelStatic } from "sequelize"; import { kebabCase } from "lodash"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; export const REPORT_TYPES = ["projectReports", "siteReports", "nurseryReports"] as const; export type ReportType = (typeof REPORT_TYPES)[number]; @@ -14,16 +15,19 @@ export const REPORT_MODELS: { [R in ReportType]: ReportClass } = { nurseryReports: NurseryReport }; -export const ENTITY_TYPES = ["projects", "sites", "nurseries", ...REPORT_TYPES] as const; +export const ENTITY_TYPES = ["projects", "sites", "nurseries", "disturbances", ...REPORT_TYPES] as const; export type EntityType = (typeof ENTITY_TYPES)[number]; -export type EntityModel = ReportModel | Project | Site | Nursery; +export type EntityModel = ReportModel | Project | Site | Nursery | Disturbance; export type EntityClass = ModelCtor & ModelStatic & { LARAVEL_TYPE: string }; export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { ...REPORT_MODELS, projects: Project, sites: Site, - nurseries: Nursery + nurseries: Nursery, + disturbances: Disturbance + // stratas: Strata::class, + //'invasives: Invasive::class, }; export const isReport = (entity: EntityModel): entity is ReportModel => @@ -38,6 +42,7 @@ export const isReport = (entity: EntityModel): entity is ReportModel => */ export async function getProjectId(entity: EntityModel) { if (entity instanceof Project) return entity.id; + if (entity instanceof Disturbance) return entity.id; if (entity instanceof Site || entity instanceof Nursery || entity instanceof ProjectReport) return entity.projectId; const parentClass: ModelCtor = entity instanceof SiteReport ? Site : Nursery; diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts new file mode 100644 index 00000000..23a495d0 --- /dev/null +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -0,0 +1,32 @@ +import { + AllowNull, + AutoIncrement, + Column, + HasMany, + Index, + Model, + PrimaryKey, + Scopes, + Table +} from "sequelize-typescript"; +import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; + +@Table({ tableName: "v2_disturbances", underscored: true, paranoid: true }) +export class Disturbance extends Model { + static readonly LARAVEL_TYPE = "App\\Models\\SiteSubmissionDisturbance"; + + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column({ type: UUID, defaultValue: UUIDV4 }) + uuid: string; + + @Column(BIGINT.UNSIGNED) + disturbanceableType: number; + + @Column(BIGINT.UNSIGNED) + disturbanceableId: number; +} From 5a4bb00982ddf71b0b42191e3c291aaafa999298 Mon Sep 17 00:00:00 2001 From: LimberHope Date: Wed, 23 Apr 2025 10:48:33 -0400 Subject: [PATCH 54/98] [TM-1953] add entities service user to permissions --- .../src/entities/processors/site-report.processor.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/entity-service/src/entities/processors/site-report.processor.ts b/apps/entity-service/src/entities/processors/site-report.processor.ts index 619aa49f..b08f94a7 100644 --- a/apps/entity-service/src/entities/processors/site-report.processor.ts +++ b/apps/entity-service/src/entities/processors/site-report.processor.ts @@ -60,7 +60,7 @@ export class SiteReportProcessor extends EntityProcessor< }); } - async findMany(query: EntityQueryDto, userId?: number) { + async findMany(query: EntityQueryDto) { const siteAssociation: Includeable = { association: "site", attributes: ["id", "uuid", "name"], @@ -93,9 +93,13 @@ export class SiteReportProcessor extends EntityProcessor< if (frameworkPermissions?.length > 0) { builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); } else if (permissions?.includes("manage-own")) { - builder.where({ "$site.project.id$": { [Op.in]: ProjectUser.userProjectsSubquery(userId) } }); + builder.where({ + "$site.project.id$": { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } + }); } else if (permissions?.includes("projects-manage")) { - builder.where({ "$site.project.id$": { [Op.in]: ProjectUser.projectsManageSubquery(userId) } }); + builder.where({ + "$site.project.id$": { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } + }); } const associationFieldMap = { From a9ff19f90e6720f877f52711ea068377f7f47ec2 Mon Sep 17 00:00:00 2001 From: LimberHope Date: Wed, 23 Apr 2025 12:15:49 -0400 Subject: [PATCH 55/98] [TM-1953] add entities service user to permissions --- .../entities/processors/nursery-report.processor.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/entity-service/src/entities/processors/nursery-report.processor.ts b/apps/entity-service/src/entities/processors/nursery-report.processor.ts index 34f4b3e7..8fb59c5f 100644 --- a/apps/entity-service/src/entities/processors/nursery-report.processor.ts +++ b/apps/entity-service/src/entities/processors/nursery-report.processor.ts @@ -52,7 +52,7 @@ export class NurseryReportProcessor extends EntityProcessor< }); } - async findMany(query: EntityQueryDto, userId?: number) { + async findMany(query: EntityQueryDto) { const nurseryAssociation: Includeable = { association: "nursery", attributes: ["id", "uuid", "name"], @@ -85,9 +85,13 @@ export class NurseryReportProcessor extends EntityProcessor< if (frameworkPermissions?.length > 0) { builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); } else if (permissions?.includes("manage-own")) { - builder.where({ "$nursery.project.id$": { [Op.in]: ProjectUser.userProjectsSubquery(userId) } }); + builder.where({ + "$nursery.project.id$": { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } + }); } else if (permissions?.includes("projects-manage")) { - builder.where({ "$nursery.project.id$": { [Op.in]: ProjectUser.projectsManageSubquery(userId) } }); + builder.where({ + "$nursery.project.id$": { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } + }); } const associationFieldMap = { From ed02175b28e00f6a3372f8bc6289681ca7f4b718 Mon Sep 17 00:00:00 2001 From: LimberHope Date: Wed, 23 Apr 2025 12:31:01 -0400 Subject: [PATCH 56/98] [TM-1953] update site and nursery-report processor spect --- .../src/entities/processors/nursery-report.processor.spec.ts | 2 +- .../src/entities/processors/site-report.processor.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/entity-service/src/entities/processors/nursery-report.processor.spec.ts b/apps/entity-service/src/entities/processors/nursery-report.processor.spec.ts index f7f69fa7..3399e033 100644 --- a/apps/entity-service/src/entities/processors/nursery-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/nursery-report.processor.spec.ts @@ -60,7 +60,7 @@ describe("NurseryReportProcessor", () => { }: { permissions?: string[]; sortField?: string; sortUp?: boolean; total?: number } = {} ) { policyService.getPermissions.mockResolvedValue(permissions); - const { models, paginationTotal } = await processor.findMany(query as EntityQueryDto, userId); + const { models, paginationTotal } = await processor.findMany(query as EntityQueryDto); expect(models.length).toBe(expected.length); expect(paginationTotal).toBe(total); diff --git a/apps/entity-service/src/entities/processors/site-report.processor.spec.ts b/apps/entity-service/src/entities/processors/site-report.processor.spec.ts index c2b65bce..ed14eaea 100644 --- a/apps/entity-service/src/entities/processors/site-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/site-report.processor.spec.ts @@ -56,7 +56,7 @@ describe("SiteReportProcessor", () => { }: { permissions?: string[]; sortField?: string; sortUp?: boolean; total?: number } = {} ) { policyService.getPermissions.mockResolvedValue(permissions); - const { models, paginationTotal } = await processor.findMany(query as EntityQueryDto, userId); + const { models, paginationTotal } = await processor.findMany(query as EntityQueryDto); expect(models.length).toBe(expected.length); expect(paginationTotal).toBe(total); From 9f9aa459eaea8b8c44c93e1366b3a47701e629f0 Mon Sep 17 00:00:00 2001 From: JORGE Date: Wed, 23 Apr 2025 14:51:08 -0400 Subject: [PATCH 57/98] [TM-1922] add sideload for associations --- .../src/entities/dto/entity-query.dto.ts | 16 ++++++--- .../entities/processors/entity-processor.ts | 4 +-- .../processors/project-report.processor.ts | 35 ++++++++++++++++++- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/apps/entity-service/src/entities/dto/entity-query.dto.ts b/apps/entity-service/src/entities/dto/entity-query.dto.ts index bd39a986..836dd11e 100644 --- a/apps/entity-service/src/entities/dto/entity-query.dto.ts +++ b/apps/entity-service/src/entities/dto/entity-query.dto.ts @@ -4,12 +4,16 @@ import { NumberPage } from "@terramatch-microservices/common/dto/page.dto"; import { MAX_PAGE_SIZE, PROCESSABLE_ENTITIES, - ProcessableEntity, + PROCESSABLE_ASSOCIATIONS, POLYGON_STATUSES_FILTERS, PolygonStatusFilter } from "../entities.service"; import { Type } from "class-transformer"; +export const VALID_SIDELOAD_TYPES = [...PROCESSABLE_ENTITIES, ...PROCESSABLE_ASSOCIATIONS] as const; + +export type SideloadType = (typeof VALID_SIDELOAD_TYPES)[number]; + class QuerySort { @ApiProperty({ name: "sort[field]", required: false }) @IsOptional() @@ -22,9 +26,13 @@ class QuerySort { } export class EntitySideload { - @IsIn(PROCESSABLE_ENTITIES) - @ApiProperty({ name: "entity", enum: PROCESSABLE_ENTITIES, description: "Entity type to sideload" }) - entity: ProcessableEntity; + @IsIn(VALID_SIDELOAD_TYPES) + @ApiProperty({ + name: "entity", + enum: VALID_SIDELOAD_TYPES, + description: "Entity or association type to sideload" + }) + entity: SideloadType; @ApiProperty({ name: "pageSize", description: "The page size to include." }) @IsInt() diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index c7a7cc1e..342bb853 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -2,7 +2,7 @@ import { Model, ModelCtor } from "sequelize-typescript"; import { Attributes, col, fn, WhereOptions } from "sequelize"; import { DocumentBuilder, getStableRequestQuery, IndexData } from "@terramatch-microservices/common/util"; import { EntitiesService, ProcessableEntity } from "../entities.service"; -import { EntityQueryDto } from "../dto/entity-query.dto"; +import { EntityQueryDto, SideloadType } from "../dto/entity-query.dto"; import { BadRequestException, Type } from "@nestjs/common"; import { EntityDto } from "../dto/entity.dto"; import { EntityModel } from "@terramatch-microservices/database/constants/entities"; @@ -72,7 +72,7 @@ export abstract class EntityProcessor< async processSideload( document: DocumentBuilder, model: ModelType, - entity: ProcessableEntity, + entity: SideloadType, pageSize: number ): Promise { throw new BadRequestException("This entity does not support sideloading"); diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index 46b69802..301c0885 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -6,7 +6,7 @@ import { ProjectReportLightDto, ProjectReportMedia } from "../dto/project-report.dto"; -import { EntityQueryDto } from "../dto/entity-query.dto"; +import { EntityQueryDto, SideloadType } from "../dto/entity-query.dto"; import { Includeable, Op } from "sequelize"; import { BadRequestException } from "@nestjs/common"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; @@ -21,6 +21,10 @@ import { } from "@terramatch-microservices/database/entities"; import { sumBy } from "lodash"; import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { ProcessableAssociation } from "../entities.service"; +import { DocumentBuilder } from "@terramatch-microservices/common/util"; + +const SUPPORTED_ASSOCIATIONS: ProcessableAssociation[] = ["demographics", "seedings", "treeSpecies"]; export class ProjectReportProcessor extends EntityProcessor< ProjectReport, @@ -128,6 +132,35 @@ export class ProjectReportProcessor extends EntityProcessor< return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; } + async processSideload( + document: DocumentBuilder, + model: ProjectReport, + entity: SideloadType, + pageSize: number + ): Promise { + if (SUPPORTED_ASSOCIATIONS.includes(entity as ProcessableAssociation)) { + const processor = this.entitiesService.createAssociationProcessor( + "projectReports", + model.uuid, + entity as ProcessableAssociation + ); + const entityDocument = new DocumentBuilder(entity as string); + await processor.addDtos(entityDocument); + const serialized = entityDocument.serialize(); + if (serialized.data) { + if (Array.isArray(serialized.data)) { + serialized.data.forEach(resource => { + document.addIncluded(resource.id, resource.attributes); + }); + } else { + document.addIncluded(serialized.data.id, serialized.data.attributes); + } + } + } else { + throw new BadRequestException(`Project reports only support sideloading: ${SUPPORTED_ASSOCIATIONS.join(", ")}`); + } + } + async getFullDto(projectReport: ProjectReport) { const taskId = projectReport.taskId; const reportTitle = await this.getReportTitle(projectReport); From 2bc8411b1b17ebe5fef441d75e974e143184d4ed Mon Sep 17 00:00:00 2001 From: JORGE Date: Wed, 23 Apr 2025 15:08:33 -0400 Subject: [PATCH 58/98] [TM-1922]remove unused variables --- .../src/entities/processors/project-report.processor.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index edbed1e4..e7a3bcc9 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -132,12 +132,7 @@ export class ProjectReportProcessor extends ReportProcessor< return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; } - async processSideload( - document: DocumentBuilder, - model: ProjectReport, - entity: SideloadType, - pageSize: number - ): Promise { + async processSideload(document: DocumentBuilder, model: ProjectReport, entity: SideloadType): Promise { if (SUPPORTED_ASSOCIATIONS.includes(entity as ProcessableAssociation)) { const processor = this.entitiesService.createAssociationProcessor( "projectReports", From a61bb7c4612815fd3a9ab1cd4cefc5745d3869f1 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 24 Apr 2025 06:22:02 -0600 Subject: [PATCH 59/98] [TM-1937] remove nothingToReport from fulldto and change data type in entities --- apps/entity-service/src/entities/dto/nursery-report.dto.ts | 3 --- apps/entity-service/src/entities/dto/site-report.dto.ts | 3 --- libs/database/src/lib/entities/nursery-report.entity.ts | 4 ++-- libs/database/src/lib/entities/site-report.entity.ts | 4 ++-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/entity-service/src/entities/dto/nursery-report.dto.ts b/apps/entity-service/src/entities/dto/nursery-report.dto.ts index 38ee638d..f02800b2 100644 --- a/apps/entity-service/src/entities/dto/nursery-report.dto.ts +++ b/apps/entity-service/src/entities/dto/nursery-report.dto.ts @@ -159,9 +159,6 @@ export class NurseryReportFullDto extends NurseryReportLightDto { @ApiProperty({ nullable: true }) feedbackFields: string[] | null; - @ApiProperty() - nothingToReport: boolean; - @ApiProperty({ nullable: true }) completion: number | null; diff --git a/apps/entity-service/src/entities/dto/site-report.dto.ts b/apps/entity-service/src/entities/dto/site-report.dto.ts index b3240da5..33b21dd2 100644 --- a/apps/entity-service/src/entities/dto/site-report.dto.ts +++ b/apps/entity-service/src/entities/dto/site-report.dto.ts @@ -155,9 +155,6 @@ export class SiteReportFullDto extends SiteReportLightDto { @ApiProperty({ nullable: true }) feedbackFields: string[] | null; - @ApiProperty({ nullable: true }) - nothingToReport: boolean | null; - @ApiProperty({ nullable: true }) completion: number | null; diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index 3288100f..070a25c5 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; import { Nursery } from "./nursery.entity"; import { TreeSpecies } from "./tree-species.entity"; import { COMPLETE_REPORT_STATUSES, ReportStatus, UpdateRequestStatus } from "../constants/status"; @@ -171,7 +171,7 @@ export class NurseryReport extends Model { seedlingsYoungTrees: number | null; @AllowNull - @Column(TINYINT) + @Column(BOOLEAN) nothingToReport: boolean; @AllowNull diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 45155618..d83f5d33 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; @@ -304,7 +304,7 @@ export class SiteReport extends Model { feedbackFields: string[] | null; @AllowNull - @Column(TINYINT) + @Column(BOOLEAN) nothingToReport: boolean | null; @HasMany(() => TreeSpecies, { From 05e7142836011f13c37fbcc590c83e5003deab7f Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 24 Apr 2025 07:08:58 -0600 Subject: [PATCH 60/98] [TM-1937] Fx uuidv4 issue --- libs/database/src/lib/entities/nursery-report.entity.ts | 2 +- libs/database/src/lib/entities/site-report.entity.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/database/src/lib/entities/nursery-report.entity.ts b/libs/database/src/lib/entities/nursery-report.entity.ts index bb247220..2fcb20f0 100644 --- a/libs/database/src/lib/entities/nursery-report.entity.ts +++ b/libs/database/src/lib/entities/nursery-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { Nursery } from "./nursery.entity"; import { TreeSpecies } from "./tree-species.entity"; import { COMPLETE_REPORT_STATUSES, ReportStatus, ReportStatusStates, UpdateRequestStatus } from "../constants/status"; diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 3b5bfb7f..c9affb2a 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -11,7 +11,7 @@ import { Scopes, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; From d75fadef9c7b321b3ba3058f98664a16c2e17441 Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 10:01:58 -0400 Subject: [PATCH 61/98] [TM-1922] add included dto function --- .../entity-associations.controller.spec.ts | 1 + .../processors/association-processor.ts | 18 ++++++++++ .../project-report.processor.spec.ts | 35 +++++++++++++++++++ .../processors/project-report.processor.ts | 13 +------ 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/apps/entity-service/src/entities/entity-associations.controller.spec.ts b/apps/entity-service/src/entities/entity-associations.controller.spec.ts index 5db280af..0f8c02e1 100644 --- a/apps/entity-service/src/entities/entity-associations.controller.spec.ts +++ b/apps/entity-service/src/entities/entity-associations.controller.spec.ts @@ -14,6 +14,7 @@ class StubProcessor extends AssociationProcessor { DTO = DemographicDto; addDtos = jest.fn(() => Promise.resolve()); + addIncludedDtos = jest.fn(() => Promise.resolve()); getAssociations = jest.fn(() => Promise.resolve([] as Demographic[])); } diff --git a/apps/entity-service/src/entities/processors/association-processor.ts b/apps/entity-service/src/entities/processors/association-processor.ts index b042d1fe..38cf91e4 100644 --- a/apps/entity-service/src/entities/processors/association-processor.ts +++ b/apps/entity-service/src/entities/processors/association-processor.ts @@ -70,4 +70,22 @@ export abstract class AssociationProcessor { + const associations = await this.getAssociations(await this.getBaseEntity()); + + const additionalProps = { entityType: this.entityType, entityUuid: this.entityUuid }; + const indexIds: string[] = []; + for (const association of associations) { + indexIds.push(association.uuid); + document.addIncluded(association.uuid, new this.DTO(association, additionalProps)); + } + + const resource = getDtoType(this.DTO); + document.addIndexData({ + resource, + requestPath: `/entities/v3/${this.entityType}/${this.entityUuid}/${resource}`, + ids: indexIds + }); + } } diff --git a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts index ca07465a..1b12f45b 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts @@ -17,11 +17,15 @@ import { ProjectReportProcessor } from "./project-report.processor"; import { DateTime } from "luxon"; import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; +import { DocumentBuilder } from "@terramatch-microservices/common/util"; +import { AssociationProcessor } from "./association-processor"; describe("ProjectReportProcessor", () => { let processor: ProjectReportProcessor; let policyService: DeepMocked; let userId: number; + let entitiesService: DeepMocked; + let mockAssociationProcessor: Partial>; beforeAll(async () => { userId = (await UserFactory.create()).id; @@ -40,6 +44,20 @@ describe("ProjectReportProcessor", () => { }).compile(); processor = module.get(EntitiesService).createEntityProcessor("projectReports") as ProjectReportProcessor; + + mockAssociationProcessor = { + addIncludedDtos: jest.fn().mockResolvedValue(undefined) + }; + + entitiesService = createMock({ + createAssociationProcessor: jest.fn().mockReturnValue(mockAssociationProcessor) + }); + + const moduleRef = await Test.createTestingModule({ + providers: [ProjectReportProcessor, { provide: EntitiesService, useValue: entitiesService }] + }).compile(); + + processor = moduleRef.get(ProjectReportProcessor); }); describe("findMany", () => { @@ -432,4 +450,21 @@ describe("ProjectReportProcessor", () => { }); }); }); + + describe("processSideload", () => { + it("should use addIncludedDtos for supported associations", async () => { + const document = new DocumentBuilder("projectReports"); + const projectReport = new ProjectReport(); + projectReport.uuid = "test-uuid"; + + await processor.processSideload(document, projectReport, "demographics"); + + expect(entitiesService.createAssociationProcessor).toHaveBeenCalledWith( + "projectReports", + "test-uuid", + "demographics" + ); + expect(mockAssociationProcessor.addIncludedDtos).toHaveBeenCalledWith(document); + }); + }); }); diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index e7a3bcc9..c6a7b226 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -139,18 +139,7 @@ export class ProjectReportProcessor extends ReportProcessor< model.uuid, entity as ProcessableAssociation ); - const entityDocument = new DocumentBuilder(entity as string); - await processor.addDtos(entityDocument); - const serialized = entityDocument.serialize(); - if (serialized.data) { - if (Array.isArray(serialized.data)) { - serialized.data.forEach(resource => { - document.addIncluded(resource.id, resource.attributes); - }); - } else { - document.addIncluded(serialized.data.id, serialized.data.attributes); - } - } + await processor.addIncludedDtos(document); } else { throw new BadRequestException(`Project reports only support sideloading: ${SUPPORTED_ASSOCIATIONS.join(", ")}`); } From 073dd30c421f94bf22d563dc8bf2d8dacc451986 Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 10:10:29 -0400 Subject: [PATCH 62/98] [TM-1922] change any to type --- .../src/entities/processors/project-report.processor.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts index 1b12f45b..77ba7913 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts @@ -19,13 +19,15 @@ import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { DocumentBuilder } from "@terramatch-microservices/common/util"; import { AssociationProcessor } from "./association-processor"; +import { UuidModel } from "@terramatch-microservices/database/types/util"; +import { AssociationDto } from "../dto/association.dto"; describe("ProjectReportProcessor", () => { let processor: ProjectReportProcessor; let policyService: DeepMocked; let userId: number; let entitiesService: DeepMocked; - let mockAssociationProcessor: Partial>; + let mockAssociationProcessor: Partial>>; beforeAll(async () => { userId = (await UserFactory.create()).id; From 01dc0ce7dcc5b483b295b75155a1d3b6ebb61c5a Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 24 Apr 2025 08:24:25 -0600 Subject: [PATCH 63/98] [TM-1937] Fix status check --- libs/database/src/lib/constants/status.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index a1abe88c..5df96b11 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -62,7 +62,7 @@ export const ReportStatusStates: States = { transitionValidForModel: (from: ReportStatus, to: ReportStatus, report: ReportModel) => { if ((from === DUE && to === AWAITING_APPROVAL) || (from === AWAITING_APPROVAL && to === STARTED)) { // these two transitions are only allowed for site / nursery reports when the nothingToReport flag is true; - return !(report instanceof ProjectReport) && report.nothingToReport; + return !(report instanceof ProjectReport) && (report.nothingToReport || false); } return true; From 212e42eb5998680619088b9fd1d5728b0fec957e Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 24 Apr 2025 10:08:07 -0600 Subject: [PATCH 64/98] [TM-1959] feat: update DisturbancesProcessor to support Disturbance entity and related DTOs --- .../src/entities/dto/disturbance.dto.ts | 138 ++++++++++++++++++ .../processors/disturbances.processor.ts | 38 ++--- libs/database/src/lib/entities/index.ts | 1 + 3 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 apps/entity-service/src/entities/dto/disturbance.dto.ts diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts new file mode 100644 index 00000000..fe591a28 --- /dev/null +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -0,0 +1,138 @@ +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { AdditionalProps, EntityDto } from "./entity.dto"; +import { Nursery } from "@terramatch-microservices/database/entities"; +import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { ApiProperty } from "@nestjs/swagger"; +import { + ENTITY_STATUSES, + EntityStatus, + UPDATE_REQUEST_STATUSES, + UpdateRequestStatus +} from "@terramatch-microservices/database/constants/status"; +import { MediaDto } from "./media.dto"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; + +@JsonApiDto({ type: "nurseries" }) +export class DisturbanceLightDto extends EntityDto { + constructor(nursery?: Disturbance, props?: AdditionalNurseryLightProps) { + super(); + if (nursery != null) { + this.populate(DisturbanceLightDto, { + ...pickApiProperties(nursery, DisturbanceLightDto), + lightResource: true, + ...props, + // these two are untyped and marked optional in the base model. + createdAt: nursery.createdAt as Date, + updatedAt: nursery.updatedAt as Date + }); + } + } + + @ApiProperty({ nullable: true }) + name: string | null; + + @ApiProperty({ nullable: true, description: "Framework key for this nursery" }) + frameworkKey: string | null; + + @ApiProperty({ + nullable: true, + description: "Entity status for this nursery", + enum: ENTITY_STATUSES + }) + status: EntityStatus | null; + + @ApiProperty({ + nullable: true, + description: "Update request status for this nursery", + enum: UPDATE_REQUEST_STATUSES + }) + updateRequestStatus: UpdateRequestStatus | null; + + @ApiProperty({ + nullable: true, + description: "The associated project name" + }) + projectName: string | null; + + @ApiProperty({ + nullable: true, + description: "The associated project organisation name" + }) + organisationName: string | null; + + @ApiProperty({ nullable: true }) + startDate: Date | null; + + @ApiProperty({ nullable: true }) + endDate: Date | null; + + @ApiProperty({ nullable: true }) + seedlingsGrownCount: number | null; + + @ApiProperty() + createdAt: Date; + + @ApiProperty() + updatedAt: Date; +} + +export type AdditionalNurseryLightProps = Pick; +export type AdditionalNurseryFullProps = AdditionalNurseryLightProps & + AdditionalProps>; +export type NurseryMedia = Pick; + +export class DisturbanceFullDto extends DisturbanceLightDto { + constructor(nursery: Disturbance, props: AdditionalNurseryFullProps) { + super(); + this.populate(DisturbanceFullDto, { + ...pickApiProperties(nursery, DisturbanceFullDto), + lightResource: false, + // these two are untyped and marked optional in the base model. + createdAt: nursery.createdAt as Date, + updatedAt: nursery.updatedAt as Date, + ...props + }); + } + + @ApiProperty({ nullable: true }) + feedback: string | null; + + @ApiProperty({ nullable: true }) + feedbackFields: string[] | null; + + @ApiProperty({ nullable: true }) + type: string | null; + + @ApiProperty({ nullable: true }) + seedlingGrown: number | null; + + @ApiProperty({ nullable: true }) + plantingContribution: string | null; + + @ApiProperty({ nullable: true }) + oldModel: string | null; + + @ApiProperty({ nullable: true }) + nurseryReportsTotal: number | null; + + @ApiProperty({ nullable: true }) + overdueNurseryReportsTotal: number | null; + + @ApiProperty({ nullable: true }) + organisationName: string | null; + + @ApiProperty({ nullable: true }) + projectName: string | null; + + @ApiProperty({ nullable: true }) + projectUuid: string | null; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + file: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + otherAdditionalDocuments: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + photos: MediaDto[]; +} diff --git a/apps/entity-service/src/entities/processors/disturbances.processor.ts b/apps/entity-service/src/entities/processors/disturbances.processor.ts index 9329a4a0..75e860e0 100644 --- a/apps/entity-service/src/entities/processors/disturbances.processor.ts +++ b/apps/entity-service/src/entities/processors/disturbances.processor.ts @@ -6,26 +6,28 @@ import { col, fn, Includeable, Op } from "sequelize"; import { BadRequestException, NotAcceptableException } from "@nestjs/common"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { DisturbanceFullDto, DisturbanceLightDto } from "../dto/disturbance.dto"; export class DisturbancesProcessor extends EntityProcessor< - Nursery, - NurseryLightDto, - NurseryFullDto, + Disturbance, + DisturbanceLightDto, + DisturbanceFullDto, EntityUpdateAttributes > { - readonly LIGHT_DTO = NurseryLightDto; - readonly FULL_DTO = NurseryFullDto; + readonly LIGHT_DTO = DisturbanceLightDto; + readonly FULL_DTO = DisturbanceFullDto; async findOne(uuid: string) { - return await Nursery.findOne({ - where: { uuid }, - include: [ + return await Disturbance.findOne({ + where: { uuid } + /*include: [ { association: "project", attributes: ["uuid", "name"], include: [{ association: "organisation", attributes: ["name"] }] } - ] + ]*/ }); } @@ -36,7 +38,7 @@ export class DisturbancesProcessor extends EntityProcessor< include: [{ association: "organisation", attributes: ["name"] }] }; - const builder = await this.entitiesService.buildQuery(Nursery, query, [projectAssociation]); + const builder = await this.entitiesService.buildQuery(Disturbance, query, [projectAssociation]); if (query.sort != null) { if (["name", "startDate", "status", "updateRequestStatus", "createdAt"].includes(query.sort.field)) { builder.order([query.sort.field, query.sort.direction ?? "ASC"]); @@ -102,7 +104,7 @@ export class DisturbancesProcessor extends EntityProcessor< return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; } - async getFullDto(nursery: Nursery) { + async getFullDto(nursery: Disturbance) { const nurseryId = nursery.id; const nurseryReportsTotal = await NurseryReport.nurseries([nurseryId]).count(); @@ -116,14 +118,14 @@ export class DisturbancesProcessor extends EntityProcessor< ...(this.entitiesService.mapMediaCollection(await Media.for(nursery).findAll(), Nursery.MEDIA) as NurseryMedia) }; - return { id: nursery.uuid, dto: new NurseryFullDto(nursery, props) }; + return { id: nursery.uuid, dto: new DisturbanceFullDto(nursery, props) }; } - async getLightDto(nursery: Nursery) { + async getLightDto(nursery: Disturbance) { const nurseryId = nursery.id; const seedlingsGrownCount = await this.getSeedlingsGrownCount(nurseryId); - return { id: nursery.uuid, dto: new NurseryLightDto(nursery, { seedlingsGrownCount }) }; + return { id: nursery.uuid, dto: new DisturbanceLightDto(nursery, { seedlingsGrownCount }) }; } protected async getTotalOverdueReports(nurseryId: number) { @@ -144,16 +146,16 @@ export class DisturbancesProcessor extends EntityProcessor< ); } - async delete(nursery: Nursery) { + async delete(disturbance: Disturbance) { const permissions = await this.entitiesService.getPermissions(); - const managesOwn = permissions.includes("manage-own") && !permissions.includes(`framework-${nursery.frameworkKey}`); + const managesOwn = permissions.includes("manage-own"); // TODO validate this condition if (managesOwn) { - const reportCount = await NurseryReport.count({ where: { nurseryId: nursery.id } }); + const reportCount = await NurseryReport.count({ where: { nurseryId: disturbance.id } }); if (reportCount > 0) { throw new NotAcceptableException("You can only delete nurseries that do not have reports"); } } - await super.delete(nursery); + await super.delete(disturbance); } } diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index c2fe5b7d..dd378c34 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -4,6 +4,7 @@ export * from "./application.entity"; export * from "./delayed-job.entity"; export * from "./demographic.entity"; export * from "./demographic-entry.entity"; +export * from "./disturbance.entity"; export * from "./form-submission.entity"; export * from "./form-question.entity"; export * from "./framework.entity"; From 76e76f0563625c757ee578157f89698c669ae4dd Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 12:13:43 -0400 Subject: [PATCH 65/98] [TM-1922] rollback to original test --- .../project-report.processor.spec.ts | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts index 77ba7913..ca07465a 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts @@ -17,17 +17,11 @@ import { ProjectReportProcessor } from "./project-report.processor"; import { DateTime } from "luxon"; import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; -import { DocumentBuilder } from "@terramatch-microservices/common/util"; -import { AssociationProcessor } from "./association-processor"; -import { UuidModel } from "@terramatch-microservices/database/types/util"; -import { AssociationDto } from "../dto/association.dto"; describe("ProjectReportProcessor", () => { let processor: ProjectReportProcessor; let policyService: DeepMocked; let userId: number; - let entitiesService: DeepMocked; - let mockAssociationProcessor: Partial>>; beforeAll(async () => { userId = (await UserFactory.create()).id; @@ -46,20 +40,6 @@ describe("ProjectReportProcessor", () => { }).compile(); processor = module.get(EntitiesService).createEntityProcessor("projectReports") as ProjectReportProcessor; - - mockAssociationProcessor = { - addIncludedDtos: jest.fn().mockResolvedValue(undefined) - }; - - entitiesService = createMock({ - createAssociationProcessor: jest.fn().mockReturnValue(mockAssociationProcessor) - }); - - const moduleRef = await Test.createTestingModule({ - providers: [ProjectReportProcessor, { provide: EntitiesService, useValue: entitiesService }] - }).compile(); - - processor = moduleRef.get(ProjectReportProcessor); }); describe("findMany", () => { @@ -452,21 +432,4 @@ describe("ProjectReportProcessor", () => { }); }); }); - - describe("processSideload", () => { - it("should use addIncludedDtos for supported associations", async () => { - const document = new DocumentBuilder("projectReports"); - const projectReport = new ProjectReport(); - projectReport.uuid = "test-uuid"; - - await processor.processSideload(document, projectReport, "demographics"); - - expect(entitiesService.createAssociationProcessor).toHaveBeenCalledWith( - "projectReports", - "test-uuid", - "demographics" - ); - expect(mockAssociationProcessor.addIncludedDtos).toHaveBeenCalledWith(document); - }); - }); }); From 97fa61484c27988d9031caba921b2c4e5500df66 Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 12:26:00 -0400 Subject: [PATCH 66/98] [TM-1922] add sideload demographic test --- .../project-report.processor.spec.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts index ca07465a..a1cf6cec 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.spec.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.spec.ts @@ -10,13 +10,16 @@ import { ProjectFactory, ProjectReportFactory, ProjectUserFactory, - UserFactory + UserFactory, + DemographicFactory } from "@terramatch-microservices/database/factories"; import { BadRequestException } from "@nestjs/common/exceptions/bad-request.exception"; import { ProjectReportProcessor } from "./project-report.processor"; import { DateTime } from "luxon"; import { PolicyService } from "@terramatch-microservices/common"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; +import { buildJsonApi } from "@terramatch-microservices/common/util"; +import { ProjectReportLightDto } from "../dto/project-report.dto"; describe("ProjectReportProcessor", () => { let processor: ProjectReportProcessor; @@ -432,4 +435,22 @@ describe("ProjectReportProcessor", () => { }); }); }); + + describe("processSideload", () => { + it("should include sideloaded demographics", async () => { + const projectReport = await ProjectReportFactory.create(); + await DemographicFactory.forProjectReportWorkday.create({ demographicalId: projectReport.id }); + await DemographicFactory.forProjectReportJobs.create({ demographicalId: projectReport.id }); + + policyService.getPermissions.mockResolvedValue(["projects-read"]); + const document = buildJsonApi(ProjectReportLightDto); + await processor.addIndex(document, { + sideloads: [{ entity: "demographics", pageSize: 5 }] + }); + + const result = document.serialize(); + expect(result.included?.length).toBe(2); + expect(result.included.filter(({ type }) => type === "demographics").length).toBe(2); + }); + }); }); From f47cd2b74edf8de4211a22a597cf7608b196418b Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 13:47:32 -0400 Subject: [PATCH 67/98] [TM-1922] unify addDtos --- .../entity-associations.controller.spec.ts | 1 - .../processors/association-processor.ts | 26 +++++-------------- .../processors/project-report.processor.ts | 2 +- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/apps/entity-service/src/entities/entity-associations.controller.spec.ts b/apps/entity-service/src/entities/entity-associations.controller.spec.ts index 0f8c02e1..5db280af 100644 --- a/apps/entity-service/src/entities/entity-associations.controller.spec.ts +++ b/apps/entity-service/src/entities/entity-associations.controller.spec.ts @@ -14,7 +14,6 @@ class StubProcessor extends AssociationProcessor { DTO = DemographicDto; addDtos = jest.fn(() => Promise.resolve()); - addIncludedDtos = jest.fn(() => Promise.resolve()); getAssociations = jest.fn(() => Promise.resolve([] as Demographic[])); } diff --git a/apps/entity-service/src/entities/processors/association-processor.ts b/apps/entity-service/src/entities/processors/association-processor.ts index 38cf91e4..2dd79761 100644 --- a/apps/entity-service/src/entities/processors/association-processor.ts +++ b/apps/entity-service/src/entities/processors/association-processor.ts @@ -53,32 +53,18 @@ export abstract class AssociationProcessor { + async addDtos(document: DocumentBuilder, asIncluded = false): Promise { const associations = await this.getAssociations(await this.getBaseEntity()); const additionalProps = { entityType: this.entityType, entityUuid: this.entityUuid }; const indexIds: string[] = []; for (const association of associations) { indexIds.push(association.uuid); - document.addData(association.uuid, new this.DTO(association, additionalProps)); - } - - const resource = getDtoType(this.DTO); - document.addIndexData({ - resource, - requestPath: `/entities/v3/${this.entityType}/${this.entityUuid}/${resource}`, - ids: indexIds - }); - } - - async addIncludedDtos(document: DocumentBuilder): Promise { - const associations = await this.getAssociations(await this.getBaseEntity()); - - const additionalProps = { entityType: this.entityType, entityUuid: this.entityUuid }; - const indexIds: string[] = []; - for (const association of associations) { - indexIds.push(association.uuid); - document.addIncluded(association.uuid, new this.DTO(association, additionalProps)); + if (asIncluded) { + document.addIncluded(association.uuid, new this.DTO(association, additionalProps)); + } else { + document.addData(association.uuid, new this.DTO(association, additionalProps)); + } } const resource = getDtoType(this.DTO); diff --git a/apps/entity-service/src/entities/processors/project-report.processor.ts b/apps/entity-service/src/entities/processors/project-report.processor.ts index c6a7b226..420499b1 100644 --- a/apps/entity-service/src/entities/processors/project-report.processor.ts +++ b/apps/entity-service/src/entities/processors/project-report.processor.ts @@ -139,7 +139,7 @@ export class ProjectReportProcessor extends ReportProcessor< model.uuid, entity as ProcessableAssociation ); - await processor.addIncludedDtos(document); + await processor.addDtos(document, true); } else { throw new BadRequestException(`Project reports only support sideloading: ${SUPPORTED_ASSOCIATIONS.join(", ")}`); } From 1abc04d82687482cbbe8b544ecea43cf4806a95d Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Thu, 24 Apr 2025 11:56:36 -0600 Subject: [PATCH 68/98] [TM-1937] Fix logic for status check --- libs/database/src/lib/constants/status.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts index 5df96b11..5c9bb5c2 100644 --- a/libs/database/src/lib/constants/status.ts +++ b/libs/database/src/lib/constants/status.ts @@ -62,7 +62,7 @@ export const ReportStatusStates: States = { transitionValidForModel: (from: ReportStatus, to: ReportStatus, report: ReportModel) => { if ((from === DUE && to === AWAITING_APPROVAL) || (from === AWAITING_APPROVAL && to === STARTED)) { // these two transitions are only allowed for site / nursery reports when the nothingToReport flag is true; - return !(report instanceof ProjectReport) && (report.nothingToReport || false); + return !(report instanceof ProjectReport) && report.nothingToReport === true; } return true; From ee3a46d681dabd4941522de11b514e4a323eb09e Mon Sep 17 00:00:00 2001 From: JORGE Date: Thu, 24 Apr 2025 16:01:09 -0400 Subject: [PATCH 69/98] [TM-1922] add attributes to light site --- apps/entity-service/src/entities/dto/site.dto.ts | 8 +++++++- .../src/entities/processors/site.processor.ts | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/entity-service/src/entities/dto/site.dto.ts b/apps/entity-service/src/entities/dto/site.dto.ts index 8645ac65..978155fa 100644 --- a/apps/entity-service/src/entities/dto/site.dto.ts +++ b/apps/entity-service/src/entities/dto/site.dto.ts @@ -56,6 +56,12 @@ export class SiteLightDto extends EntityDto { @ApiProperty() treesPlantedCount: number; + @ApiProperty({ nullable: true }) + hectaresToRestoreGoal: number; + + @ApiProperty() + totalHectaresRestoredSum: number; + @ApiProperty() createdAt: Date; @@ -63,7 +69,7 @@ export class SiteLightDto extends EntityDto { updatedAt: Date; } -export type AdditionalSiteLightProps = Pick; +export type AdditionalSiteLightProps = Pick; export type AdditionalSiteFullProps = AdditionalSiteLightProps & AdditionalProps>; export type SiteMedia = Pick; diff --git a/apps/entity-service/src/entities/processors/site.processor.ts b/apps/entity-service/src/entities/processors/site.processor.ts index b627e252..b1e11db3 100644 --- a/apps/entity-service/src/entities/processors/site.processor.ts +++ b/apps/entity-service/src/entities/processors/site.processor.ts @@ -168,7 +168,9 @@ export class SiteProcessor extends EntityProcessor Date: Fri, 25 Apr 2025 07:50:19 -0600 Subject: [PATCH 70/98] [TM-1959] feat: update Disturbance DTOs and processor to enhance support for Disturbance entity --- .../src/entities/dto/disturbance.dto.ts | 99 ++----------------- .../processors/disturbances.processor.ts | 51 ++-------- .../src/lib/policies/disturbance.policy.ts | 48 +++++++++ .../common/src/lib/policies/policy.service.ts | 5 +- 4 files changed, 65 insertions(+), 138 deletions(-) create mode 100644 libs/common/src/lib/policies/disturbance.policy.ts diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index fe591a28..0d2be2bf 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -3,27 +3,20 @@ import { AdditionalProps, EntityDto } from "./entity.dto"; import { Nursery } from "@terramatch-microservices/database/entities"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; -import { - ENTITY_STATUSES, - EntityStatus, - UPDATE_REQUEST_STATUSES, - UpdateRequestStatus -} from "@terramatch-microservices/database/constants/status"; -import { MediaDto } from "./media.dto"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -@JsonApiDto({ type: "nurseries" }) +@JsonApiDto({ type: "disturbances" }) export class DisturbanceLightDto extends EntityDto { - constructor(nursery?: Disturbance, props?: AdditionalNurseryLightProps) { + constructor(disturbance?: Disturbance, props?: AdditionalNurseryLightProps) { super(); - if (nursery != null) { + if (disturbance != null) { this.populate(DisturbanceLightDto, { - ...pickApiProperties(nursery, DisturbanceLightDto), + ...pickApiProperties(disturbance, DisturbanceLightDto), lightResource: true, ...props, // these two are untyped and marked optional in the base model. - createdAt: nursery.createdAt as Date, - updatedAt: nursery.updatedAt as Date + createdAt: disturbance.createdAt as Date, + updatedAt: disturbance.updatedAt as Date }); } } @@ -31,44 +24,6 @@ export class DisturbanceLightDto extends EntityDto { @ApiProperty({ nullable: true }) name: string | null; - @ApiProperty({ nullable: true, description: "Framework key for this nursery" }) - frameworkKey: string | null; - - @ApiProperty({ - nullable: true, - description: "Entity status for this nursery", - enum: ENTITY_STATUSES - }) - status: EntityStatus | null; - - @ApiProperty({ - nullable: true, - description: "Update request status for this nursery", - enum: UPDATE_REQUEST_STATUSES - }) - updateRequestStatus: UpdateRequestStatus | null; - - @ApiProperty({ - nullable: true, - description: "The associated project name" - }) - projectName: string | null; - - @ApiProperty({ - nullable: true, - description: "The associated project organisation name" - }) - organisationName: string | null; - - @ApiProperty({ nullable: true }) - startDate: Date | null; - - @ApiProperty({ nullable: true }) - endDate: Date | null; - - @ApiProperty({ nullable: true }) - seedlingsGrownCount: number | null; - @ApiProperty() createdAt: Date; @@ -76,10 +31,9 @@ export class DisturbanceLightDto extends EntityDto { updatedAt: Date; } -export type AdditionalNurseryLightProps = Pick; +export type AdditionalNurseryLightProps = DisturbanceLightDto; export type AdditionalNurseryFullProps = AdditionalNurseryLightProps & AdditionalProps>; -export type NurseryMedia = Pick; export class DisturbanceFullDto extends DisturbanceLightDto { constructor(nursery: Disturbance, props: AdditionalNurseryFullProps) { @@ -94,45 +48,6 @@ export class DisturbanceFullDto extends DisturbanceLightDto { }); } - @ApiProperty({ nullable: true }) - feedback: string | null; - - @ApiProperty({ nullable: true }) - feedbackFields: string[] | null; - - @ApiProperty({ nullable: true }) - type: string | null; - - @ApiProperty({ nullable: true }) - seedlingGrown: number | null; - - @ApiProperty({ nullable: true }) - plantingContribution: string | null; - - @ApiProperty({ nullable: true }) - oldModel: string | null; - - @ApiProperty({ nullable: true }) - nurseryReportsTotal: number | null; - - @ApiProperty({ nullable: true }) - overdueNurseryReportsTotal: number | null; - - @ApiProperty({ nullable: true }) - organisationName: string | null; - - @ApiProperty({ nullable: true }) - projectName: string | null; - @ApiProperty({ nullable: true }) projectUuid: string | null; - - @ApiProperty({ type: () => MediaDto, isArray: true }) - file: MediaDto[]; - - @ApiProperty({ type: () => MediaDto, isArray: true }) - otherAdditionalDocuments: MediaDto[]; - - @ApiProperty({ type: () => MediaDto, isArray: true }) - photos: MediaDto[]; } diff --git a/apps/entity-service/src/entities/processors/disturbances.processor.ts b/apps/entity-service/src/entities/processors/disturbances.processor.ts index 75e860e0..d5610a6c 100644 --- a/apps/entity-service/src/entities/processors/disturbances.processor.ts +++ b/apps/entity-service/src/entities/processors/disturbances.processor.ts @@ -93,57 +93,18 @@ export class DisturbancesProcessor extends EntityProcessor< }); } - if (query.projectUuid != null) { - const project = await Project.findOne({ where: { uuid: query.projectUuid }, attributes: ["id"] }); - if (project == null) { - throw new BadRequestException(`Project with uuid ${query.projectUuid} not found`); - } - builder.where({ projectId: project.id }); - } - return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; } - async getFullDto(nursery: Disturbance) { - const nurseryId = nursery.id; - - const nurseryReportsTotal = await NurseryReport.nurseries([nurseryId]).count(); - const seedlingsGrownCount = await this.getSeedlingsGrownCount(nurseryId); - const overdueNurseryReportsTotal = await this.getTotalOverdueReports(nurseryId); - const props: AdditionalNurseryFullProps = { - seedlingsGrownCount, - nurseryReportsTotal, - overdueNurseryReportsTotal, - - ...(this.entitiesService.mapMediaCollection(await Media.for(nursery).findAll(), Nursery.MEDIA) as NurseryMedia) - }; - - return { id: nursery.uuid, dto: new DisturbanceFullDto(nursery, props) }; - } - - async getLightDto(nursery: Disturbance) { - const nurseryId = nursery.id; - - const seedlingsGrownCount = await this.getSeedlingsGrownCount(nurseryId); - return { id: nursery.uuid, dto: new DisturbanceLightDto(nursery, { seedlingsGrownCount }) }; - } + async getFullDto(disturbance: Disturbance) { + const nurseryId = disturbance.id; - protected async getTotalOverdueReports(nurseryId: number) { - const countOpts = { where: { dueAt: { [Op.lt]: new Date() } } }; - return await NurseryReport.incomplete().nurseries([nurseryId]).count(countOpts); + return { id: disturbance.uuid, dto: new DisturbanceFullDto(disturbance, null) }; } - private async getSeedlingsGrownCount(nurseryId: number) { - return ( - ( - await NurseryReport.nurseries([nurseryId]) - .approved() - .findAll({ - raw: true, - attributes: [[fn("SUM", col("seedlings_young_trees")), "seedlingsYoungTrees"]] - }) - )[0].seedlingsYoungTrees ?? 0 - ); + async getLightDto(disturbance: Disturbance) { + const nurseryId = disturbance.id; + return { id: disturbance.uuid, dto: new DisturbanceLightDto(disturbance, null) }; } async delete(disturbance: Disturbance) { diff --git a/libs/common/src/lib/policies/disturbance.policy.ts b/libs/common/src/lib/policies/disturbance.policy.ts new file mode 100644 index 00000000..b5e8baf1 --- /dev/null +++ b/libs/common/src/lib/policies/disturbance.policy.ts @@ -0,0 +1,48 @@ +import { Disturbance, Project, Site, User } from "@terramatch-microservices/database/entities"; +import { UserPermissionsPolicy } from "./user-permissions.policy"; + +export class DisturbancePolicy extends UserPermissionsPolicy { + async addRules() { + if (this.permissions.includes("view-dashboard") || this.permissions.includes("reports-manage")) { + this.builder.can("read", Disturbance); + return; + } + + if (this.permissions.includes("manage-own")) { + const user = await this.getUser(); + if (user != null) { + const projectIds = [ + ...(user.organisationId === null + ? [] + : await Project.findAll({ where: { organisationId: user.organisationId }, attributes: ["id"] }) + ).map(({ id }) => id), + ...user.projects.map(({ id }) => id) + ]; + if (projectIds.length > 0) { + this.builder.can(["read", "delete", "update"], Site, { projectId: { $in: projectIds } }); + } + } + } + + if (this.permissions.includes("projects-manage")) { + const user = await this.getUser(); + if (user != null) { + const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); + if (projectIds.length > 0) { + this.builder.can(["read", "delete", "update", "approve"], Site, { projectId: { $in: projectIds } }); + } + } + } + } + + protected _user?: User | null; + protected async getUser() { + if (this._user != null) return this._user; + + return (this._user = await User.findOne({ + where: { id: this.userId }, + attributes: ["organisationId"], + include: [{ association: "projects", attributes: ["id"] }] + })); + } +} diff --git a/libs/common/src/lib/policies/policy.service.ts b/libs/common/src/lib/policies/policy.service.ts index c823ee97..7a50e9d4 100644 --- a/libs/common/src/lib/policies/policy.service.ts +++ b/libs/common/src/lib/policies/policy.service.ts @@ -2,6 +2,7 @@ import { Injectable, Scope, UnauthorizedException } from "@nestjs/common"; import { RequestContext } from "nestjs-request-context"; import { UserPolicy } from "./user.policy"; import { + Disturbance, Nursery, NurseryReport, Permission, @@ -24,6 +25,7 @@ import { SitePolicy } from "./site.policy"; import { NurseryReportPolicy } from "./nursery-report.policy"; import { NurseryPolicy } from "./nursery.policy"; import { TMLogger } from "../util/tm-logger"; +import { DisturbancePolicy } from "@terramatch-microservices/common/policies/disturbance.policy"; type EntityClass = { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -43,7 +45,8 @@ const POLICIES: [EntityClass, PolicyClass][] = [ [Site, SitePolicy], [SitePolygon, SitePolygonPolicy], [SiteReport, SiteReportPolicy], - [User, UserPolicy] + [User, UserPolicy], + [Disturbance, DisturbancePolicy] ]; /** From 74546adfb056c18a4b7ed21251c4875c76c1ccae Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Fri, 25 Apr 2025 11:50:11 -0400 Subject: [PATCH 71/98] [TM-1922] add pctSurvivalToDate to the light resource DTO for project reports --- apps/entity-service/src/entities/dto/project-report.dto.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/entity-service/src/entities/dto/project-report.dto.ts b/apps/entity-service/src/entities/dto/project-report.dto.ts index 9c025da9..9043e970 100644 --- a/apps/entity-service/src/entities/dto/project-report.dto.ts +++ b/apps/entity-service/src/entities/dto/project-report.dto.ts @@ -71,6 +71,9 @@ export class ProjectReportLightDto extends EntityDto { @ApiProperty() updatedAt: Date; + + @ApiProperty({ nullable: true }) + pctSurvivalToDate: number | null; } export type AdditionalProjectReportFullProps = AdditionalProps< @@ -188,9 +191,6 @@ export class ProjectReportFullDto extends ProjectReportLightDto { @ApiProperty({ nullable: true }) significantChange: string | null; - @ApiProperty({ nullable: true }) - pctSurvivalToDate: number | null; - @ApiProperty({ nullable: true }) survivalCalculation: string | null; From 650558864d725f5e2e4b47e9a0412c502f2a105f Mon Sep 17 00:00:00 2001 From: JORGE Date: Fri, 25 Apr 2025 12:10:28 -0400 Subject: [PATCH 72/98] [TM-1922] improve query sum --- .../entities/processors/entity-processor.ts | 21 +++- .../src/entities/processors/site.processor.ts | 103 ++++++++++++++++-- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index ff01f997..88d66ea4 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -73,6 +73,22 @@ export abstract class EntityProcessor< abstract getFullDto(model: ModelType): Promise>; abstract getLightDto(model: ModelType): Promise>; + async getFullDtos(models: ModelType[]): Promise[]> { + const results: DtoResult[] = []; + for (const model of models) { + results.push(await this.getFullDto(model)); + } + return results; + } + + async getLightDtos(models: ModelType[]): Promise[]> { + const results: DtoResult[] = []; + for (const model of models) { + results.push(await this.getLightDto(model)); + } + return results; + } + /* eslint-disable @typescript-eslint/no-unused-vars */ async processSideload( document: DocumentBuilder, @@ -92,8 +108,9 @@ export abstract class EntityProcessor< if (models.length !== 0) { await this.entitiesService.authorize("read", models); - for (const model of models) { - const { id, dto } = await this.getLightDto(model); + const dtoResults = await this.getLightDtos(models); + + for (const { id, dto } of dtoResults) { indexIds.push(id); if (sideloaded) document.addIncluded(id, dto); else document.addData(id, dto); diff --git a/apps/entity-service/src/entities/processors/site.processor.ts b/apps/entity-service/src/entities/processors/site.processor.ts index b1e11db3..f2a51e2b 100644 --- a/apps/entity-service/src/entities/processors/site.processor.ts +++ b/apps/entity-service/src/entities/processors/site.processor.ts @@ -15,7 +15,7 @@ import { AdditionalSiteFullProps, SiteFullDto, SiteLightDto, SiteMedia } from ". import { BadRequestException, NotAcceptableException } from "@nestjs/common"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; import { Includeable, Op } from "sequelize"; -import { sumBy } from "lodash"; +import { sumBy, groupBy } from "lodash"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { SiteUpdateAttributes } from "../dto/entity-update.dto"; import { @@ -130,6 +130,72 @@ export class SiteProcessor extends EntityProcessor> { + if (siteUuids.length === 0) return {}; + + const polygons = await SitePolygon.findAll({ + where: { + siteUuid: { [Op.in]: siteUuids }, + status: "approved", + isActive: true, + deletedAt: null + }, + attributes: ["siteUuid", "calcArea"] + }); + + const hectaresMap: Record = {}; + const polygonsBySite = groupBy(polygons, "siteUuid"); + + for (const siteUuid of siteUuids) { + const sitesPolygons = polygonsBySite[siteUuid] || []; + hectaresMap[siteUuid] = sumBy(sitesPolygons, polygon => Number(polygon.calcArea) || 0); + } + + return hectaresMap; + } + + private async getTreesPlantedCount(sites: Site[]): Promise> { + if (sites.length === 0) return {}; + + const siteIds = sites.map(site => site.id); + const approvedReports = await SiteReport.findAll({ + where: { + siteId: { [Op.in]: siteIds }, + status: "approved" + }, + attributes: ["id", "siteId"] + }); + + if (approvedReports.length === 0) { + return sites.reduce((acc, site) => ({ ...acc, [site.uuid]: 0 }), {}); + } + + const reportIds = approvedReports.map(r => r.id); + const reportBySiteId = groupBy(approvedReports, "siteId"); + + const treesPlanted = await TreeSpecies.findAll({ + where: { + speciesableId: { [Op.in]: reportIds }, + speciesableType: SiteReport.LARAVEL_TYPE, + collection: "tree-planted", + hidden: false + }, + attributes: ["speciesableId", "amount"] + }); + + const result: Record = {}; + + for (const site of sites) { + const siteId = site.id; + const siteReports = reportBySiteId[siteId] || []; + const siteReportIds = siteReports.map(r => r.id); + const treesForSite = treesPlanted.filter(t => siteReportIds.includes(t.speciesableId)); + result[site.uuid] = sumBy(treesForSite, "amount") || 0; + } + + return result; + } + async getFullDto(site: Site) { const siteId = site.id; @@ -144,8 +210,11 @@ export class SiteProcessor extends EntityProcessor { + if (sites.length === 0) return []; + + const siteUuids = sites.map(site => site.uuid); + + const [hectaresData, treesPlantedData] = await Promise.all([ + this.getHectaresRestoredSum(siteUuids), + this.getTreesPlantedCount(sites) + ]); + + return sites.map(site => ({ + id: site.uuid, + dto: new SiteLightDto(site, { + treesPlantedCount: treesPlantedData[site.uuid] || 0, + totalHectaresRestoredSum: hectaresData[site.uuid] || 0 + }) + })); + } + protected async getWorkdayCount(siteId: number, useDemographicsCutoff = false) { const dueAfter = useDemographicsCutoff ? Demographic.DEMOGRAPHIC_COUNT_CUTOFF : undefined; From ef767a798acae3af4b9cfb114c7005407a4d6ea9 Mon Sep 17 00:00:00 2001 From: JORGE Date: Fri, 25 Apr 2025 12:37:06 -0400 Subject: [PATCH 73/98] [TM-1922] replace nullish operator --- .../src/entities/processors/site.processor.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/entity-service/src/entities/processors/site.processor.ts b/apps/entity-service/src/entities/processors/site.processor.ts index f2a51e2b..54a9345d 100644 --- a/apps/entity-service/src/entities/processors/site.processor.ts +++ b/apps/entity-service/src/entities/processors/site.processor.ts @@ -147,8 +147,8 @@ export class SiteProcessor extends EntityProcessor Number(polygon.calcArea) || 0); + const sitesPolygons = polygonsBySite[siteUuid] ?? []; + hectaresMap[siteUuid] = sumBy(sitesPolygons, polygon => Number(polygon.calcArea) ?? 0); } return hectaresMap; @@ -187,10 +187,10 @@ export class SiteProcessor extends EntityProcessor r.id); const treesForSite = treesPlanted.filter(t => siteReportIds.includes(t.speciesableId)); - result[site.uuid] = sumBy(treesForSite, "amount") || 0; + result[site.uuid] = sumBy(treesForSite, "amount") ?? 0; } return result; @@ -211,7 +211,7 @@ export class SiteProcessor extends EntityProcessor ({ id: site.uuid, dto: new SiteLightDto(site, { - treesPlantedCount: treesPlantedData[site.uuid] || 0, - totalHectaresRestoredSum: hectaresData[site.uuid] || 0 + treesPlantedCount: treesPlantedData[site.uuid] ?? 0, + totalHectaresRestoredSum: hectaresData[site.uuid] ?? 0 }) })); } From db38aecf4f59fababcf344239b399f0d661f67f3 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 10:50:35 -0600 Subject: [PATCH 74/98] [TM-1959] feat: enhance Disturbance entity and DTO with additional properties for improved data handling --- .../src/entities/dto/disturbance.dto.ts | 16 +++++- .../src/lib/entities/disturbance.entity.ts | 49 +++++++++++++------ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index 0d2be2bf..3cd04c8a 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -4,6 +4,8 @@ import { Nursery } from "@terramatch-microservices/database/entities"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { AllowNull, Column } from "sequelize-typescript"; +import { STRING, TEXT } from "sequelize"; @JsonApiDto({ type: "disturbances" }) export class DisturbanceLightDto extends EntityDto { @@ -49,5 +51,17 @@ export class DisturbanceFullDto extends DisturbanceLightDto { } @ApiProperty({ nullable: true }) - projectUuid: string | null; + collection: string | null; + + @ApiProperty({ nullable: true }) + type: string | null; + + @ApiProperty({ nullable: true }) + intensity: string | null; + + @ApiProperty({ nullable: true }) + extent: string | null; + + @ApiProperty({ nullable: true }) + description: string | null; } diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts index 23a495d0..c4c0c523 100644 --- a/libs/database/src/lib/entities/disturbance.entity.ts +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -1,15 +1,5 @@ -import { - AllowNull, - AutoIncrement, - Column, - HasMany, - Index, - Model, - PrimaryKey, - Scopes, - Table -} from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, Op, STRING, TEXT, UUID, UUIDV4 } from "sequelize"; +import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_disturbances", underscored: true, paranoid: true }) export class Disturbance extends Model { @@ -24,9 +14,40 @@ export class Disturbance extends Model { @Column({ type: UUID, defaultValue: UUIDV4 }) uuid: string; - @Column(BIGINT.UNSIGNED) - disturbanceableType: number; + @Column(STRING) + disturbanceableType: string; @Column(BIGINT.UNSIGNED) disturbanceableId: number; + + @AllowNull + @Column(STRING) + collection: string | null; + + @AllowNull + @Column(STRING) + type: string | null; + + @AllowNull + @Column(STRING) + intensity: string | null; + + @AllowNull + @Column(STRING) + extent: string | null; + + @AllowNull + @Column(TEXT) + description: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + oldId: number; + + @AllowNull + @Column(STRING) + oldModel: string | null; + + @Column(TINYINT) + hidden: string | null; } From 723743012a0d1ce8929a9764c7a3e4a2113905c5 Mon Sep 17 00:00:00 2001 From: LimberHope Date: Fri, 25 Apr 2025 13:53:54 -0400 Subject: [PATCH 75/98] [TM-1942] add new fields to fields in projects and site reports --- .../src/entities/dto/project-report.dto.ts | 48 +++++++++++++++++++ .../src/entities/dto/site-report.dto.ts | 12 +++++ .../src/lib/entities/project-report.entity.ts | 21 +++++++- .../src/lib/entities/site-report.entity.ts | 6 ++- 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/apps/entity-service/src/entities/dto/project-report.dto.ts b/apps/entity-service/src/entities/dto/project-report.dto.ts index 9043e970..2000236e 100644 --- a/apps/entity-service/src/entities/dto/project-report.dto.ts +++ b/apps/entity-service/src/entities/dto/project-report.dto.ts @@ -304,4 +304,52 @@ export class ProjectReportFullDto extends ProjectReportLightDto { @ApiProperty({ type: () => MediaDto, isArray: true }) photos: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + baselineReportUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + localGovernanceOrderLetterUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + eventsMeetingsPhotos: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + localGovernanceProofOfPartnershipUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + topThreeSuccessesUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + directJobsUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + convergenceJobsUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + convergenceSchemesUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + livelihoodActivitiesUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + directLivelihoodImpactsUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + certifiedDatabaseUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + physicalAssetsPhotos: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + indirectCommunityPartnersUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + trainingCapacityBuildingUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + trainingCapacityBuildingPhotos: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + financialReportUpload: MediaDto[]; } diff --git a/apps/entity-service/src/entities/dto/site-report.dto.ts b/apps/entity-service/src/entities/dto/site-report.dto.ts index c83a075e..aa7c208b 100644 --- a/apps/entity-service/src/entities/dto/site-report.dto.ts +++ b/apps/entity-service/src/entities/dto/site-report.dto.ts @@ -292,4 +292,16 @@ export class SiteReportFullDto extends SiteReportLightDto { @ApiProperty({ type: () => MediaDto, isArray: true }) documentFiles: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + treePlantingUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + anrPhotos: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + soilWaterConservationUpload: MediaDto[]; + + @ApiProperty({ type: () => MediaDto, isArray: true }) + soilWaterConservationPhotos: MediaDto[]; } diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index 8ce02efb..772de6d9 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -51,7 +51,26 @@ export class ProjectReport extends Model { media: { dbCollection: "media", multiple: true }, file: { dbCollection: "file", multiple: true }, otherAdditionalDocuments: { dbCollection: "other_additional_documents", multiple: true }, - photos: { dbCollection: "photos", multiple: true } + photos: { dbCollection: "photos", multiple: true }, + baselineReportUpload: { dbCollection: "baseline_report_upload", multiple: true }, + localGovernanceOrderLetterUpload: { dbCollection: "local_governance_order_letter_upload", multiple: true }, + eventsMeetingsPhotos: { dbCollection: "events_meetings_photos", multiple: true }, + localGovernanceProofOfPartnershipUpload: { + dbCollection: "local_governance_proof_of_partnership_upload", + multiple: true + }, + topThreeSuccessesUpload: { dbCollection: "top_three_successes_upload", multiple: true }, + directJobsUpload: { dbCollection: "direct_jobs_upload", multiple: true }, + convergenceJobsUpload: { dbCollection: "convergence_jobs_upload", multiple: true }, + convergenceSchemesUpload: { dbCollection: "convergence_schemes_upload", multiple: true }, + livelihoodActivitiesUpload: { dbCollection: "livelihood_activities_upload", multiple: true }, + directLivelihoodImpactsUpload: { dbCollection: "direct_livelihood_impacts_upload", multiple: true }, + certifiedDatabaseUpload: { dbCollection: "certified_database_upload", multiple: true }, + physicalAssetsPhotos: { dbCollection: "physical_assets_photos", multiple: true }, + indirectCommunityPartnersUpload: { dbCollection: "indirect_community_partners_upload", multiple: true }, + trainingCapacityBuildingUpload: { dbCollection: "training_capacity_building_upload", multiple: true }, + trainingCapacityBuildingPhotos: { dbCollection: "training_capacity_building_photos", multiple: true }, + financialReportUpload: { dbCollection: "financial_report_upload", multiple: true } } as const; static incomplete() { diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index c9affb2a..969c9262 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -54,7 +54,11 @@ export class SiteReport extends Model { photos: { dbCollection: "photos", multiple: true }, treeSpecies: { dbCollection: "tree_species", multiple: true }, siteSubmission: { dbCollection: "site_submission", multiple: true }, - documentFiles: { dbCollection: "document_files", multiple: true } + documentFiles: { dbCollection: "document_files", multiple: true }, + treePlantingUpload: { dbCollection: "tree_planting_upload", multiple: true }, + anrPhotos: { dbCollection: "anr_photos", multiple: true }, + soilWaterConservationUpload: { dbCollection: "soil_water_conservation_upload", multiple: true }, + soilWaterConservationPhotos: { dbCollection: "soil_water_conservation_photos", multiple: true } } as const; static incomplete() { From 9155b915410d54f47d0e239f2237c877237101ea Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 12:12:02 -0600 Subject: [PATCH 76/98] [TM-1959] feat: extend Disturbance DTO and entity with new properties for enhanced data representation --- .../src/entities/dto/disturbance.dto.ts | 11 ++++++++++- .../src/entities/processors/disturbances.processor.ts | 3 +-- libs/database/src/lib/entities/disturbance.entity.ts | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index 3cd04c8a..0c8796d5 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -5,7 +5,7 @@ import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api import { ApiProperty } from "@nestjs/swagger"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; import { AllowNull, Column } from "sequelize-typescript"; -import { STRING, TEXT } from "sequelize"; +import { INTEGER, STRING, TEXT, TINYINT } from "sequelize"; @JsonApiDto({ type: "disturbances" }) export class DisturbanceLightDto extends EntityDto { @@ -64,4 +64,13 @@ export class DisturbanceFullDto extends DisturbanceLightDto { @ApiProperty({ nullable: true }) description: string | null; + + @ApiProperty({ nullable: true }) + oldId: number; + + @ApiProperty({ nullable: true }) + oldModel: string | null; + + @ApiProperty({ nullable: true }) + hidden: number | null; } diff --git a/apps/entity-service/src/entities/processors/disturbances.processor.ts b/apps/entity-service/src/entities/processors/disturbances.processor.ts index d5610a6c..5de525e5 100644 --- a/apps/entity-service/src/entities/processors/disturbances.processor.ts +++ b/apps/entity-service/src/entities/processors/disturbances.processor.ts @@ -1,5 +1,4 @@ -import { Media, Nursery, NurseryReport, Project, ProjectUser } from "@terramatch-microservices/database/entities"; -import { AdditionalNurseryFullProps, NurseryFullDto, NurseryLightDto, NurseryMedia } from "../dto/nursery.dto"; +import { NurseryReport, ProjectUser } from "@terramatch-microservices/database/entities"; import { EntityProcessor } from "./entity-processor"; import { EntityQueryDto } from "../dto/entity-query.dto"; import { col, fn, Includeable, Op } from "sequelize"; diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts index c4c0c523..853fd49f 100644 --- a/libs/database/src/lib/entities/disturbance.entity.ts +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -49,5 +49,5 @@ export class Disturbance extends Model { oldModel: string | null; @Column(TINYINT) - hidden: string | null; + hidden: number | null; } From 034bc09ee3980386bd62190bc548f683091d7007 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 12:37:49 -0600 Subject: [PATCH 77/98] [TM-1959] feat: refactor Disturbance processor and DTOs for improved entity handling and add Invasive entity support --- .../src/entities/dto/disturbance.dto.ts | 2 - .../src/entities/dto/invasive.dto.ts | 74 +++++++++++ .../src/entities/entities.service.ts | 8 +- ....processor.ts => disturbance.processor.ts} | 2 +- .../entities/processors/entity-processor.ts | 3 +- .../entities/processors/invasive.processor.ts | 122 ++++++++++++++++++ .../lib/email/entity-status-update.email.ts | 3 + .../entity-status-update.event-processor.ts | 2 + .../src/lib/policies/invasive.policy.ts | 48 +++++++ libs/database/src/lib/constants/entities.ts | 10 +- .../src/lib/entities/invasive.entity.ts | 45 +++++++ 11 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 apps/entity-service/src/entities/dto/invasive.dto.ts rename apps/entity-service/src/entities/processors/{disturbances.processor.ts => disturbance.processor.ts} (98%) create mode 100644 apps/entity-service/src/entities/processors/invasive.processor.ts create mode 100644 libs/common/src/lib/policies/invasive.policy.ts create mode 100644 libs/database/src/lib/entities/invasive.entity.ts diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index 0c8796d5..e42cbeb3 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -4,8 +4,6 @@ import { Nursery } from "@terramatch-microservices/database/entities"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { AllowNull, Column } from "sequelize-typescript"; -import { INTEGER, STRING, TEXT, TINYINT } from "sequelize"; @JsonApiDto({ type: "disturbances" }) export class DisturbanceLightDto extends EntityDto { diff --git a/apps/entity-service/src/entities/dto/invasive.dto.ts b/apps/entity-service/src/entities/dto/invasive.dto.ts new file mode 100644 index 00000000..01b345db --- /dev/null +++ b/apps/entity-service/src/entities/dto/invasive.dto.ts @@ -0,0 +1,74 @@ +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { AdditionalProps, EntityDto } from "./entity.dto"; +import { Nursery } from "@terramatch-microservices/database/entities"; +import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { ApiProperty } from "@nestjs/swagger"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; + +@JsonApiDto({ type: "invasives" }) +export class InvasiveLightDto extends EntityDto { + constructor(invasive?: Invasive, props?: AdditionalInvasiveLightProps) { + super(); + if (invasive != null) { + this.populate(InvasiveLightDto, { + ...pickApiProperties(invasive, InvasiveLightDto), + lightResource: true, + ...props, + // these two are untyped and marked optional in the base model. + createdAt: invasive.createdAt as Date, + updatedAt: invasive.updatedAt as Date + }); + } + } + + @ApiProperty({ nullable: true }) + name: string | null; + + @ApiProperty() + createdAt: Date; + + @ApiProperty() + updatedAt: Date; +} + +export type AdditionalInvasiveLightProps = InvasiveLightDto; +export type AdditionalNurseryFullProps = AdditionalInvasiveLightProps & + AdditionalProps>; + +export class InvasiveFullDto extends InvasiveLightDto { + constructor(invasive: Invasive, props: AdditionalNurseryFullProps) { + super(); + this.populate(InvasiveFullDto, { + ...pickApiProperties(invasive, InvasiveFullDto), + lightResource: false, + // these two are untyped and marked optional in the base model. + createdAt: invasive.createdAt as Date, + updatedAt: invasive.updatedAt as Date, + ...props + }); + } + + @ApiProperty({ nullable: true }) + collection: string | null; + + @ApiProperty({ nullable: true }) + type: string | null; + + @ApiProperty({ nullable: true }) + intensity: string | null; + + @ApiProperty({ nullable: true }) + extent: string | null; + + @ApiProperty({ nullable: true }) + description: string | null; + + @ApiProperty({ nullable: true }) + oldId: number; + + @ApiProperty({ nullable: true }) + oldModel: string | null; + + @ApiProperty({ nullable: true }) + hidden: number | null; +} diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index b0b1a3c4..99d79108 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -26,8 +26,10 @@ import { PolicyService } from "@terramatch-microservices/common"; import { EntityUpdateData } from "./dto/entity-update.dto"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { ITranslateParams } from "@transifex/native"; -import { DisturbancesProcessor } from "./processors/disturbances.processor"; +import { DisturbanceProcessor } from "./processors/disturbance.processor"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { InvasiveProcessor } from "./processors/invasive.processor"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; // The keys of this array must match the type in the resulting DTO. const ENTITY_PROCESSORS = { @@ -37,7 +39,8 @@ const ENTITY_PROCESSORS = { projectReports: ProjectReportProcessor, nurseryReports: NurseryReportProcessor, siteReports: SiteReportProcessor, - disturbances: DisturbancesProcessor + disturbances: DisturbanceProcessor, + invasives: InvasiveProcessor }; export type ProcessableEntity = keyof typeof ENTITY_PROCESSORS; @@ -99,6 +102,7 @@ export class EntitiesService { async isFrameworkAdmin(T: EntityModel) { if (T instanceof Disturbance) return; + if (T instanceof Invasive) return; const { frameworkKey } = T; return (await this.getPermissions()).includes(`framework-${frameworkKey}`); } diff --git a/apps/entity-service/src/entities/processors/disturbances.processor.ts b/apps/entity-service/src/entities/processors/disturbance.processor.ts similarity index 98% rename from apps/entity-service/src/entities/processors/disturbances.processor.ts rename to apps/entity-service/src/entities/processors/disturbance.processor.ts index 5de525e5..9f90de9e 100644 --- a/apps/entity-service/src/entities/processors/disturbances.processor.ts +++ b/apps/entity-service/src/entities/processors/disturbance.processor.ts @@ -8,7 +8,7 @@ import { EntityUpdateAttributes } from "../dto/entity-update.dto"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; import { DisturbanceFullDto, DisturbanceLightDto } from "../dto/disturbance.dto"; -export class DisturbancesProcessor extends EntityProcessor< +export class DisturbanceProcessor extends EntityProcessor< Disturbance, DisturbanceLightDto, DisturbanceFullDto, diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index 26a0aa71..14be606b 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -9,8 +9,8 @@ import { EntityModel } from "@terramatch-microservices/database/constants/entiti import { Action } from "@terramatch-microservices/database/entities/action.entity"; import { EntityUpdateData } from "../dto/entity-update.dto"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; -import { Project } from "@terramatch-microservices/database/entities"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export type Aggregate> = { func: string; @@ -120,6 +120,7 @@ export abstract class EntityProcessor< */ async update(model: ModelType, update: UpdateDto) { if (model instanceof Disturbance) return; + if (model instanceof Invasive) return; if (update.status != null) { if (this.APPROVAL_STATUSES.includes(update.status)) { diff --git a/apps/entity-service/src/entities/processors/invasive.processor.ts b/apps/entity-service/src/entities/processors/invasive.processor.ts new file mode 100644 index 00000000..586b2e3c --- /dev/null +++ b/apps/entity-service/src/entities/processors/invasive.processor.ts @@ -0,0 +1,122 @@ +import { NurseryReport, ProjectUser } from "@terramatch-microservices/database/entities"; +import { EntityProcessor } from "./entity-processor"; +import { EntityQueryDto } from "../dto/entity-query.dto"; +import { col, fn, Includeable, Op } from "sequelize"; +import { BadRequestException, NotAcceptableException } from "@nestjs/common"; +import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; +import { EntityUpdateAttributes } from "../dto/entity-update.dto"; +import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; +import { InvasiveFullDto, InvasiveLightDto } from "../dto/invasive.dto"; + +export class InvasiveProcessor extends EntityProcessor< + Invasive, + InvasiveLightDto, + InvasiveFullDto, + EntityUpdateAttributes +> { + readonly LIGHT_DTO = InvasiveLightDto; + readonly FULL_DTO = InvasiveFullDto; + + async findOne(uuid: string) { + return await Invasive.findOne({ + where: { uuid } + /*include: [ + { + association: "project", + attributes: ["uuid", "name"], + include: [{ association: "organisation", attributes: ["name"] }] + } + ]*/ + }); + } + + async findMany(query: EntityQueryDto) { + const projectAssociation: Includeable = { + association: "project", + attributes: ["uuid", "name"], + include: [{ association: "organisation", attributes: ["name"] }] + }; + + const builder = await this.entitiesService.buildQuery(Invasive, query, [projectAssociation]); + if (query.sort != null) { + if (["name", "startDate", "status", "updateRequestStatus", "createdAt"].includes(query.sort.field)) { + builder.order([query.sort.field, query.sort.direction ?? "ASC"]); + } else if (query.sort.field === "organisationName") { + builder.order(["project", "organisation", "name", query.sort.direction ?? "ASC"]); + } else if (query.sort.field === "projectName") { + builder.order(["project", "name", query.sort.direction ?? "ASC"]); + } else if (query.sort.field !== "id") { + throw new BadRequestException(`Invalid sort field: ${query.sort.field}`); + } + } + + const permissions = await this.entitiesService.getPermissions(); + const frameworkPermissions = permissions + ?.filter(name => name.startsWith("framework-")) + .map(name => name.substring("framework-".length) as FrameworkKey); + if (frameworkPermissions?.length > 0) { + builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); + } else if (permissions?.includes("manage-own")) { + builder.where({ projectId: { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } }); + } else if (permissions?.includes("projects-manage")) { + builder.where({ projectId: { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } }); + } + + const associationFieldMap = { + organisationUuid: "$project.organisation.uuid$", + country: "$project.country$", + projectUuid: "$project.uuid$" + }; + + for (const term of [ + "status", + "updateRequestStatus", + "frameworkKey", + "organisationUuid", + "country", + "projectUuid" + ]) { + if (query[term] != null) { + const field = associationFieldMap[term] ?? term; + builder.where({ [field]: query[term] }); + } + } + + if (query.search != null || query.searchFilter != null) { + builder.where({ + [Op.or]: [ + { name: { [Op.like]: `%${query.search ?? query.searchFilter}%` } }, + { "$project.name$": { [Op.like]: `%${query.search}%` } }, + { "$project.organisation.name$": { [Op.like]: `%${query.search}%` } } + ] + }); + } + + return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; + } + + async getFullDto(invasive: Invasive) { + const nurseryId = invasive.id; + + return { id: invasive.uuid, dto: new InvasiveFullDto(invasive, null) }; + } + + async getLightDto(invasive: Invasive) { + const nurseryId = invasive.id; + return { id: invasive.uuid, dto: new InvasiveLightDto(invasive, null) }; + } + + async delete(invasive: Invasive) { + const permissions = await this.entitiesService.getPermissions(); + const managesOwn = permissions.includes("manage-own"); // TODO validate this condition + if (managesOwn) { + const reportCount = await NurseryReport.count({ where: { nurseryId: invasive.id } }); + if (reportCount > 0) { + throw new NotAcceptableException("You can only delete nurseries that do not have reports"); + } + } + + await super.delete(invasive); + } +} diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index 2141981e..b1cd409d 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -23,6 +23,7 @@ import { TMLogger } from "../util/tm-logger"; import { InternalServerErrorException } from "@nestjs/common"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export class EntityStatusUpdateEmail extends EmailSender { private readonly logger = new TMLogger(EntityStatusUpdateEmail.name); @@ -39,6 +40,7 @@ export class EntityStatusUpdateEmail extends EmailSender { async send(emailService: EmailService) { const entity = await this.getEntity(); if (entity instanceof Disturbance) return; + if (entity instanceof Invasive) return; const status = entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION ? NEEDS_MORE_INFORMATION @@ -113,6 +115,7 @@ export class EntityStatusUpdateEmail extends EmailSender { private async getFeedback(entity: EntityModel) { if (entity instanceof Disturbance) return; + if (entity instanceof Invasive) return; if (![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.updateRequestStatus ?? "")) { return entity.feedback; } diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index 12d87b78..e6cf442e 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -16,6 +16,7 @@ import { get, isEmpty, map } from "lodash"; import { Op } from "sequelize"; import { STATUS_DISPLAY_STRINGS } from "@terramatch-microservices/database/constants/status"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; @@ -56,6 +57,7 @@ export class EntityStatusUpdate extends EventProcessor { const entity = this.model as EntityModel; if (entity instanceof Disturbance) return; + if (entity instanceof Invasive) return; await Action.for(entity).destroy({ where: { type: "notification" } }); diff --git a/libs/common/src/lib/policies/invasive.policy.ts b/libs/common/src/lib/policies/invasive.policy.ts new file mode 100644 index 00000000..3cc50bab --- /dev/null +++ b/libs/common/src/lib/policies/invasive.policy.ts @@ -0,0 +1,48 @@ +import { Disturbance, Project, Site, User } from "@terramatch-microservices/database/entities"; +import { UserPermissionsPolicy } from "./user-permissions.policy"; + +export class InvasivePolicy extends UserPermissionsPolicy { + async addRules() { + if (this.permissions.includes("view-dashboard") || this.permissions.includes("reports-manage")) { + this.builder.can("read", Disturbance); + return; + } + + if (this.permissions.includes("manage-own")) { + const user = await this.getUser(); + if (user != null) { + const projectIds = [ + ...(user.organisationId === null + ? [] + : await Project.findAll({ where: { organisationId: user.organisationId }, attributes: ["id"] }) + ).map(({ id }) => id), + ...user.projects.map(({ id }) => id) + ]; + if (projectIds.length > 0) { + this.builder.can(["read", "delete", "update"], Site, { projectId: { $in: projectIds } }); + } + } + } + + if (this.permissions.includes("projects-manage")) { + const user = await this.getUser(); + if (user != null) { + const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); + if (projectIds.length > 0) { + this.builder.can(["read", "delete", "update", "approve"], Site, { projectId: { $in: projectIds } }); + } + } + } + } + + protected _user?: User | null; + protected async getUser() { + if (this._user != null) return this._user; + + return (this._user = await User.findOne({ + where: { id: this.userId }, + attributes: ["organisationId"], + include: [{ association: "projects", attributes: ["id"] }] + })); + } +} diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index eb2d1c82..1c24a79f 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -3,6 +3,7 @@ import { ModelCtor } from "sequelize-typescript"; import { ModelStatic } from "sequelize"; import { kebabCase } from "lodash"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export const REPORT_TYPES = ["projectReports", "siteReports", "nurseryReports"] as const; export type ReportType = (typeof REPORT_TYPES)[number]; @@ -15,19 +16,19 @@ export const REPORT_MODELS: { [R in ReportType]: ReportClass } = { nurseryReports: NurseryReport }; -export const ENTITY_TYPES = ["projects", "sites", "nurseries", "disturbances", ...REPORT_TYPES] as const; +export const ENTITY_TYPES = ["projects", "sites", "nurseries", "disturbances", "invasives", ...REPORT_TYPES] as const; export type EntityType = (typeof ENTITY_TYPES)[number]; -export type EntityModel = ReportModel | Project | Site | Nursery | Disturbance; +export type EntityModel = ReportModel | Project | Site | Nursery | Disturbance | Invasive; export type EntityClass = ModelCtor & ModelStatic & { LARAVEL_TYPE: string }; export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { ...REPORT_MODELS, projects: Project, sites: Site, nurseries: Nursery, - disturbances: Disturbance + disturbances: Disturbance, // stratas: Strata::class, - //'invasives: Invasive::class, + invasives: Invasive }; export const isReport = (entity: EntityModel): entity is ReportModel => @@ -43,6 +44,7 @@ export const isReport = (entity: EntityModel): entity is ReportModel => export async function getProjectId(entity: EntityModel) { if (entity instanceof Project) return entity.id; if (entity instanceof Disturbance) return entity.id; + if (entity instanceof Invasive) return entity.id; if (entity instanceof Site || entity instanceof Nursery || entity instanceof ProjectReport) return entity.projectId; const parentClass: ModelCtor = entity instanceof SiteReport ? Site : Nursery; diff --git a/libs/database/src/lib/entities/invasive.entity.ts b/libs/database/src/lib/entities/invasive.entity.ts new file mode 100644 index 00000000..96618846 --- /dev/null +++ b/libs/database/src/lib/entities/invasive.entity.ts @@ -0,0 +1,45 @@ +import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; + +@Table({ tableName: "v2_invasives", underscored: true, paranoid: true }) +export class Invasive extends Model { + static readonly LARAVEL_TYPE = "App\\Models\\Invasive"; + + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column({ type: UUID, defaultValue: UUIDV4 }) + uuid: string; + + @Column(STRING) + invasiveableType: string; + + @Column(BIGINT.UNSIGNED) + invasiveableId: number; + + @AllowNull + @Column(STRING) + collection: string | null; + + @AllowNull + @Column(STRING) + type: string | null; + + @AllowNull + @Column(TEXT) + name: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + oldId: number; + + @AllowNull + @Column(STRING) + oldModel: string | null; + + @Column(TINYINT) + hidden: number | null; +} From c180fa2a8479c5f490979a2ecb95ac2d7b728190 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Apr 2025 13:07:31 -0700 Subject: [PATCH 78/98] [TM-1958] Build our initial Data API integration as its own library. --- .env.local.sample | 3 + .../src/airtable/airtable.module.ts | 8 +- libs/data-api/README.md | 3 + libs/data-api/project.json | 9 ++ libs/data-api/src/index.ts | 1 + libs/data-api/src/lib/data-api.module.ts | 21 +++ libs/data-api/src/lib/data-api.service.ts | 93 +++++++++++++ libs/data-api/tsconfig.json | 20 +++ libs/data-api/tsconfig.lib.json | 15 +++ package-lock.json | 122 +++++++++++++++++- package.json | 2 + tsconfig.base.json | 1 + 12 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 libs/data-api/README.md create mode 100644 libs/data-api/project.json create mode 100644 libs/data-api/src/index.ts create mode 100644 libs/data-api/src/lib/data-api.module.ts create mode 100644 libs/data-api/src/lib/data-api.service.ts create mode 100644 libs/data-api/tsconfig.json create mode 100644 libs/data-api/tsconfig.lib.json diff --git a/.env.local.sample b/.env.local.sample index 8f4e8f31..75a6a3ea 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -15,6 +15,9 @@ DB_PASSWORD=wri REDIS_HOST=localhost REDIS_PORT=6379 +# Shared Data API key that may be used by all devs and AWS environments +DATA_API_KEY=349c85f6-a915-4d3d-b337-0a0dafc0e5db + JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt AWS_ENDPOINT==http://minio:9000 diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts index 87e7220e..30d39c72 100644 --- a/apps/unified-database-service/src/airtable/airtable.module.ts +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -4,9 +4,15 @@ import { BullModule } from "@nestjs/bullmq"; import { Module } from "@nestjs/common"; import { AirtableService } from "./airtable.service"; import { AirtableProcessor } from "./airtable.processor"; +import { DataApiModule } from "@terramatch-microservices/data-api"; @Module({ - imports: [CommonModule, ConfigModule.forRoot({ isGlobal: true }), BullModule.registerQueue({ name: "airtable" })], + imports: [ + CommonModule, + ConfigModule.forRoot({ isGlobal: true }), + BullModule.registerQueue({ name: "airtable" }), + DataApiModule + ], providers: [AirtableService, AirtableProcessor], exports: [AirtableService] }) diff --git a/libs/data-api/README.md b/libs/data-api/README.md new file mode 100644 index 00000000..12b2ecbd --- /dev/null +++ b/libs/data-api/README.md @@ -0,0 +1,3 @@ +# data-api + +This library was generated with [Nx](https://nx.dev). diff --git a/libs/data-api/project.json b/libs/data-api/project.json new file mode 100644 index 00000000..36393448 --- /dev/null +++ b/libs/data-api/project.json @@ -0,0 +1,9 @@ +{ + "name": "data-api", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/data-api/src", + "projectType": "library", + "tags": [], + "// targets": "to see all targets run: nx show project data-api --web", + "targets": {} +} diff --git a/libs/data-api/src/index.ts b/libs/data-api/src/index.ts new file mode 100644 index 00000000..dd65e99f --- /dev/null +++ b/libs/data-api/src/index.ts @@ -0,0 +1 @@ +export * from "./lib/data-api.module"; diff --git a/libs/data-api/src/lib/data-api.module.ts b/libs/data-api/src/lib/data-api.module.ts new file mode 100644 index 00000000..463f7ebd --- /dev/null +++ b/libs/data-api/src/lib/data-api.module.ts @@ -0,0 +1,21 @@ +import { Module } from "@nestjs/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; +import { DataApiService } from "./data-api.service"; +import { RedisModule } from "@nestjs-modules/ioredis"; + +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true }), + RedisModule.forRootAsync({ + imports: [ConfigModule.forRoot({ isGlobal: true })], + inject: [ConfigService], + useFactory: (configService: ConfigService) => ({ + type: "single", + url: `redis://${configService.get("REDIS_HOST")}:${configService.get("REDIS_PORT")}` + }) + }) + ], + providers: [DataApiService], + exports: [DataApiService] +}) +export class DataApiModule {} diff --git a/libs/data-api/src/lib/data-api.service.ts b/libs/data-api/src/lib/data-api.service.ts new file mode 100644 index 00000000..1439d5a4 --- /dev/null +++ b/libs/data-api/src/lib/data-api.service.ts @@ -0,0 +1,93 @@ +import { Injectable, InternalServerErrorException } from "@nestjs/common"; +import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; +import Redis from "ioredis"; +import { InjectRedis } from "@nestjs-modules/ioredis"; +import { ConfigService } from "@nestjs/config"; + +const KEY_NAMESPACE = "data-api:"; + +const DATA_API_DATASET = "https://data-api.globalforestwatch.org/dataset"; +const GADM_QUERY = "/gadm_administrative_boundaries/v4.1.85/query"; +const GADM_CACHE_DURATION = 60 * 60 * 3; // keep GADM definitions in redis for 3 hours. + +const gadmLevel0 = () => ` + SELECT country AS name, gid_0 AS iso + FROM gadm_administrative_boundaries + WHERE adm_level = '0' + AND gid_0 NOT IN ('Z01', 'Z02', 'Z03', 'Z04', 'Z05', 'Z06', 'Z07', 'Z08', 'Z09', 'TWN', 'XCA', 'ESH', 'XSP') +`; + +const gadmLevel1 = (level0: string) => ` + SELECT name_1 AS name, gid_1 AS id + FROM gadm_administrative_boundaries + WHERE adm_level='1' + AND gid_0 = '${level0}' +`; + +const gadmLevel2 = (level0: string, level1: string) => ` + SELECT gid_2 as id, name_2 as name + FROM gadm_administrative_boundaries + WHERE gid_0 = '${level0}' + AND gid_1 = '${level1}' + AND adm_level='2' + AND type_2 NOT IN ('Waterbody', 'Water body', 'Water Body') +`; + +/** + * A service for accessing, and in some cases caching, data from the Data API. It's not super + * clear how this service will expand in the future, so for now it's not trying to be too + * clever about how it presents the APIs that it supports. + */ +@Injectable() +export class DataApiService { + private readonly logger = new TMLogger(DataApiService.name); + + constructor(@InjectRedis() private readonly redis: Redis, private readonly configService: ConfigService) {} + + async gadmLevel0() { + return await this.getDataset("gadm-level-0", GADM_QUERY, gadmLevel0(), GADM_CACHE_DURATION); + } + + async gadmLevel1(level0: string) { + return await this.getDataset(`gadm-level-1:${level0}`, GADM_QUERY, gadmLevel1(level0), GADM_CACHE_DURATION); + } + + async gadmLevel2(level0: string, level1: string) { + return await this.getDataset( + `gadm-level-2:${level0}:${level1}`, + GADM_QUERY, + gadmLevel2(level0, level1), + GADM_CACHE_DURATION + ); + } + + private async getDataset(key: string, queryPath: string, query: string, cacheDuration: number) { + const current = await this.redis.get(`${KEY_NAMESPACE}${key}`); + if (current != null) return JSON.parse(current); + + const appFrontend = this.configService.get("APP_FRONT_END"); + const dataApiKey = this.configService.get("DATA_API_KEY"); + if (appFrontend == null || dataApiKey == null) { + throw new InternalServerErrorException("APP_FRONT_END and DATA_API_KEY are required"); + } + + this.logger.log(`Cache miss, running query for ${key}`); + const params = new URLSearchParams(); + params.append("sql", query); + const response = await fetch(`${DATA_API_DATASET}${queryPath}?${params}`, { + headers: { + Origin: new URL(appFrontend).hostname, + ["x-api-key"]: dataApiKey + } + }); + + if (response.status !== 200) { + throw new InternalServerErrorException(response.statusText); + } + + const json = (await response.json()) as { data: any }; + await this.redis.set(`${KEY_NAMESPACE}${key}`, JSON.stringify(json.data), "EX", cacheDuration); + + return json.data; + } +} diff --git a/libs/data-api/tsconfig.json b/libs/data-api/tsconfig.json new file mode 100644 index 00000000..e97cb764 --- /dev/null +++ b/libs/data-api/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "importHelpers": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noPropertyAccessFromIndexSignature": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/libs/data-api/tsconfig.lib.json b/libs/data-api/tsconfig.lib.json new file mode 100644 index 00000000..0260985f --- /dev/null +++ b/libs/data-api/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": ["node"], + "target": "es2021", + "strictNullChecks": true, + "noImplicitAny": true, + "strictBindCallApply": true, + "forceConsistentCasingInFileNames": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*.ts"] +} diff --git a/package-lock.json b/package-lock.json index 0d5a7d4c..bcd28dcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.766.0", "@casl/ability": "^6.7.3", + "@nestjs-modules/ioredis": "^2.0.2", "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.0.11", "@nestjs/config": "^4.0.1", @@ -35,6 +36,7 @@ "debug": "^4.4.0", "geojson": "^0.5.0", "hbs": "^4.2.0", + "ioredis": "^5.6.1", "lodash": "^4.17.21", "luxon": "^3.5.0", "mariadb": "^3.4.0", @@ -3512,6 +3514,114 @@ "@tybys/wasm-util": "^0.9.0" } }, + "node_modules/@nestjs-modules/ioredis": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@nestjs-modules/ioredis/-/ioredis-2.0.2.tgz", + "integrity": "sha512-8pzSvT8R3XP6p8ZzQmEN8OnY0yWrJ/elFhwQK+PID2zf1SLBkAZ18bDcx3SKQ2atledt0gd9kBeP5xT4MlyS7Q==", + "optionalDependencies": { + "@nestjs/terminus": "10.2.0" + }, + "peerDependencies": { + "@nestjs/common": ">=6.7.0", + "@nestjs/core": ">=6.7.0", + "ioredis": ">=5.0.0" + } + }, + "node_modules/@nestjs-modules/ioredis/node_modules/@nestjs/sequelize": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/sequelize/-/sequelize-10.0.1.tgz", + "integrity": "sha512-ICzmkB0IHLm3x9SBGG8c/T9KLU9CRmtLWtuMsLHwX373HBgiKHd8Wu/5b+FM+rNRh8zWINiTTjKHoOx9kjvSjA==", + "optional": true, + "peer": true, + "dependencies": { + "uuid": "9.0.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.13 || ^0.2.0", + "rxjs": "^7.2.0", + "sequelize": "^6.3.5", + "sequelize-typescript": "^2.0.0" + } + }, + "node_modules/@nestjs-modules/ioredis/node_modules/@nestjs/terminus": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/@nestjs/terminus/-/terminus-10.2.0.tgz", + "integrity": "sha512-zPs98xvJ4ogEimRQOz8eU90mb7z+W/kd/mL4peOgrJ/VqER+ibN2Cboj65uJZW3XuNhpOqaeYOJte86InJd44A==", + "optional": true, + "dependencies": { + "boxen": "5.1.2", + "check-disk-space": "3.4.0" + }, + "peerDependencies": { + "@grpc/grpc-js": "*", + "@grpc/proto-loader": "*", + "@mikro-orm/core": "*", + "@mikro-orm/nestjs": "*", + "@nestjs/axios": "^1.0.0 || ^2.0.0 || ^3.0.0", + "@nestjs/common": "^9.0.0 || ^10.0.0", + "@nestjs/core": "^9.0.0 || ^10.0.0", + "@nestjs/microservices": "^9.0.0 || ^10.0.0", + "@nestjs/mongoose": "^9.0.0 || ^10.0.0", + "@nestjs/sequelize": "^9.0.0 || ^10.0.0", + "@nestjs/typeorm": "^9.0.0 || ^10.0.0", + "@prisma/client": "*", + "mongoose": "*", + "reflect-metadata": "0.1.x", + "rxjs": "7.x", + "sequelize": "*", + "typeorm": "*" + }, + "peerDependenciesMeta": { + "@grpc/grpc-js": { + "optional": true + }, + "@grpc/proto-loader": { + "optional": true + }, + "@mikro-orm/core": { + "optional": true + }, + "@mikro-orm/nestjs": { + "optional": true + }, + "@nestjs/axios": { + "optional": true + }, + "@nestjs/microservices": { + "optional": true + }, + "@nestjs/mongoose": { + "optional": true + }, + "@nestjs/sequelize": { + "optional": true + }, + "@nestjs/typeorm": { + "optional": true + }, + "@prisma/client": { + "optional": true + }, + "mongoose": { + "optional": true + }, + "sequelize": { + "optional": true + }, + "typeorm": { + "optional": true + } + } + }, + "node_modules/@nestjs-modules/ioredis/node_modules/reflect-metadata": { + "version": "0.1.14", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", + "integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==", + "optional": true, + "peer": true + }, "node_modules/@nestjs/bull-shared": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.2.tgz", @@ -12061,9 +12171,9 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", - "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", "dev": true, "dependencies": { "@types/http-proxy": "^1.17.8", @@ -12399,9 +12509,9 @@ } }, "node_modules/ioredis": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.0.tgz", - "integrity": "sha512-tBZlIIWbndeWBWCXWZiqtOF/yxf6yZX3tAlTJ7nfo5jhd6dctNxF7QnYlZLZ1a0o0pDoen7CgZqO+zjNaFbJAg==", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", + "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", diff --git a/package.json b/package.json index 0e4b25e0..fa7a8a00 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.766.0", "@casl/ability": "^6.7.3", + "@nestjs-modules/ioredis": "^2.0.2", "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.0.11", "@nestjs/config": "^4.0.1", @@ -39,6 +40,7 @@ "debug": "^4.4.0", "geojson": "^0.5.0", "hbs": "^4.2.0", + "ioredis": "^5.6.1", "lodash": "^4.17.21", "luxon": "^3.5.0", "mariadb": "^3.4.0", diff --git a/tsconfig.base.json b/tsconfig.base.json index abeb17c7..22b19687 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "paths": { "@terramatch-microservices/common": ["libs/common/src/index.ts"], "@terramatch-microservices/common/*": ["libs/common/src/lib/*"], + "@terramatch-microservices/data-api": ["libs/data-api/src/index.ts"], "@terramatch-microservices/database": ["libs/database/src/index.ts"], "@terramatch-microservices/database/*": ["libs/database/src/lib/*"], "@terramatch-microservices/tm-v3-cli": ["tm-v3-cli/src/index.ts"] From 7ea63ccaaf665092781bbab271b44e0827ad512b Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 14:56:02 -0600 Subject: [PATCH 79/98] [TM-1959] feat: refactor Disturbance and Invasive DTOs to extend AssociationDto for improved data handling --- .../src/entities/dto/disturbance.dto.ts | 47 +------ .../src/entities/dto/invasive.dto.ts | 58 +-------- .../src/entities/entities.service.ts | 26 +++- .../processors/disturbance.processor.ts | 121 ----------------- .../entities/processors/invasive.processor.ts | 122 ------------------ .../src/lib/policies/invasive.policy.ts | 4 +- .../common/src/lib/policies/policy.service.ts | 5 +- libs/database/src/lib/entities/index.ts | 1 + 8 files changed, 39 insertions(+), 345 deletions(-) delete mode 100644 apps/entity-service/src/entities/processors/disturbance.processor.ts delete mode 100644 apps/entity-service/src/entities/processors/invasive.processor.ts diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index e42cbeb3..c839aebd 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -1,50 +1,15 @@ import { JsonApiDto } from "@terramatch-microservices/common/decorators"; -import { AdditionalProps, EntityDto } from "./entity.dto"; -import { Nursery } from "@terramatch-microservices/database/entities"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; +import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto"; @JsonApiDto({ type: "disturbances" }) -export class DisturbanceLightDto extends EntityDto { - constructor(disturbance?: Disturbance, props?: AdditionalNurseryLightProps) { - super(); - if (disturbance != null) { - this.populate(DisturbanceLightDto, { - ...pickApiProperties(disturbance, DisturbanceLightDto), - lightResource: true, - ...props, - // these two are untyped and marked optional in the base model. - createdAt: disturbance.createdAt as Date, - updatedAt: disturbance.updatedAt as Date - }); - } - } - - @ApiProperty({ nullable: true }) - name: string | null; - - @ApiProperty() - createdAt: Date; - - @ApiProperty() - updatedAt: Date; -} - -export type AdditionalNurseryLightProps = DisturbanceLightDto; -export type AdditionalNurseryFullProps = AdditionalNurseryLightProps & - AdditionalProps>; - -export class DisturbanceFullDto extends DisturbanceLightDto { - constructor(nursery: Disturbance, props: AdditionalNurseryFullProps) { - super(); - this.populate(DisturbanceFullDto, { - ...pickApiProperties(nursery, DisturbanceFullDto), - lightResource: false, - // these two are untyped and marked optional in the base model. - createdAt: nursery.createdAt as Date, - updatedAt: nursery.updatedAt as Date, - ...props +export class DisturbanceDto extends AssociationDto { + constructor(disturbance: Disturbance, additional: AssociationDtoAdditionalProps) { + super({ + ...pickApiProperties(disturbance, DisturbanceDto), + ...additional }); } diff --git a/apps/entity-service/src/entities/dto/invasive.dto.ts b/apps/entity-service/src/entities/dto/invasive.dto.ts index 01b345db..028fcb1f 100644 --- a/apps/entity-service/src/entities/dto/invasive.dto.ts +++ b/apps/entity-service/src/entities/dto/invasive.dto.ts @@ -1,67 +1,23 @@ import { JsonApiDto } from "@terramatch-microservices/common/decorators"; -import { AdditionalProps, EntityDto } from "./entity.dto"; -import { Nursery } from "@terramatch-microservices/database/entities"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; +import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto"; @JsonApiDto({ type: "invasives" }) -export class InvasiveLightDto extends EntityDto { - constructor(invasive?: Invasive, props?: AdditionalInvasiveLightProps) { - super(); - if (invasive != null) { - this.populate(InvasiveLightDto, { - ...pickApiProperties(invasive, InvasiveLightDto), - lightResource: true, - ...props, - // these two are untyped and marked optional in the base model. - createdAt: invasive.createdAt as Date, - updatedAt: invasive.updatedAt as Date - }); - } - } - - @ApiProperty({ nullable: true }) - name: string | null; - - @ApiProperty() - createdAt: Date; - - @ApiProperty() - updatedAt: Date; -} - -export type AdditionalInvasiveLightProps = InvasiveLightDto; -export type AdditionalNurseryFullProps = AdditionalInvasiveLightProps & - AdditionalProps>; - -export class InvasiveFullDto extends InvasiveLightDto { - constructor(invasive: Invasive, props: AdditionalNurseryFullProps) { - super(); - this.populate(InvasiveFullDto, { - ...pickApiProperties(invasive, InvasiveFullDto), - lightResource: false, - // these two are untyped and marked optional in the base model. - createdAt: invasive.createdAt as Date, - updatedAt: invasive.updatedAt as Date, - ...props +export class InvasiveDto extends AssociationDto { + constructor(invasive: Invasive, additional: AssociationDtoAdditionalProps) { + super({ + ...pickApiProperties(invasive, InvasiveDto), + ...additional }); } - @ApiProperty({ nullable: true }) - collection: string | null; - @ApiProperty({ nullable: true }) type: string | null; @ApiProperty({ nullable: true }) - intensity: string | null; - - @ApiProperty({ nullable: true }) - extent: string | null; - - @ApiProperty({ nullable: true }) - description: string | null; + name: string | null; @ApiProperty({ nullable: true }) oldId: number; diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index 99d79108..9c8e8155 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -5,7 +5,14 @@ import { EntityProcessor } from "./processors/entity-processor"; import { EntityQueryDto } from "./dto/entity-query.dto"; import { PaginatedQueryBuilder } from "@terramatch-microservices/database/util/paginated-query.builder"; import { MediaService } from "@terramatch-microservices/common/media/media.service"; -import { Demographic, Media, Seeding, TreeSpecies, User } from "@terramatch-microservices/database/entities"; +import { + Demographic, + Disturbance, + Media, + Seeding, + TreeSpecies, + User +} from "@terramatch-microservices/database/entities"; import { MediaDto } from "./dto/media.dto"; import { MediaCollection } from "@terramatch-microservices/database/types/media"; import { groupBy } from "lodash"; @@ -26,10 +33,9 @@ import { PolicyService } from "@terramatch-microservices/common"; import { EntityUpdateData } from "./dto/entity-update.dto"; import { LocalizationService } from "@terramatch-microservices/common/localization/localization.service"; import { ITranslateParams } from "@transifex/native"; -import { DisturbanceProcessor } from "./processors/disturbance.processor"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { InvasiveProcessor } from "./processors/invasive.processor"; import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; +import { DisturbanceDto } from "./dto/disturbance.dto"; +import { InvasiveDto } from "./dto/invasive.dto"; // The keys of this array must match the type in the resulting DTO. const ENTITY_PROCESSORS = { @@ -38,9 +44,7 @@ const ENTITY_PROCESSORS = { nurseries: NurseryProcessor, projectReports: ProjectReportProcessor, nurseryReports: NurseryReportProcessor, - siteReports: SiteReportProcessor, - disturbances: DisturbanceProcessor, - invasives: InvasiveProcessor + siteReports: SiteReportProcessor }; export type ProcessableEntity = keyof typeof ENTITY_PROCESSORS; @@ -72,6 +76,14 @@ const ASSOCIATION_PROCESSORS = { attributes: ["uuid", "name", "taxonId", "collection", [fn("SUM", col("amount")), "amount"]], group: ["taxonId", "name", "collection"] }) + ), + disturbances: AssociationProcessor.buildSimpleProcessor( + DisturbanceDto, + ({ id: disturbanceableId }, disturbanceableType) => + Disturbance.findAll({ where: { disturbanceableType, disturbanceableId, hidden: false } }) + ), + invasives: AssociationProcessor.buildSimpleProcessor(InvasiveDto, ({ id: invasiveableId }, invasiveableType) => + Invasive.findAll({ where: { invasiveableType, invasiveableId, hidden: false } }) ) }; diff --git a/apps/entity-service/src/entities/processors/disturbance.processor.ts b/apps/entity-service/src/entities/processors/disturbance.processor.ts deleted file mode 100644 index 9f90de9e..00000000 --- a/apps/entity-service/src/entities/processors/disturbance.processor.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { NurseryReport, ProjectUser } from "@terramatch-microservices/database/entities"; -import { EntityProcessor } from "./entity-processor"; -import { EntityQueryDto } from "../dto/entity-query.dto"; -import { col, fn, Includeable, Op } from "sequelize"; -import { BadRequestException, NotAcceptableException } from "@nestjs/common"; -import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; -import { EntityUpdateAttributes } from "../dto/entity-update.dto"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { DisturbanceFullDto, DisturbanceLightDto } from "../dto/disturbance.dto"; - -export class DisturbanceProcessor extends EntityProcessor< - Disturbance, - DisturbanceLightDto, - DisturbanceFullDto, - EntityUpdateAttributes -> { - readonly LIGHT_DTO = DisturbanceLightDto; - readonly FULL_DTO = DisturbanceFullDto; - - async findOne(uuid: string) { - return await Disturbance.findOne({ - where: { uuid } - /*include: [ - { - association: "project", - attributes: ["uuid", "name"], - include: [{ association: "organisation", attributes: ["name"] }] - } - ]*/ - }); - } - - async findMany(query: EntityQueryDto) { - const projectAssociation: Includeable = { - association: "project", - attributes: ["uuid", "name"], - include: [{ association: "organisation", attributes: ["name"] }] - }; - - const builder = await this.entitiesService.buildQuery(Disturbance, query, [projectAssociation]); - if (query.sort != null) { - if (["name", "startDate", "status", "updateRequestStatus", "createdAt"].includes(query.sort.field)) { - builder.order([query.sort.field, query.sort.direction ?? "ASC"]); - } else if (query.sort.field === "organisationName") { - builder.order(["project", "organisation", "name", query.sort.direction ?? "ASC"]); - } else if (query.sort.field === "projectName") { - builder.order(["project", "name", query.sort.direction ?? "ASC"]); - } else if (query.sort.field !== "id") { - throw new BadRequestException(`Invalid sort field: ${query.sort.field}`); - } - } - - const permissions = await this.entitiesService.getPermissions(); - const frameworkPermissions = permissions - ?.filter(name => name.startsWith("framework-")) - .map(name => name.substring("framework-".length) as FrameworkKey); - if (frameworkPermissions?.length > 0) { - builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); - } else if (permissions?.includes("manage-own")) { - builder.where({ projectId: { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } }); - } else if (permissions?.includes("projects-manage")) { - builder.where({ projectId: { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } }); - } - - const associationFieldMap = { - organisationUuid: "$project.organisation.uuid$", - country: "$project.country$", - projectUuid: "$project.uuid$" - }; - - for (const term of [ - "status", - "updateRequestStatus", - "frameworkKey", - "organisationUuid", - "country", - "projectUuid" - ]) { - if (query[term] != null) { - const field = associationFieldMap[term] ?? term; - builder.where({ [field]: query[term] }); - } - } - - if (query.search != null || query.searchFilter != null) { - builder.where({ - [Op.or]: [ - { name: { [Op.like]: `%${query.search ?? query.searchFilter}%` } }, - { "$project.name$": { [Op.like]: `%${query.search}%` } }, - { "$project.organisation.name$": { [Op.like]: `%${query.search}%` } } - ] - }); - } - - return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; - } - - async getFullDto(disturbance: Disturbance) { - const nurseryId = disturbance.id; - - return { id: disturbance.uuid, dto: new DisturbanceFullDto(disturbance, null) }; - } - - async getLightDto(disturbance: Disturbance) { - const nurseryId = disturbance.id; - return { id: disturbance.uuid, dto: new DisturbanceLightDto(disturbance, null) }; - } - - async delete(disturbance: Disturbance) { - const permissions = await this.entitiesService.getPermissions(); - const managesOwn = permissions.includes("manage-own"); // TODO validate this condition - if (managesOwn) { - const reportCount = await NurseryReport.count({ where: { nurseryId: disturbance.id } }); - if (reportCount > 0) { - throw new NotAcceptableException("You can only delete nurseries that do not have reports"); - } - } - - await super.delete(disturbance); - } -} diff --git a/apps/entity-service/src/entities/processors/invasive.processor.ts b/apps/entity-service/src/entities/processors/invasive.processor.ts deleted file mode 100644 index 586b2e3c..00000000 --- a/apps/entity-service/src/entities/processors/invasive.processor.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { NurseryReport, ProjectUser } from "@terramatch-microservices/database/entities"; -import { EntityProcessor } from "./entity-processor"; -import { EntityQueryDto } from "../dto/entity-query.dto"; -import { col, fn, Includeable, Op } from "sequelize"; -import { BadRequestException, NotAcceptableException } from "@nestjs/common"; -import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; -import { EntityUpdateAttributes } from "../dto/entity-update.dto"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; -import { InvasiveFullDto, InvasiveLightDto } from "../dto/invasive.dto"; - -export class InvasiveProcessor extends EntityProcessor< - Invasive, - InvasiveLightDto, - InvasiveFullDto, - EntityUpdateAttributes -> { - readonly LIGHT_DTO = InvasiveLightDto; - readonly FULL_DTO = InvasiveFullDto; - - async findOne(uuid: string) { - return await Invasive.findOne({ - where: { uuid } - /*include: [ - { - association: "project", - attributes: ["uuid", "name"], - include: [{ association: "organisation", attributes: ["name"] }] - } - ]*/ - }); - } - - async findMany(query: EntityQueryDto) { - const projectAssociation: Includeable = { - association: "project", - attributes: ["uuid", "name"], - include: [{ association: "organisation", attributes: ["name"] }] - }; - - const builder = await this.entitiesService.buildQuery(Invasive, query, [projectAssociation]); - if (query.sort != null) { - if (["name", "startDate", "status", "updateRequestStatus", "createdAt"].includes(query.sort.field)) { - builder.order([query.sort.field, query.sort.direction ?? "ASC"]); - } else if (query.sort.field === "organisationName") { - builder.order(["project", "organisation", "name", query.sort.direction ?? "ASC"]); - } else if (query.sort.field === "projectName") { - builder.order(["project", "name", query.sort.direction ?? "ASC"]); - } else if (query.sort.field !== "id") { - throw new BadRequestException(`Invalid sort field: ${query.sort.field}`); - } - } - - const permissions = await this.entitiesService.getPermissions(); - const frameworkPermissions = permissions - ?.filter(name => name.startsWith("framework-")) - .map(name => name.substring("framework-".length) as FrameworkKey); - if (frameworkPermissions?.length > 0) { - builder.where({ frameworkKey: { [Op.in]: frameworkPermissions } }); - } else if (permissions?.includes("manage-own")) { - builder.where({ projectId: { [Op.in]: ProjectUser.userProjectsSubquery(this.entitiesService.userId) } }); - } else if (permissions?.includes("projects-manage")) { - builder.where({ projectId: { [Op.in]: ProjectUser.projectsManageSubquery(this.entitiesService.userId) } }); - } - - const associationFieldMap = { - organisationUuid: "$project.organisation.uuid$", - country: "$project.country$", - projectUuid: "$project.uuid$" - }; - - for (const term of [ - "status", - "updateRequestStatus", - "frameworkKey", - "organisationUuid", - "country", - "projectUuid" - ]) { - if (query[term] != null) { - const field = associationFieldMap[term] ?? term; - builder.where({ [field]: query[term] }); - } - } - - if (query.search != null || query.searchFilter != null) { - builder.where({ - [Op.or]: [ - { name: { [Op.like]: `%${query.search ?? query.searchFilter}%` } }, - { "$project.name$": { [Op.like]: `%${query.search}%` } }, - { "$project.organisation.name$": { [Op.like]: `%${query.search}%` } } - ] - }); - } - - return { models: await builder.execute(), paginationTotal: await builder.paginationTotal() }; - } - - async getFullDto(invasive: Invasive) { - const nurseryId = invasive.id; - - return { id: invasive.uuid, dto: new InvasiveFullDto(invasive, null) }; - } - - async getLightDto(invasive: Invasive) { - const nurseryId = invasive.id; - return { id: invasive.uuid, dto: new InvasiveLightDto(invasive, null) }; - } - - async delete(invasive: Invasive) { - const permissions = await this.entitiesService.getPermissions(); - const managesOwn = permissions.includes("manage-own"); // TODO validate this condition - if (managesOwn) { - const reportCount = await NurseryReport.count({ where: { nurseryId: invasive.id } }); - if (reportCount > 0) { - throw new NotAcceptableException("You can only delete nurseries that do not have reports"); - } - } - - await super.delete(invasive); - } -} diff --git a/libs/common/src/lib/policies/invasive.policy.ts b/libs/common/src/lib/policies/invasive.policy.ts index 3cc50bab..da8add96 100644 --- a/libs/common/src/lib/policies/invasive.policy.ts +++ b/libs/common/src/lib/policies/invasive.policy.ts @@ -1,10 +1,10 @@ -import { Disturbance, Project, Site, User } from "@terramatch-microservices/database/entities"; +import { Invasive, Project, Site, User } from "@terramatch-microservices/database/entities"; import { UserPermissionsPolicy } from "./user-permissions.policy"; export class InvasivePolicy extends UserPermissionsPolicy { async addRules() { if (this.permissions.includes("view-dashboard") || this.permissions.includes("reports-manage")) { - this.builder.can("read", Disturbance); + this.builder.can("read", Invasive); return; } diff --git a/libs/common/src/lib/policies/policy.service.ts b/libs/common/src/lib/policies/policy.service.ts index 7a50e9d4..ab8236c0 100644 --- a/libs/common/src/lib/policies/policy.service.ts +++ b/libs/common/src/lib/policies/policy.service.ts @@ -3,6 +3,7 @@ import { RequestContext } from "nestjs-request-context"; import { UserPolicy } from "./user.policy"; import { Disturbance, + Invasive, Nursery, NurseryReport, Permission, @@ -26,6 +27,7 @@ import { NurseryReportPolicy } from "./nursery-report.policy"; import { NurseryPolicy } from "./nursery.policy"; import { TMLogger } from "../util/tm-logger"; import { DisturbancePolicy } from "@terramatch-microservices/common/policies/disturbance.policy"; +import { InvasivePolicy } from "@terramatch-microservices/common/policies/invasive.policy"; type EntityClass = { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -46,7 +48,8 @@ const POLICIES: [EntityClass, PolicyClass][] = [ [SitePolygon, SitePolygonPolicy], [SiteReport, SiteReportPolicy], [User, UserPolicy], - [Disturbance, DisturbancePolicy] + [Disturbance, DisturbancePolicy], + [Invasive, InvasivePolicy] ]; /** diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index dd378c34..791a632e 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -18,6 +18,7 @@ export * from "./indicator-output-msu-carbon.entity"; export * from "./indicator-output-tree-count.entity"; export * from "./indicator-output-tree-cover.entity"; export * from "./indicator-output-tree-cover-loss.entity"; +export * from "./invasive.entity"; export * from "./landscape-geometry.entity"; export * from "./localization-key"; export * from "./media.entity"; From 0d12828899dee17cb16ba72a214755792dcdbf7e Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 15:31:57 -0600 Subject: [PATCH 80/98] [TM-1959] feat: add Strata entity and DTO support to entity processing and update related services --- .../src/entities/dto/strata.dto.ts | 29 +++++++++++ .../src/entities/entities.service.ts | 5 ++ .../lib/email/entity-status-update.email.ts | 6 --- .../entity-status-update.event-processor.ts | 6 --- .../src/lib/policies/disturbance.policy.ts | 48 ------------------- .../src/lib/policies/invasive.policy.ts | 48 ------------------- .../common/src/lib/policies/policy.service.ts | 8 +--- libs/database/src/lib/constants/entities.ts | 9 ++-- libs/database/src/lib/entities/index.ts | 1 + .../src/lib/entities/stratas.entity.ts | 35 ++++++++++++++ 10 files changed, 75 insertions(+), 120 deletions(-) create mode 100644 apps/entity-service/src/entities/dto/strata.dto.ts delete mode 100644 libs/common/src/lib/policies/disturbance.policy.ts delete mode 100644 libs/common/src/lib/policies/invasive.policy.ts create mode 100644 libs/database/src/lib/entities/stratas.entity.ts diff --git a/apps/entity-service/src/entities/dto/strata.dto.ts b/apps/entity-service/src/entities/dto/strata.dto.ts new file mode 100644 index 00000000..32b78e45 --- /dev/null +++ b/apps/entity-service/src/entities/dto/strata.dto.ts @@ -0,0 +1,29 @@ +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { ApiProperty } from "@nestjs/swagger"; +import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; +import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto"; +import { Strata } from "@terramatch-microservices/database/entities/stratas.entity"; +import { AllowNull, Column } from "sequelize-typescript"; +import { INTEGER, STRING } from "sequelize"; + +@JsonApiDto({ type: "stratas" }) +export class StrataDto extends AssociationDto { + constructor(strata: Strata, additional: AssociationDtoAdditionalProps) { + super({ + ...pickApiProperties(strata, StrataDto), + ...additional + }); + } + + @AllowNull + @Column(STRING) + description: string | null; + + @AllowNull + @Column(INTEGER) + extent: string | null; + + @ApiProperty({ nullable: true }) + hidden: number | null; +} diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index 9c8e8155..a49e735c 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -36,6 +36,8 @@ import { ITranslateParams } from "@transifex/native"; import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; import { DisturbanceDto } from "./dto/disturbance.dto"; import { InvasiveDto } from "./dto/invasive.dto"; +import { Strata } from "@terramatch-microservices/database/entities/stratas.entity"; +import { StrataDto } from "./dto/strata.dto"; // The keys of this array must match the type in the resulting DTO. const ENTITY_PROCESSORS = { @@ -84,6 +86,9 @@ const ASSOCIATION_PROCESSORS = { ), invasives: AssociationProcessor.buildSimpleProcessor(InvasiveDto, ({ id: invasiveableId }, invasiveableType) => Invasive.findAll({ where: { invasiveableType, invasiveableId, hidden: false } }) + ), + stratas: AssociationProcessor.buildSimpleProcessor(StrataDto, ({ id: stratasableId }, stratasableType) => + Strata.findAll({ where: { stratasableType, stratasableId, hidden: false } }) ) }; diff --git a/libs/common/src/lib/email/entity-status-update.email.ts b/libs/common/src/lib/email/entity-status-update.email.ts index b1cd409d..ef0f8504 100644 --- a/libs/common/src/lib/email/entity-status-update.email.ts +++ b/libs/common/src/lib/email/entity-status-update.email.ts @@ -22,8 +22,6 @@ import { Op } from "sequelize"; import { TMLogger } from "../util/tm-logger"; import { InternalServerErrorException } from "@nestjs/common"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export class EntityStatusUpdateEmail extends EmailSender { private readonly logger = new TMLogger(EntityStatusUpdateEmail.name); @@ -39,8 +37,6 @@ export class EntityStatusUpdateEmail extends EmailSender { async send(emailService: EmailService) { const entity = await this.getEntity(); - if (entity instanceof Disturbance) return; - if (entity instanceof Invasive) return; const status = entity.status === NEEDS_MORE_INFORMATION || entity.updateRequestStatus === NEEDS_MORE_INFORMATION ? NEEDS_MORE_INFORMATION @@ -114,8 +110,6 @@ export class EntityStatusUpdateEmail extends EmailSender { } private async getFeedback(entity: EntityModel) { - if (entity instanceof Disturbance) return; - if (entity instanceof Invasive) return; if (![APPROVED, NEEDS_MORE_INFORMATION].includes(entity.updateRequestStatus ?? "")) { return entity.feedback; } diff --git a/libs/common/src/lib/events/entity-status-update.event-processor.ts b/libs/common/src/lib/events/entity-status-update.event-processor.ts index e6cf442e..7adb7692 100644 --- a/libs/common/src/lib/events/entity-status-update.event-processor.ts +++ b/libs/common/src/lib/events/entity-status-update.event-processor.ts @@ -15,8 +15,6 @@ import { Action, AuditStatus, FormQuestion } from "@terramatch-microservices/dat import { get, isEmpty, map } from "lodash"; import { Op } from "sequelize"; import { STATUS_DISPLAY_STRINGS } from "@terramatch-microservices/database/constants/status"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export type StatusUpdateModel = LaravelModel & StatusModel & FeedbackModel; @@ -55,10 +53,6 @@ export class EntityStatusUpdate extends EventProcessor { private async updateActions() { this.logger.log(`Updating actions [${JSON.stringify({ model: this.model.constructor.name, id: this.model.id })}]`); const entity = this.model as EntityModel; - - if (entity instanceof Disturbance) return; - if (entity instanceof Invasive) return; - await Action.for(entity).destroy({ where: { type: "notification" } }); if (entity.status !== "awaiting-approval") { diff --git a/libs/common/src/lib/policies/disturbance.policy.ts b/libs/common/src/lib/policies/disturbance.policy.ts deleted file mode 100644 index b5e8baf1..00000000 --- a/libs/common/src/lib/policies/disturbance.policy.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Disturbance, Project, Site, User } from "@terramatch-microservices/database/entities"; -import { UserPermissionsPolicy } from "./user-permissions.policy"; - -export class DisturbancePolicy extends UserPermissionsPolicy { - async addRules() { - if (this.permissions.includes("view-dashboard") || this.permissions.includes("reports-manage")) { - this.builder.can("read", Disturbance); - return; - } - - if (this.permissions.includes("manage-own")) { - const user = await this.getUser(); - if (user != null) { - const projectIds = [ - ...(user.organisationId === null - ? [] - : await Project.findAll({ where: { organisationId: user.organisationId }, attributes: ["id"] }) - ).map(({ id }) => id), - ...user.projects.map(({ id }) => id) - ]; - if (projectIds.length > 0) { - this.builder.can(["read", "delete", "update"], Site, { projectId: { $in: projectIds } }); - } - } - } - - if (this.permissions.includes("projects-manage")) { - const user = await this.getUser(); - if (user != null) { - const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); - if (projectIds.length > 0) { - this.builder.can(["read", "delete", "update", "approve"], Site, { projectId: { $in: projectIds } }); - } - } - } - } - - protected _user?: User | null; - protected async getUser() { - if (this._user != null) return this._user; - - return (this._user = await User.findOne({ - where: { id: this.userId }, - attributes: ["organisationId"], - include: [{ association: "projects", attributes: ["id"] }] - })); - } -} diff --git a/libs/common/src/lib/policies/invasive.policy.ts b/libs/common/src/lib/policies/invasive.policy.ts deleted file mode 100644 index da8add96..00000000 --- a/libs/common/src/lib/policies/invasive.policy.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Invasive, Project, Site, User } from "@terramatch-microservices/database/entities"; -import { UserPermissionsPolicy } from "./user-permissions.policy"; - -export class InvasivePolicy extends UserPermissionsPolicy { - async addRules() { - if (this.permissions.includes("view-dashboard") || this.permissions.includes("reports-manage")) { - this.builder.can("read", Invasive); - return; - } - - if (this.permissions.includes("manage-own")) { - const user = await this.getUser(); - if (user != null) { - const projectIds = [ - ...(user.organisationId === null - ? [] - : await Project.findAll({ where: { organisationId: user.organisationId }, attributes: ["id"] }) - ).map(({ id }) => id), - ...user.projects.map(({ id }) => id) - ]; - if (projectIds.length > 0) { - this.builder.can(["read", "delete", "update"], Site, { projectId: { $in: projectIds } }); - } - } - } - - if (this.permissions.includes("projects-manage")) { - const user = await this.getUser(); - if (user != null) { - const projectIds = user.projects.filter(({ ProjectUser }) => ProjectUser.isManaging).map(({ id }) => id); - if (projectIds.length > 0) { - this.builder.can(["read", "delete", "update", "approve"], Site, { projectId: { $in: projectIds } }); - } - } - } - } - - protected _user?: User | null; - protected async getUser() { - if (this._user != null) return this._user; - - return (this._user = await User.findOne({ - where: { id: this.userId }, - attributes: ["organisationId"], - include: [{ association: "projects", attributes: ["id"] }] - })); - } -} diff --git a/libs/common/src/lib/policies/policy.service.ts b/libs/common/src/lib/policies/policy.service.ts index ab8236c0..c823ee97 100644 --- a/libs/common/src/lib/policies/policy.service.ts +++ b/libs/common/src/lib/policies/policy.service.ts @@ -2,8 +2,6 @@ import { Injectable, Scope, UnauthorizedException } from "@nestjs/common"; import { RequestContext } from "nestjs-request-context"; import { UserPolicy } from "./user.policy"; import { - Disturbance, - Invasive, Nursery, NurseryReport, Permission, @@ -26,8 +24,6 @@ import { SitePolicy } from "./site.policy"; import { NurseryReportPolicy } from "./nursery-report.policy"; import { NurseryPolicy } from "./nursery.policy"; import { TMLogger } from "../util/tm-logger"; -import { DisturbancePolicy } from "@terramatch-microservices/common/policies/disturbance.policy"; -import { InvasivePolicy } from "@terramatch-microservices/common/policies/invasive.policy"; type EntityClass = { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -47,9 +43,7 @@ const POLICIES: [EntityClass, PolicyClass][] = [ [Site, SitePolicy], [SitePolygon, SitePolygonPolicy], [SiteReport, SiteReportPolicy], - [User, UserPolicy], - [Disturbance, DisturbancePolicy], - [Invasive, InvasivePolicy] + [User, UserPolicy] ]; /** diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 1c24a79f..8f425873 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -16,19 +16,18 @@ export const REPORT_MODELS: { [R in ReportType]: ReportClass } = { nurseryReports: NurseryReport }; -export const ENTITY_TYPES = ["projects", "sites", "nurseries", "disturbances", "invasives", ...REPORT_TYPES] as const; +export const ENTITY_TYPES = ["projects", "sites", "nurseries", ...REPORT_TYPES] as const; export type EntityType = (typeof ENTITY_TYPES)[number]; -export type EntityModel = ReportModel | Project | Site | Nursery | Disturbance | Invasive; +export type EntityModel = ReportModel | Project | Site | Nursery; export type EntityClass = ModelCtor & ModelStatic & { LARAVEL_TYPE: string }; export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { ...REPORT_MODELS, projects: Project, sites: Site, - nurseries: Nursery, - disturbances: Disturbance, + nurseries: Nursery + // disturbances: Disturbance, // stratas: Strata::class, - invasives: Invasive }; export const isReport = (entity: EntityModel): entity is ReportModel => diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 791a632e..814fe76c 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -39,6 +39,7 @@ export * from "./seeding.entity"; export * from "./site.entity"; export * from "./site-polygon.entity"; export * from "./site-report.entity"; +export * from "./stratas.entity"; export * from "./tree-species.entity"; export * from "./tree-species-research.entity"; export * from "./update-request.entity"; diff --git a/libs/database/src/lib/entities/stratas.entity.ts b/libs/database/src/lib/entities/stratas.entity.ts new file mode 100644 index 00000000..799762fc --- /dev/null +++ b/libs/database/src/lib/entities/stratas.entity.ts @@ -0,0 +1,35 @@ +import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; + +@Table({ tableName: "v2_stratas", underscored: true, paranoid: true }) +export class Strata extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column({ type: UUID, defaultValue: UUIDV4 }) + uuid: string; + + @AllowNull + @Column(INTEGER.UNSIGNED) + owner_id: number | null; + + @Column(STRING) + stratasableType: string; + + @Column(BIGINT.UNSIGNED) + stratasableId: number; + + @AllowNull + @Column(STRING) + description: string | null; + + @AllowNull + @Column(INTEGER) + extent: string | null; + + @Column(TINYINT) + hidden: number | null; +} From b87ddf4cb0634e73eb138b572497b3c7d04ec5d2 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 15:33:02 -0600 Subject: [PATCH 81/98] [TM-1959] feat: remove Disturbance and Invasive checks from update method for streamlined processing --- .../src/entities/processors/entity-processor.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/entity-service/src/entities/processors/entity-processor.ts b/apps/entity-service/src/entities/processors/entity-processor.ts index 14be606b..c7a7cc1e 100644 --- a/apps/entity-service/src/entities/processors/entity-processor.ts +++ b/apps/entity-service/src/entities/processors/entity-processor.ts @@ -9,8 +9,6 @@ import { EntityModel } from "@terramatch-microservices/database/constants/entiti import { Action } from "@terramatch-microservices/database/entities/action.entity"; import { EntityUpdateData } from "../dto/entity-update.dto"; import { APPROVED, NEEDS_MORE_INFORMATION } from "@terramatch-microservices/database/constants/status"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export type Aggregate> = { func: string; @@ -119,9 +117,6 @@ export abstract class EntityProcessor< * and set the appropriate fields and then call super.update() */ async update(model: ModelType, update: UpdateDto) { - if (model instanceof Disturbance) return; - if (model instanceof Invasive) return; - if (update.status != null) { if (this.APPROVAL_STATUSES.includes(update.status)) { await this.entitiesService.authorize("approve", model); From 015e4274f83f02a600c28793995da691431f300b Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 16:01:13 -0600 Subject: [PATCH 82/98] [TM-1959] remove unused import for Invasive entity in strata.dto.ts --- apps/entity-service/src/entities/dto/strata.dto.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/entity-service/src/entities/dto/strata.dto.ts b/apps/entity-service/src/entities/dto/strata.dto.ts index 32b78e45..0c7a058c 100644 --- a/apps/entity-service/src/entities/dto/strata.dto.ts +++ b/apps/entity-service/src/entities/dto/strata.dto.ts @@ -1,7 +1,6 @@ import { JsonApiDto } from "@terramatch-microservices/common/decorators"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { ApiProperty } from "@nestjs/swagger"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto"; import { Strata } from "@terramatch-microservices/database/entities/stratas.entity"; import { AllowNull, Column } from "sequelize-typescript"; From 5c582ce4e09d43cf0aa99e110a5cb36ed3b98ebc Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 16:05:48 -0600 Subject: [PATCH 83/98] [TM-1959] refactor: clean up imports in entities and stratas.entity files --- libs/database/src/lib/constants/entities.ts | 4 +--- libs/database/src/lib/entities/stratas.entity.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 8f425873..6dab9839 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -1,9 +1,7 @@ -import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport } from "../entities"; +import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport, Disturbance, Invasive } from "../entities"; import { ModelCtor } from "sequelize-typescript"; import { ModelStatic } from "sequelize"; import { kebabCase } from "lodash"; -import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity"; -import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity"; export const REPORT_TYPES = ["projectReports", "siteReports", "nurseryReports"] as const; export type ReportType = (typeof REPORT_TYPES)[number]; diff --git a/libs/database/src/lib/entities/stratas.entity.ts b/libs/database/src/lib/entities/stratas.entity.ts index 799762fc..2a3325d0 100644 --- a/libs/database/src/lib/entities/stratas.entity.ts +++ b/libs/database/src/lib/entities/stratas.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { BIGINT, INTEGER, STRING, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_stratas", underscored: true, paranoid: true }) export class Strata extends Model { From 16b860cc69eaa0a79d41ba0c8ae4ee0afc2f03bf Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Apr 2025 15:14:49 -0700 Subject: [PATCH 84/98] [TM-1958] Use the data API to get the gadm level 0 and level 1 names from the codes in DB entities. --- .../src/airtable/airtable.processor.ts | 11 ++++-- .../src/airtable/entities/airtable-entity.ts | 23 ++++++++++- .../entities/organisation.airtable-entity.ts | 38 +++++++++++++++++-- .../entities/project-pitch.airtable-entity.ts | 36 ++++++++++++++++-- .../entities/project.airtable-entity.ts | 32 ++++++++++------ libs/data-api/src/index.ts | 2 + libs/data-api/src/lib/data-api.service.ts | 28 ++++++++------ 7 files changed, 135 insertions(+), 35 deletions(-) diff --git a/apps/unified-database-service/src/airtable/airtable.processor.ts b/apps/unified-database-service/src/airtable/airtable.processor.ts index 9b7bfe96..1938bd63 100644 --- a/apps/unified-database-service/src/airtable/airtable.processor.ts +++ b/apps/unified-database-service/src/airtable/airtable.processor.ts @@ -21,6 +21,7 @@ import { import * as Sentry from "@sentry/node"; import { SlackService } from "@terramatch-microservices/common/slack/slack.service"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; +import { DataApiService } from "@terramatch-microservices/data-api"; export const AIRTABLE_ENTITIES = { applications: ApplicationEntity, @@ -66,7 +67,11 @@ export class AirtableProcessor extends WorkerHost { private readonly logger = new TMLogger(AirtableProcessor.name); private readonly base: Airtable.Base; - constructor(private readonly config: ConfigService, private readonly slack: SlackService) { + constructor( + private readonly config: ConfigService, + private readonly slack: SlackService, + private readonly dataApi: DataApiService + ) { super(); this.base = new Airtable({ apiKey: this.config.get("AIRTABLE_API_KEY") }).base(this.config.get("AIRTABLE_BASE_ID")); } @@ -104,7 +109,7 @@ export class AirtableProcessor extends WorkerHost { throw new InternalServerErrorException(`Entity mapping not found for entity type ${entityType}`); } - const entity = new entityClass(); + const entity = new entityClass(this.dataApi); await entity.updateBase(this.base, { startPage, updatedSince }); this.logger.log(`Completed entity update: ${JSON.stringify({ entityType, updatedSince })}`); @@ -119,7 +124,7 @@ export class AirtableProcessor extends WorkerHost { throw new InternalServerErrorException(`Entity mapping not found for entity type ${entityType}`); } - const entity = new entityClass(); + const entity = new entityClass(this.dataApi); await entity.deleteStaleRecords(this.base, deletedSince); this.logger.log(`Completed entity delete: ${JSON.stringify({ entityType, deletedSince })}`); diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts index 85c4947d..4d64ce2a 100644 --- a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts @@ -1,9 +1,11 @@ import { Model, ModelCtor, ModelType } from "sequelize-typescript"; -import { cloneDeep, flatten, groupBy, isEmpty, isObject, isString, uniq } from "lodash"; +import { cloneDeep, flatten, groupBy, isEmpty, isObject, isString, keyBy, mapValues, merge, uniq } from "lodash"; import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; import Airtable from "airtable"; import { UuidModel } from "@terramatch-microservices/database/types/util"; import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; +import { DataApiService } from "@terramatch-microservices/data-api"; +import { Dictionary } from "factory-girl-ts"; // The Airtable API only supports bulk updates of up to 10 rows. const AIRTABLE_PAGE_SIZE = 10; @@ -23,6 +25,8 @@ export abstract class AirtableEntity, Associa protected readonly logger = new TMLogger(AirtableEntity.name); + constructor(protected dataApi: DataApiService) {} + /** * If an airtable entity provides a concrete type for Associations, this method should be overridden * to execute the necessary DB queries and provide a mapping of record number to concrete instance @@ -249,6 +253,23 @@ export abstract class AirtableEntity, Associa return associations; } + + protected _gadmCountryNames: Dictionary; + protected async gadmCountryNames() { + return (this._gadmCountryNames ??= mapValues(keyBy(await this.dataApi.gadmLevel0(), "iso"), "name")); + } + + protected _gadmStateNames: Dictionary> = {}; + protected async gadmStateNames(countryIsos: string[]) { + return merge( + {}, + ...(await Promise.all( + countryIsos.map(async iso => { + return (this._gadmStateNames[iso] ??= mapValues(keyBy(await this.dataApi.gadmLevel1(iso), "id"), "name")); + }) + )) + ); + } } export type Include = { diff --git a/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts index 00b3de15..d298d111 100644 --- a/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts @@ -1,8 +1,15 @@ -import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; import { Organisation } from "@terramatch-microservices/database/entities"; +import { filter, flatten, uniq } from "lodash"; -const COLUMNS: ColumnMapping[] = [ - ...commonEntityColumns("organisation"), +type OrganisationAssociations = { + hqCountryName: string; + countryNames: string[]; + stateNames: string[]; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("organisation"), "status", "type", "private", @@ -15,10 +22,12 @@ const COLUMNS: ColumnMapping[] = [ "hqState", "hqZipcode", "hqCountry", + associatedValueColumn("hqCountryName", "hqCountry"), "leadershipTeamTxt", "foundingDate", "description", "countries", + associatedValueColumn("countryNames", "countries"), "languages", "treeCareApproach", "relevantExperienceYears", @@ -51,6 +60,7 @@ const COLUMNS: ColumnMapping[] = [ "engagementYouth", "currency", "states", + associatedValueColumn("stateNames", "states"), "district", "accountNumber1", "accountNumber2", @@ -87,8 +97,28 @@ const COLUMNS: ColumnMapping[] = [ "additionalComments" ]; -export class OrganisationEntity extends AirtableEntity { +export class OrganisationEntity extends AirtableEntity { readonly TABLE_NAME = "Organisations"; readonly COLUMNS = COLUMNS; readonly MODEL = Organisation; + + async loadAssociations(organisations: Organisation[]) { + const countryNames = await this.gadmCountryNames(); + const stateCountries = filter( + uniq(flatten(organisations.map(({ states }) => states?.map(state => state.split(".")[0])))) + ); + const stateNames = await this.gadmStateNames(stateCountries); + + return organisations.reduce( + (associations, { id, hqCountry, countries, states }) => ({ + ...associations, + [id]: { + hqCountryName: hqCountry == null ? null : countryNames[hqCountry], + countryNames: filter(countries?.map(country => countryNames[country])), + stateNames: filter(states?.map(state => stateNames[state])) + } + }), + {} as Record + ); + } } diff --git a/apps/unified-database-service/src/airtable/entities/project-pitch.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/project-pitch.airtable-entity.ts index 4fdef71d..857ba34d 100644 --- a/apps/unified-database-service/src/airtable/entities/project-pitch.airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/project-pitch.airtable-entity.ts @@ -1,8 +1,14 @@ -import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; import { ProjectPitch } from "@terramatch-microservices/database/entities"; +import { filter, flatten, uniq } from "lodash"; -const COLUMNS: ColumnMapping[] = [ - ...commonEntityColumns("pitch"), +type ProjectPitchAssociations = { + projectCountryName: string; + stateNames: string[]; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("pitch"), "totalTrees", "totalHectares", "restorationInterventionTypes", @@ -10,6 +16,9 @@ const COLUMNS: ColumnMapping[] = [ "restorationStrategy", "projectObjectives", "projectCountry", + associatedValueColumn("projectCountryName", "projectCountry"), + "states", + associatedValueColumn("stateNames", "states"), "projectName", "projectBudget", "status", @@ -43,8 +52,27 @@ const COLUMNS: ColumnMapping[] = [ "goalTreesRestoredDirectSeeding" ]; -export class ProjectPitchEntity extends AirtableEntity { +export class ProjectPitchEntity extends AirtableEntity { readonly TABLE_NAME = "Project Pitches"; readonly COLUMNS = COLUMNS; readonly MODEL = ProjectPitch; + + async loadAssociations(pitches: ProjectPitch[]) { + const countryNames = await this.gadmCountryNames(); + const stateCountries = filter( + uniq(flatten(pitches.map(({ states }) => states?.map(state => state.split(".")[0])))) + ); + const stateNames = await this.gadmStateNames(stateCountries); + + return pitches.reduce( + (associations, { id, projectCountry, states }) => ({ + ...associations, + [id]: { + projectCountryName: projectCountry == null ? null : countryNames[projectCountry], + stateNames: filter(states?.map(state => stateNames[state])) + } + }), + {} as Record + ); + } } diff --git a/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts index 7fdb2013..b9b4b1eb 100644 --- a/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts @@ -6,8 +6,8 @@ import { Site, SitePolygon } from "@terramatch-microservices/database/entities"; -import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity"; -import { flatten, groupBy } from "lodash"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { filter, flatten, groupBy, uniq } from "lodash"; const loadApprovedSites = async (projectIds: number[]) => groupBy( @@ -29,6 +29,8 @@ const loadSitePolygons = async (siteUuids: string[]) => type ProjectAssociations = { sitePolygons: SitePolygon[]; + countryName: string; + stateNames: string[]; }; const COLUMNS: ColumnMapping[] = [ @@ -53,6 +55,7 @@ const COLUMNS: ColumnMapping[] = [ }, "status", "country", + associatedValueColumn("countryName", "country"), "description", "plantingStartDate", "plantingEndDate", @@ -90,6 +93,7 @@ const COLUMNS: ColumnMapping[] = [ "descriptionOfProjectTimeline", "landholderCommEngage", "states", + associatedValueColumn("stateNames", "states"), "detailedInterventionTypes", "waterSource", "baselineBiodiversity", @@ -112,19 +116,25 @@ export class ProjectEntity extends AirtableEntity const approvedSites = await loadApprovedSites(projectIds); const allSiteUuids = flatten(Object.values(approvedSites).map(sites => sites.map(({ uuid }) => uuid))); const sitePolygons = await loadSitePolygons(allSiteUuids); + const countryNames = await this.gadmCountryNames(); + const stateCountries = filter( + uniq(flatten(projects.map(({ states }) => states?.map(state => state.split(".")[0])))) + ); + const stateNames = await this.gadmStateNames(stateCountries); - return projectIds.reduce((associations, projectId) => { - const sites = approvedSites[projectId] ?? []; - - return { + return projects.reduce( + (associations, { id, country, states }) => ({ ...associations, - [projectId]: { - sitePolygons: sites.reduce( + [id]: { + sitePolygons: (approvedSites[id] ?? []).reduce( (polygons, { uuid }) => [...polygons, ...(sitePolygons[uuid] ?? [])], [] as SitePolygon[] - ) + ), + countryName: country == null ? null : countryNames[country], + stateNames: filter(states?.map(state => stateNames[state])) } - }; - }, {} as Record); + }), + {} as Record + ); } } diff --git a/libs/data-api/src/index.ts b/libs/data-api/src/index.ts index dd65e99f..55c54120 100644 --- a/libs/data-api/src/index.ts +++ b/libs/data-api/src/index.ts @@ -1 +1,3 @@ export * from "./lib/data-api.module"; + +export { DataApiService } from "./lib/data-api.service"; diff --git a/libs/data-api/src/lib/data-api.service.ts b/libs/data-api/src/lib/data-api.service.ts index 1439d5a4..9afa98e1 100644 --- a/libs/data-api/src/lib/data-api.service.ts +++ b/libs/data-api/src/lib/data-api.service.ts @@ -24,15 +24,24 @@ const gadmLevel1 = (level0: string) => ` AND gid_0 = '${level0}' `; -const gadmLevel2 = (level0: string, level1: string) => ` +const gadmLevel2 = (level1: string) => ` SELECT gid_2 as id, name_2 as name FROM gadm_administrative_boundaries - WHERE gid_0 = '${level0}' - AND gid_1 = '${level1}' + WHERE gid_1 = '${level1}' AND adm_level='2' AND type_2 NOT IN ('Waterbody', 'Water body', 'Water Body') `; +type GadmCountry = { + name: string; + iso: string; +}; + +type GadmLevelCode = { + name: string; + id: string; +}; + /** * A service for accessing, and in some cases caching, data from the Data API. It's not super * clear how this service will expand in the future, so for now it's not trying to be too @@ -44,21 +53,16 @@ export class DataApiService { constructor(@InjectRedis() private readonly redis: Redis, private readonly configService: ConfigService) {} - async gadmLevel0() { + async gadmLevel0(): Promise { return await this.getDataset("gadm-level-0", GADM_QUERY, gadmLevel0(), GADM_CACHE_DURATION); } - async gadmLevel1(level0: string) { + async gadmLevel1(level0: string): Promise { return await this.getDataset(`gadm-level-1:${level0}`, GADM_QUERY, gadmLevel1(level0), GADM_CACHE_DURATION); } - async gadmLevel2(level0: string, level1: string) { - return await this.getDataset( - `gadm-level-2:${level0}:${level1}`, - GADM_QUERY, - gadmLevel2(level0, level1), - GADM_CACHE_DURATION - ); + async gadmLevel2(level1: string): Promise { + return await this.getDataset(`gadm-level-2:${level1}`, GADM_QUERY, gadmLevel2(level1), GADM_CACHE_DURATION); } private async getDataset(key: string, queryPath: string, query: string, cacheDuration: number) { From 60a1a1cd1b911c36dfb11afc87a929640159957f Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 16:37:31 -0600 Subject: [PATCH 85/98] [TM-1959] refactor: remove unused entity references in entities.ts --- libs/database/src/lib/constants/entities.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 6dab9839..4c890478 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -24,8 +24,6 @@ export const ENTITY_MODELS: { [E in EntityType]: EntityClass } = { projects: Project, sites: Site, nurseries: Nursery - // disturbances: Disturbance, - // stratas: Strata::class, }; export const isReport = (entity: EntityModel): entity is ReportModel => @@ -40,8 +38,6 @@ export const isReport = (entity: EntityModel): entity is ReportModel => */ export async function getProjectId(entity: EntityModel) { if (entity instanceof Project) return entity.id; - if (entity instanceof Disturbance) return entity.id; - if (entity instanceof Invasive) return entity.id; if (entity instanceof Site || entity instanceof Nursery || entity instanceof ProjectReport) return entity.projectId; const parentClass: ModelCtor = entity instanceof SiteReport ? Site : Nursery; From df95a317e3ac6b17996cb937a3486b01b25cfdc1 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Fri, 25 Apr 2025 16:52:29 -0600 Subject: [PATCH 86/98] [TM-1959] refactor: remove unused Invasive entity import in entities.ts --- libs/database/src/lib/constants/entities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/database/src/lib/constants/entities.ts b/libs/database/src/lib/constants/entities.ts index 4c890478..97f92b30 100644 --- a/libs/database/src/lib/constants/entities.ts +++ b/libs/database/src/lib/constants/entities.ts @@ -1,4 +1,4 @@ -import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport, Disturbance, Invasive } from "../entities"; +import { Nursery, NurseryReport, Project, ProjectReport, Site, SiteReport } from "../entities"; import { ModelCtor } from "sequelize-typescript"; import { ModelStatic } from "sequelize"; import { kebabCase } from "lodash"; From 1e02ccbea9284481cf67620fd6fcbbcf850b87ec Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Apr 2025 16:46:43 -0700 Subject: [PATCH 87/98] [TM-1958] Update coverage for the unified database service. --- .../src/airtable/airtable.processor.spec.ts | 4 +- .../src/airtable/airtable.service.spec.ts | 5 + .../airtable/entities/airtable-entity.spec.ts | 87 +++++++--- .../src/lib/factories/organisation.factory.ts | 20 ++- .../lib/factories/project-pitch.factory.ts | 10 +- .../src/lib/factories/project.factory.ts | 24 +-- libs/database/src/lib/util/gadm-mock-data.ts | 151 ++++++++++++++++++ 7 files changed, 266 insertions(+), 35 deletions(-) create mode 100644 libs/database/src/lib/util/gadm-mock-data.ts diff --git a/apps/unified-database-service/src/airtable/airtable.processor.spec.ts b/apps/unified-database-service/src/airtable/airtable.processor.spec.ts index 7bb29e06..c6814624 100644 --- a/apps/unified-database-service/src/airtable/airtable.processor.spec.ts +++ b/apps/unified-database-service/src/airtable/airtable.processor.spec.ts @@ -5,6 +5,7 @@ import { createMock } from "@golevelup/ts-jest"; import { InternalServerErrorException, NotImplementedException } from "@nestjs/common"; import { Job } from "bullmq"; import { SlackService } from "@terramatch-microservices/common/slack/slack.service"; +import { DataApiService } from "@terramatch-microservices/data-api"; jest.mock("airtable", () => jest.fn(() => ({ @@ -20,7 +21,8 @@ describe("AirtableProcessor", () => { providers: [ AirtableProcessor, { provide: ConfigService, useValue: createMock() }, - { provide: SlackService, useValue: createMock() } + { provide: SlackService, useValue: createMock() }, + { provide: DataApiService, useValue: createMock() } ] }).compile(); diff --git a/apps/unified-database-service/src/airtable/airtable.service.spec.ts b/apps/unified-database-service/src/airtable/airtable.service.spec.ts index 46248155..3cca68f0 100644 --- a/apps/unified-database-service/src/airtable/airtable.service.spec.ts +++ b/apps/unified-database-service/src/airtable/airtable.service.spec.ts @@ -3,6 +3,7 @@ import { AirtableService } from "./airtable.service"; import { Queue } from "bullmq"; import { Test } from "@nestjs/testing"; import { getQueueToken } from "@nestjs/bullmq"; +import { DataApiService } from "@terramatch-microservices/data-api"; describe("AirtableService", () => { let service: AirtableService; @@ -15,6 +16,10 @@ describe("AirtableService", () => { { provide: getQueueToken("airtable"), useValue: (queue = createMock()) + }, + { + provide: DataApiService, + useValue: createMock() } ] }).compile(); diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts index 3726c19c..04e92a7e 100644 --- a/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts @@ -5,6 +5,7 @@ import { Demographic, DemographicEntry, Framework, + FundingProgramme, Nursery, NurseryReport, Organisation, @@ -39,10 +40,12 @@ import { ApplicationEntity, DemographicEntity, DemographicEntryEntity, + FundingProgrammeEntity, NurseryEntity, NurseryReportEntity, OrganisationEntity, ProjectEntity, + ProjectPitchEntity, ProjectReportEntity, SiteEntity, SiteReportEntity, @@ -53,6 +56,14 @@ import { Model } from "sequelize-typescript"; import { FrameworkKey } from "@terramatch-microservices/database/constants/framework"; import { FindOptions, Op } from "sequelize"; import { DateTime } from "luxon"; +import { DataApiService } from "@terramatch-microservices/data-api"; +import { createMock } from "@golevelup/ts-jest"; +import { + COUNTRIES, + gadmLevel0Mock, + gadmLevel1Mock, + STATES +} from "@terramatch-microservices/database/util/gadm-mock-data"; const airtableUpdate = jest.fn, [{ fields: object }[], object]>(() => Promise.resolve()); const airtableSelectFirstPage = jest.fn, never>(() => Promise.resolve([])); @@ -64,6 +75,8 @@ const Base = jest.fn(() => ({ destroy: airtableDestroy })) as unknown as Airtable.Base; +const dataApi = createMock({ gadmLevel0: gadmLevel0Mock, gadmLevel1: gadmLevel1Mock }); + const mapEntityColumns = jest.fn(() => Promise.resolve({})); export class StubEntity extends AirtableEntity { readonly TABLE_NAME = "stubs"; @@ -121,18 +134,18 @@ describe("AirtableEntity", () => { it("re-raises mapping errors", async () => { mapEntityColumns.mockRejectedValue(new Error("mapping error")); - await expect(new StubEntity().updateBase(null, { startPage: 0 })).rejects.toThrow("mapping error"); + await expect(new StubEntity(dataApi).updateBase(null, { startPage: 0 })).rejects.toThrow("mapping error"); mapEntityColumns.mockReset(); }); it("re-raises airtable errors", async () => { airtableUpdate.mockRejectedValue(new Error("airtable error")); - await expect(new StubEntity().updateBase(Base)).rejects.toThrow("airtable error"); + await expect(new StubEntity(dataApi).updateBase(Base)).rejects.toThrow("airtable error"); airtableUpdate.mockReset(); }); it("includes the updatedSince timestamp in the query", async () => { - const entity = new StubEntity(); + const entity = new StubEntity(dataApi); const spy = jest.spyOn(entity as never, "getUpdatePageFindOptions") as jest.SpyInstance>; const updatedSince = new Date(); await entity.updateBase(Base, { updatedSince }); @@ -143,7 +156,7 @@ describe("AirtableEntity", () => { }); it("skips the updatedSince timestamp if the model doesn't support it", async () => { - const entity = new StubEntity(); + const entity = new StubEntity(dataApi); // @ts-expect-error overriding readonly property for test. (entity as never).SUPPORTS_UPDATED_SINCE = false; const spy = jest.spyOn(entity as never, "getUpdatePageFindOptions") as jest.SpyInstance>; @@ -180,13 +193,13 @@ describe("AirtableEntity", () => { it("re-raises search errors", async () => { airtableSelectFirstPage.mockRejectedValue(new Error("select error")); - await expect(new SiteEntity().deleteStaleRecords(Base, deletedSince)).rejects.toThrow("select error"); + await expect(new SiteEntity(dataApi).deleteStaleRecords(Base, deletedSince)).rejects.toThrow("select error"); }); it("re-raises delete errors", async () => { airtableSelectFirstPage.mockResolvedValue([{ id: "fakeid", fields: { uuid: "fakeuuid" } }]); airtableDestroy.mockRejectedValue(new Error("delete error")); - await expect(new SiteEntity().deleteStaleRecords(Base, deletedSince)).rejects.toThrow("delete error"); + await expect(new SiteEntity(dataApi).deleteStaleRecords(Base, deletedSince)).rejects.toThrow("delete error"); airtableDestroy.mockReset(); }); @@ -196,7 +209,7 @@ describe("AirtableEntity", () => { fields: { uuid } })); airtableSelectFirstPage.mockResolvedValue(searchResult); - await new SiteEntity().deleteStaleRecords(Base, deletedSince); + await new SiteEntity(dataApi).deleteStaleRecords(Base, deletedSince); expect(airtableSelect).toHaveBeenCalledTimes(1); expect(airtableSelect).toHaveBeenCalledWith( expect.objectContaining({ @@ -254,7 +267,7 @@ describe("AirtableEntity", () => { it("sends all records to airtable", async () => { await testAirtableUpdates( - new ApplicationEntity(), + new ApplicationEntity(dataApi), applications, ({ uuid, organisationUuid, fundingProgrammeUuid }) => ({ fields: { @@ -327,7 +340,7 @@ describe("AirtableEntity", () => { it("sends all records to airtable", async () => { await testAirtableUpdates( - new DemographicEntity(), + new DemographicEntity(dataApi), demographics, ({ uuid, collection, demographicalType, demographicalId }) => ({ fields: { @@ -385,7 +398,7 @@ describe("AirtableEntity", () => { it("sends all records to airtable", async () => { await testAirtableUpdates( - new DemographicEntryEntity(), + new DemographicEntryEntity(dataApi), entries, ({ id, type, subtype, name, amount, demographicId }) => ({ fields: { @@ -401,6 +414,21 @@ describe("AirtableEntity", () => { }); }); + describe("FundingProgrammeEntity", () => { + let fundingProgrammes: FundingProgramme[]; + + beforeAll(async () => { + await FundingProgramme.truncate(); + fundingProgrammes = await FundingProgrammeFactory.createMany(2); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new FundingProgrammeEntity(dataApi), fundingProgrammes, ({ uuid, name }) => ({ + fields: { uuid, name } + })); + }); + }); + describe("NurseryEntity", () => { let projectUuids: Record; let nurseries: Nursery[]; @@ -422,7 +450,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new NurseryEntity(), nurseries, ({ uuid, name, projectId, status }) => ({ + await testAirtableUpdates(new NurseryEntity(dataApi), nurseries, ({ uuid, name, projectId, status }) => ({ fields: { uuid, name, @@ -454,7 +482,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new NurseryReportEntity(), reports, ({ uuid, nurseryId, status, dueAt }) => ({ + await testAirtableUpdates(new NurseryReportEntity(dataApi), reports, ({ uuid, nurseryId, status, dueAt }) => ({ fields: { uuid, nurseryUuid: nurseryUuids[nurseryId], @@ -479,7 +507,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new OrganisationEntity(), organisations, ({ uuid, name, status }) => ({ + await testAirtableUpdates(new OrganisationEntity(dataApi), organisations, ({ uuid, name, status }) => ({ fields: { uuid, name, @@ -574,7 +602,7 @@ describe("AirtableEntity", () => { it("sends all records to airtable", async () => { await testAirtableUpdates( - new ProjectEntity(), + new ProjectEntity(dataApi), projects, ({ uuid, name, frameworkKey, organisationId, applicationId }) => ({ fields: { @@ -590,6 +618,27 @@ describe("AirtableEntity", () => { }); }); + describe("ProjectPitchEntity", () => { + let pitches: ProjectPitch[]; + + beforeAll(async () => { + await ProjectPitch.truncate(); + pitches = await ProjectPitchFactory.createMany(3); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new ProjectPitchEntity(dataApi), pitches, ({ uuid, projectCountry, states }) => ({ + fields: { + uuid, + projectCountry, + projectCountryName: COUNTRIES[projectCountry], + states, + stateNames: states.map(state => STATES[state.split(".")[0]][state]) + } + })); + }); + }); + describe("ProjectReportEntity", () => { let projectUuids: Record; let reports: ProjectReport[]; @@ -654,7 +703,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new ProjectReportEntity(), reports, ({ uuid, projectId, status, dueAt }) => ({ + await testAirtableUpdates(new ProjectReportEntity(dataApi), reports, ({ uuid, projectId, status, dueAt }) => ({ fields: { uuid, projectUuid: projectUuids[projectId], @@ -684,7 +733,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new SiteEntity(), sites, ({ uuid, name, projectId, status }) => ({ + await testAirtableUpdates(new SiteEntity(dataApi), sites, ({ uuid, name, projectId, status }) => ({ fields: { uuid, name, @@ -720,7 +769,7 @@ describe("AirtableEntity", () => { }); it("sends all records to airtable", async () => { - await testAirtableUpdates(new SiteReportEntity(), reports, ({ id, uuid, siteId, status, dueAt }) => ({ + await testAirtableUpdates(new SiteReportEntity(dataApi), reports, ({ id, uuid, siteId, status, dueAt }) => ({ fields: { uuid, siteUuid: siteUuids[siteId], @@ -796,7 +845,7 @@ describe("AirtableEntity", () => { it("sends all records to airtable", async () => { await testAirtableUpdates( - new TreeSpeciesEntity(), + new TreeSpeciesEntity(dataApi), trees, ({ uuid, name, amount, collection, speciesableType, speciesableId }) => ({ fields: { @@ -826,7 +875,7 @@ describe("AirtableEntity", () => { super.getDeletePageFindOptions(deletedSince, page); } const deletedSince = new Date(); - const result = new Test().getDeletePageFindOptions(deletedSince, 0); + const result = new Test(dataApi).getDeletePageFindOptions(deletedSince, 0); expect(result.where[Op.or]).not.toBeNull(); expect(result.where[Op.or]?.[Op.and]?.updatedAt?.[Op.gte]).toBe(deletedSince); }); diff --git a/libs/database/src/lib/factories/organisation.factory.ts b/libs/database/src/lib/factories/organisation.factory.ts index b9dfdc6b..103ee4b7 100644 --- a/libs/database/src/lib/factories/organisation.factory.ts +++ b/libs/database/src/lib/factories/organisation.factory.ts @@ -2,9 +2,19 @@ import { Organisation } from "../entities"; import { FactoryGirl } from "factory-girl-ts"; import { faker } from "@faker-js/faker"; import { ORGANISATION_STATUSES } from "../constants/status"; +import { fakerCountries, fakerStates } from "../util/gadm-mock-data"; -export const OrganisationFactory = FactoryGirl.define(Organisation, async () => ({ - status: faker.helpers.arrayElement(ORGANISATION_STATUSES), - type: "non-profit-organisation", - name: faker.company.name() -})); +export const OrganisationFactory = FactoryGirl.define(Organisation, async () => { + const countries = fakerCountries(2); + const hqCountry = faker.helpers.arrayElement(countries); + const states = fakerStates(countries, 5); + + return { + status: faker.helpers.arrayElement(ORGANISATION_STATUSES), + type: "non-profit-organisation", + name: faker.company.name(), + hqCountry, + countries, + states + }; +}); diff --git a/libs/database/src/lib/factories/project-pitch.factory.ts b/libs/database/src/lib/factories/project-pitch.factory.ts index f8c12dc0..71f0806b 100644 --- a/libs/database/src/lib/factories/project-pitch.factory.ts +++ b/libs/database/src/lib/factories/project-pitch.factory.ts @@ -1,4 +1,12 @@ import { FactoryGirl } from "factory-girl-ts"; import { ProjectPitch } from "../entities"; +import { fakerCountries, fakerStates } from "../util/gadm-mock-data"; -export const ProjectPitchFactory = FactoryGirl.define(ProjectPitch, async () => ({})); +export const ProjectPitchFactory = FactoryGirl.define(ProjectPitch, async () => { + const projectCountry = fakerCountries()[0]; + + return { + projectCountry, + states: fakerStates([projectCountry], 3) + }; +}); diff --git a/libs/database/src/lib/factories/project.factory.ts b/libs/database/src/lib/factories/project.factory.ts index 094bc8a7..4bf70e09 100644 --- a/libs/database/src/lib/factories/project.factory.ts +++ b/libs/database/src/lib/factories/project.factory.ts @@ -5,15 +5,21 @@ import { UPDATE_REQUEST_STATUSES } from "../constants/status"; import { ApplicationFactory } from "./application.factory"; import { OrganisationFactory } from "./organisation.factory"; import { FRAMEWORK_KEYS } from "../constants/framework"; +import { fakerCountries, fakerStates } from "../util/gadm-mock-data"; const CONTINENTS = ["africa", "australia", "south-america", "asia", "north-america"]; -export const ProjectFactory = FactoryGirl.define(Project, async () => ({ - name: faker.animal.petName(), - frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), - updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), - applicationId: ApplicationFactory.associate("id"), - organisationId: OrganisationFactory.associate("id"), - continent: faker.helpers.arrayElement(CONTINENTS), - survivalRate: faker.number.int({ min: 20, max: 80 }) -})); +export const ProjectFactory = FactoryGirl.define(Project, async () => { + const country = fakerCountries()[0]; + return { + name: faker.animal.petName(), + frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), + applicationId: ApplicationFactory.associate("id"), + organisationId: OrganisationFactory.associate("id"), + continent: faker.helpers.arrayElement(CONTINENTS), + country, + states: fakerStates([country], 3), + survivalRate: faker.number.int({ min: 20, max: 80 }) + }; +}); diff --git a/libs/database/src/lib/util/gadm-mock-data.ts b/libs/database/src/lib/util/gadm-mock-data.ts new file mode 100644 index 00000000..2ac85df3 --- /dev/null +++ b/libs/database/src/lib/util/gadm-mock-data.ts @@ -0,0 +1,151 @@ +import { faker } from "@faker-js/faker"; +import { merge } from "lodash"; + +export const COUNTRIES = { + ESP: "Spain", + USA: "United States", + MEX: "México", + CHL: "Chile" +} as const; + +type Country = keyof typeof COUNTRIES; + +export const fakerCountries = (num = 1) => faker.helpers.uniqueArray(Object.keys(COUNTRIES), num) as Country[]; + +export const gadmLevel0Mock = async () => Object.entries(COUNTRIES).map(([iso, name]) => ({ iso, name })); + +export const STATES = { + ESP: { + "ESP.4_1": "Castilla-La Mancha", + "ESP.5_1": "Castilla y León", + "ESP.6_1": "Cataluña", + "ESP.12_1": "Galicia", + "ESP.1_1": "Andalucía", + "ESP.2_1": "Aragón", + "ESP.3_1": "Cantabria", + "ESP.7_1": "Ceuta y Melilla", + "ESP.8_1": "Comunidad de Madrid", + "ESP.9_1": "Comunidad Foral de Navarra", + "ESP.10_1": "Comunidad Valenciana", + "ESP.11_1": "Extremadura", + "ESP.13_1": "Islas Baleares", + "ESP.14_1": "Islas Canarias", + "ESP.15_1": "La Rioja", + "ESP.16_1": "País Vasco", + "ESP.17_1": "Principado de Asturias", + "ESP.18_1": "Región de Murcia" + }, + USA: { + "USA.1_1": "Alabama", + "USA.2_1": "Alaska", + "USA.41_1": "South Carolina", + "USA.42_1": "South Dakota", + "USA.43_1": "Tennessee", + "USA.44_1": "Texas", + "USA.45_1": "Utah", + "USA.46_1": "Vermont", + "USA.47_1": "Virginia", + "USA.48_1": "Washington", + "USA.49_1": "West Virginia", + "USA.50_1": "Wisconsin", + "USA.36_1": "Ohio", + "USA.3_1": "Arizona", + "USA.4_1": "Arkansas", + "USA.5_1": "California", + "USA.9_1": "District of Columbia", + "USA.10_1": "Florida", + "USA.11_1": "Georgia", + "USA.12_1": "Hawaii", + "USA.13_1": "Idaho", + "USA.14_1": "Illinois", + "USA.15_1": "Indiana", + "USA.16_1": "Iowa", + "USA.17_1": "Kansas", + "USA.18_1": "Kentucky", + "USA.19_1": "Louisiana", + "USA.20_1": "Maine", + "USA.21_1": "Maryland", + "USA.22_1": "Massachusetts", + "USA.23_1": "Michigan", + "USA.24_1": "Minnesota", + "USA.25_1": "Mississippi", + "USA.26_1": "Missouri", + "USA.27_1": "Montana", + "USA.28_1": "Nebraska", + "USA.29_1": "Nevada", + "USA.30_1": "New Hampshire", + "USA.31_1": "New Jersey", + "USA.32_1": "New Mexico", + "USA.33_1": "New York", + "USA.34_1": "North Carolina", + "USA.35_1": "North Dakota", + "USA.37_1": "Oklahoma", + "USA.38_1": "Oregon", + "USA.39_1": "Pennsylvania", + "USA.40_1": "Rhode Island", + "USA.6_1": "Colorado", + "USA.7_1": "Connecticut", + "USA.8_1": "Delaware", + "USA.51_1": "Wyoming" + }, + MEX: { + "MEX.5_1": "Chiapas", + "MEX.26_1": "Sonora", + "MEX.27_1": "Tabasco", + "MEX.28_1": "Tamaulipas", + "MEX.29_1": "Tlaxcala", + "MEX.30_1": "Veracruz", + "MEX.31_1": "Yucatán", + "MEX.32_1": "Zacatecas", + "MEX.18_1": "Nayarit", + "MEX.19_1": "Nuevo León", + "MEX.20_1": "Oaxaca", + "MEX.21_1": "Puebla", + "MEX.22_1": "Querétaro", + "MEX.11_1": "Guanajuato", + "MEX.12_1": "Guerrero", + "MEX.13_1": "Hidalgo", + "MEX.14_1": "Jalisco", + "MEX.15_1": "México", + "MEX.16_1": "Michoacán", + "MEX.17_1": "Morelos", + "MEX.23_1": "Quintana Roo", + "MEX.24_1": "San Luis Potosí", + "MEX.25_1": "Sinaloa", + "MEX.4_1": "Campeche", + "MEX.6_1": "Chihuahua", + "MEX.7_1": "Coahuila", + "MEX.8_1": "Colima", + "MEX.9_1": "Distrito Federal", + "MEX.10_1": "Durango", + "MEX.1_1": "Aguascalientes", + "MEX.3_1": "Baja California", + "MEX.2_1": "Baja California Sur" + }, + CHL: { + "CHL.6_1": "Bío-Bío", + "CHL.7_1": "Coquimbo", + "CHL.8_1": "Libertador General Bernardo O'Hi", + "CHL.2_1": "Antofagasta", + "CHL.3_1": "Araucanía", + "CHL.4_1": "Arica y Parinacota", + "CHL.5_1": "Atacama", + "CHL.1_1": "Aysén del General Ibañez del Cam", + "CHL.9_1": "Los Lagos", + "CHL.10_1": "Los Ríos", + "CHL.11_1": "Magallanes y Antártica Chilena", + "CHL.12_1": "Maule", + "CHL.13_1": "Ñuble", + "CHL.14_1": "Santiago Metropolitan", + "CHL.15_1": "Tarapacá", + "CHL.16_1": "Valparaíso" + } +} as const; + +export const fakerStates = (countries: Country[], num = 1) => { + const options = Object.keys(merge({}, ...countries.map(country => STATES[country]))); + return faker.helpers.uniqueArray(options, num); +}; + +export const gadmLevel1Mock = async (level0: Country) => + Object.entries(STATES[level0]).map(([id, name]) => ({ id, name })); From b886d33ac9fbb28627168eabd4647ddee1723f2b Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Sat, 26 Apr 2025 19:00:19 -0600 Subject: [PATCH 88/98] [TM-1959] Add entities to controller response --- .../src/entities/entity-associations.controller.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/entity-service/src/entities/entity-associations.controller.ts b/apps/entity-service/src/entities/entity-associations.controller.ts index 3d541c3e..b9ab7694 100644 --- a/apps/entity-service/src/entities/entity-associations.controller.ts +++ b/apps/entity-service/src/entities/entity-associations.controller.ts @@ -8,6 +8,9 @@ import { DemographicCollections, DemographicDto, DemographicEntryDto } from "./d import { ExceptionResponse, JsonApiResponse } from "@terramatch-microservices/common/decorators"; import { SeedingDto } from "./dto/seeding.dto"; import { TreeSpeciesDto } from "./dto/tree-species.dto"; +import { DisturbanceDto } from "./dto/disturbance.dto"; +import { InvasiveDto } from "./dto/invasive.dto"; +import { StrataDto } from "./dto/strata.dto"; @Controller("entities/v3/:entity/:uuid") @ApiExtraModels(DemographicEntryDto, DemographicCollections) @@ -22,7 +25,10 @@ export class EntityAssociationsController { @JsonApiResponse([ { data: DemographicDto, hasMany: true }, { data: SeedingDto, hasMany: true }, - { data: TreeSpeciesDto, hasMany: true } + { data: TreeSpeciesDto, hasMany: true }, + { data: DisturbanceDto, hasMany: true }, + { data: InvasiveDto, hasMany: true }, + { data: StrataDto, hasMany: true } ]) @ExceptionResponse(BadRequestException, { description: "Param types invalid" }) @ExceptionResponse(NotFoundException, { description: "Base entity not found" }) From 93552a5a1d3f2b4d6bc789340f1d5ff8d073c486 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 28 Apr 2025 11:49:41 -0700 Subject: [PATCH 89/98] [TM-1958] Data API specs --- libs/data-api/.eslintrc.json | 18 ++++ libs/data-api/jest.config.ts | 11 +++ .../data-api/src/lib/data-api.service.spec.ts | 90 +++++++++++++++++++ libs/data-api/src/lib/data-api.service.ts | 7 +- libs/data-api/tsconfig.json | 9 +- libs/data-api/tsconfig.lib.json | 3 +- libs/data-api/tsconfig.spec.json | 9 ++ package-lock.json | 26 ++++++ package.json | 1 + 9 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 libs/data-api/.eslintrc.json create mode 100644 libs/data-api/jest.config.ts create mode 100644 libs/data-api/src/lib/data-api.service.spec.ts create mode 100644 libs/data-api/tsconfig.spec.json diff --git a/libs/data-api/.eslintrc.json b/libs/data-api/.eslintrc.json new file mode 100644 index 00000000..9d9c0db5 --- /dev/null +++ b/libs/data-api/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/data-api/jest.config.ts b/libs/data-api/jest.config.ts new file mode 100644 index 00000000..ea293553 --- /dev/null +++ b/libs/data-api/jest.config.ts @@ -0,0 +1,11 @@ +/* eslint-disable */ +export default { + displayName: "data-api", + preset: "../../jest.preset.js", + testEnvironment: "node", + transform: { + "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "/tsconfig.spec.json" }] + }, + moduleFileExtensions: ["ts", "js", "html"], + coverageDirectory: "../../coverage/libs/data-api" +}; diff --git a/libs/data-api/src/lib/data-api.service.spec.ts b/libs/data-api/src/lib/data-api.service.spec.ts new file mode 100644 index 00000000..4922246f --- /dev/null +++ b/libs/data-api/src/lib/data-api.service.spec.ts @@ -0,0 +1,90 @@ +import { Test } from "@nestjs/testing"; +import { createMock, DeepMocked, PartialFuncReturn } from "@golevelup/ts-jest"; +import { DataApiService, gadmLevel2 } from "./data-api.service"; +import { ConfigService } from "@nestjs/config"; +import Redis from "ioredis"; +import { getRedisConnectionToken } from "@nestjs-modules/ioredis"; +import fetchMock from "jest-fetch-mock"; +import { InternalServerErrorException } from "@nestjs/common"; + +describe("DataApiService", () => { + let service: DataApiService; + let redis: DeepMocked; + let config: DeepMocked; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + DataApiService, + { + provide: ConfigService, + useValue: (config = createMock({ + get: (key: string): PartialFuncReturn => { + if (key === "APP_FRONT_END") return "https://unittests.terramatch.org"; + if (key === "DATA_API_KEY") return "test-api-key"; + return ""; + } + })) + }, + { + provide: getRedisConnectionToken("default"), + useValue: (redis = createMock({ get: () => Promise.resolve(null) })) + } + ] + }).compile(); + + service = module.get(DataApiService); + fetchMock.enableMocks(); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + fetchMock.resetMocks(); + }); + + it("should check the redis cache first", async () => { + redis.get.mockResolvedValue(JSON.stringify({ cached: "foo" })); + const result = await service.gadmLevel0(); + expect(result).toEqual({ cached: "foo" }); + expect(redis.get).toHaveBeenCalledWith("data-api:gadm-level-0"); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("should throw if the environment is not configured correctly", async () => { + config.get.mockReturnValue(null); + await expect(service.gadmLevel0()).rejects.toThrow(InternalServerErrorException); + }); + + it("should throw an error if an error is returned from the data api", async () => { + fetchMock.mockResolvedValue({ status: 404 } as Response); + await expect(service.gadmLevel1("ESP")).rejects.toThrow(InternalServerErrorException); + }); + + it("should return and cache the dataset", async () => { + fetchMock.mockResolvedValue({ + status: 200, + ok: true, + json: () => Promise.resolve({ data: { foo: "mocked data" } }) + } as Response); + const result = await service.gadmLevel2("USA.CA"); + expect(result).toEqual({ foo: "mocked data" }); + + const params = new URLSearchParams(); + params.append("sql", gadmLevel2("USA.CA")); + expect(fetch).toHaveBeenCalledWith( + `https://data-api.globalforestwatch.org/dataset/gadm_administrative_boundaries/v4.1.85/query?${params}`, + expect.objectContaining({ + headers: { + Origin: "unittests.terramatch.org", + "x-api-key": "test-api-key" + } + }) + ); + expect(redis.set).toHaveBeenCalledWith( + "data-api:gadm-level-2:USA.CA", + JSON.stringify({ foo: "mocked data" }), + "EX", + 60 * 60 * 3 + ); + }); +}); diff --git a/libs/data-api/src/lib/data-api.service.ts b/libs/data-api/src/lib/data-api.service.ts index 9afa98e1..4646f521 100644 --- a/libs/data-api/src/lib/data-api.service.ts +++ b/libs/data-api/src/lib/data-api.service.ts @@ -24,7 +24,8 @@ const gadmLevel1 = (level0: string) => ` AND gid_0 = '${level0}' `; -const gadmLevel2 = (level1: string) => ` +// Exported for testing only +export const gadmLevel2 = (level1: string) => ` SELECT gid_2 as id, name_2 as name FROM gadm_administrative_boundaries WHERE gid_1 = '${level1}' @@ -81,7 +82,7 @@ export class DataApiService { const response = await fetch(`${DATA_API_DATASET}${queryPath}?${params}`, { headers: { Origin: new URL(appFrontend).hostname, - ["x-api-key"]: dataApiKey + "x-api-key": dataApiKey } }); @@ -89,7 +90,7 @@ export class DataApiService { throw new InternalServerErrorException(response.statusText); } - const json = (await response.json()) as { data: any }; + const json = (await response.json()) as { data: never }; await this.redis.set(`${KEY_NAMESPACE}${key}`, JSON.stringify(json.data), "EX", cacheDuration); return json.data; diff --git a/libs/data-api/tsconfig.json b/libs/data-api/tsconfig.json index e97cb764..84c538bf 100644 --- a/libs/data-api/tsconfig.json +++ b/libs/data-api/tsconfig.json @@ -4,17 +4,20 @@ "module": "commonjs", "forceConsistentCasingInFileNames": true, "strict": true, - "importHelpers": true, + "strictPropertyInitialization": false, "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "noPropertyAccessFromIndexSignature": true + "noFallthroughCasesInSwitch": true }, "files": [], "include": [], "references": [ { "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" } ] } diff --git a/libs/data-api/tsconfig.lib.json b/libs/data-api/tsconfig.lib.json index 0260985f..c297a248 100644 --- a/libs/data-api/tsconfig.lib.json +++ b/libs/data-api/tsconfig.lib.json @@ -11,5 +11,6 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] } diff --git a/libs/data-api/tsconfig.spec.json b/libs/data-api/tsconfig.spec.json new file mode 100644 index 00000000..f6d8ffcc --- /dev/null +++ b/libs/data-api/tsconfig.spec.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": ["jest.config.ts", "src/**/*.test.ts", "src/**/*.spec.ts", "src/**/*.d.ts"] +} diff --git a/package-lock.json b/package-lock.json index bcd28dcc..242be6ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,6 +88,7 @@ "husky": "^9.1.7", "jest": "^29.7.0", "jest-environment-node": "^29.7.0", + "jest-fetch-mock": "^3.0.3", "nx": "20.6.1", "prettier": "^2.6.2", "sqlite3": "^5.1.7", @@ -13176,6 +13177,25 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/jest-fetch-mock": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz", + "integrity": "sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==", + "dev": true, + "dependencies": { + "cross-fetch": "^3.0.4", + "promise-polyfill": "^8.1.3" + } + }, + "node_modules/jest-fetch-mock/node_modules/cross-fetch": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", + "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", + "dev": true, + "dependencies": { + "node-fetch": "^2.7.0" + } + }, "node_modules/jest-get-type": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", @@ -16561,6 +16581,12 @@ "dev": true, "optional": true }, + "node_modules/promise-polyfill": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", + "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==", + "dev": true + }, "node_modules/promise-retry": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", diff --git a/package.json b/package.json index fa7a8a00..738c55e8 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "husky": "^9.1.7", "jest": "^29.7.0", "jest-environment-node": "^29.7.0", + "jest-fetch-mock": "^3.0.3", "nx": "20.6.1", "prettier": "^2.6.2", "sqlite3": "^5.1.7", From a290e004caa9c3f8e335a1e2dc82dc5434f7573d Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Tue, 29 Apr 2025 07:06:50 -0600 Subject: [PATCH 90/98] [TM-1959] refactor: remove unused properties from disturbance and invasive DTOs and entities --- apps/entity-service/src/entities/dto/disturbance.dto.ts | 9 --------- apps/entity-service/src/entities/dto/invasive.dto.ts | 9 --------- apps/entity-service/src/entities/dto/strata.dto.ts | 3 --- apps/entity-service/src/entities/entities.service.ts | 5 +---- libs/database/src/lib/entities/disturbance.entity.ts | 8 -------- libs/database/src/lib/entities/invasive.entity.ts | 8 -------- libs/database/src/lib/entities/stratas.entity.ts | 2 +- 7 files changed, 2 insertions(+), 42 deletions(-) diff --git a/apps/entity-service/src/entities/dto/disturbance.dto.ts b/apps/entity-service/src/entities/dto/disturbance.dto.ts index c839aebd..e1c20e36 100644 --- a/apps/entity-service/src/entities/dto/disturbance.dto.ts +++ b/apps/entity-service/src/entities/dto/disturbance.dto.ts @@ -27,13 +27,4 @@ export class DisturbanceDto extends AssociationDto { @ApiProperty({ nullable: true }) description: string | null; - - @ApiProperty({ nullable: true }) - oldId: number; - - @ApiProperty({ nullable: true }) - oldModel: string | null; - - @ApiProperty({ nullable: true }) - hidden: number | null; } diff --git a/apps/entity-service/src/entities/dto/invasive.dto.ts b/apps/entity-service/src/entities/dto/invasive.dto.ts index 028fcb1f..6cde2a43 100644 --- a/apps/entity-service/src/entities/dto/invasive.dto.ts +++ b/apps/entity-service/src/entities/dto/invasive.dto.ts @@ -18,13 +18,4 @@ export class InvasiveDto extends AssociationDto { @ApiProperty({ nullable: true }) name: string | null; - - @ApiProperty({ nullable: true }) - oldId: number; - - @ApiProperty({ nullable: true }) - oldModel: string | null; - - @ApiProperty({ nullable: true }) - hidden: number | null; } diff --git a/apps/entity-service/src/entities/dto/strata.dto.ts b/apps/entity-service/src/entities/dto/strata.dto.ts index 0c7a058c..b1f2b530 100644 --- a/apps/entity-service/src/entities/dto/strata.dto.ts +++ b/apps/entity-service/src/entities/dto/strata.dto.ts @@ -22,7 +22,4 @@ export class StrataDto extends AssociationDto { @AllowNull @Column(INTEGER) extent: string | null; - - @ApiProperty({ nullable: true }) - hidden: number | null; } diff --git a/apps/entity-service/src/entities/entities.service.ts b/apps/entity-service/src/entities/entities.service.ts index 5b84bf4d..246e2ac9 100644 --- a/apps/entity-service/src/entities/entities.service.ts +++ b/apps/entity-service/src/entities/entities.service.ts @@ -119,10 +119,7 @@ export class EntitiesService { await this.policyService.authorize(action, subject); } - async isFrameworkAdmin(T: EntityModel) { - if (T instanceof Disturbance) return; - if (T instanceof Invasive) return; - const { frameworkKey } = T; + async isFrameworkAdmin({ frameworkKey }: T) { return (await this.getPermissions()).includes(`framework-${frameworkKey}`); } diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts index 853fd49f..ad1ae364 100644 --- a/libs/database/src/lib/entities/disturbance.entity.ts +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -40,14 +40,6 @@ export class Disturbance extends Model { @Column(TEXT) description: string | null; - @AllowNull - @Column(INTEGER.UNSIGNED) - oldId: number; - - @AllowNull - @Column(STRING) - oldModel: string | null; - @Column(TINYINT) hidden: number | null; } diff --git a/libs/database/src/lib/entities/invasive.entity.ts b/libs/database/src/lib/entities/invasive.entity.ts index 96618846..06f9aa91 100644 --- a/libs/database/src/lib/entities/invasive.entity.ts +++ b/libs/database/src/lib/entities/invasive.entity.ts @@ -32,14 +32,6 @@ export class Invasive extends Model { @Column(TEXT) name: string | null; - @AllowNull - @Column(INTEGER.UNSIGNED) - oldId: number; - - @AllowNull - @Column(STRING) - oldModel: string | null; - @Column(TINYINT) hidden: number | null; } diff --git a/libs/database/src/lib/entities/stratas.entity.ts b/libs/database/src/lib/entities/stratas.entity.ts index 2a3325d0..5a7219d8 100644 --- a/libs/database/src/lib/entities/stratas.entity.ts +++ b/libs/database/src/lib/entities/stratas.entity.ts @@ -14,7 +14,7 @@ export class Strata extends Model { @AllowNull @Column(INTEGER.UNSIGNED) - owner_id: number | null; + ownerId: number | null; @Column(STRING) stratasableType: string; From 0f210a872ccc8438647b1daf40cdc995a261d15b Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Tue, 29 Apr 2025 07:10:43 -0600 Subject: [PATCH 91/98] [TM-1959] refactor: remove unused INTEGER import from disturbance and invasive entities --- apps/entity-service/src/entities/dto/strata.dto.ts | 1 - libs/database/src/lib/entities/disturbance.entity.ts | 2 +- libs/database/src/lib/entities/invasive.entity.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/entity-service/src/entities/dto/strata.dto.ts b/apps/entity-service/src/entities/dto/strata.dto.ts index b1f2b530..a78121d7 100644 --- a/apps/entity-service/src/entities/dto/strata.dto.ts +++ b/apps/entity-service/src/entities/dto/strata.dto.ts @@ -1,6 +1,5 @@ import { JsonApiDto } from "@terramatch-microservices/common/decorators"; import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; -import { ApiProperty } from "@nestjs/swagger"; import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto"; import { Strata } from "@terramatch-microservices/database/entities/stratas.entity"; import { AllowNull, Column } from "sequelize-typescript"; diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts index ad1ae364..a0e9b9d1 100644 --- a/libs/database/src/lib/entities/disturbance.entity.ts +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { BIGINT, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_disturbances", underscored: true, paranoid: true }) export class Disturbance extends Model { diff --git a/libs/database/src/lib/entities/invasive.entity.ts b/libs/database/src/lib/entities/invasive.entity.ts index 06f9aa91..8a3ca9e6 100644 --- a/libs/database/src/lib/entities/invasive.entity.ts +++ b/libs/database/src/lib/entities/invasive.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { BIGINT, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_invasives", underscored: true, paranoid: true }) export class Invasive extends Model { From 9751f0a4add2d9d3c649f595efaf61f014b937c8 Mon Sep 17 00:00:00 2001 From: Ismael Martinez Date: Tue, 29 Apr 2025 10:36:28 -0600 Subject: [PATCH 92/98] [TM-1959] refactor: mark oldId and oldModel as deprecated in Disturbance and Invasive entities --- .../src/lib/entities/disturbance.entity.ts | 16 +++++++++++++++- .../database/src/lib/entities/invasive.entity.ts | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/libs/database/src/lib/entities/disturbance.entity.ts b/libs/database/src/lib/entities/disturbance.entity.ts index a0e9b9d1..fcee8ab8 100644 --- a/libs/database/src/lib/entities/disturbance.entity.ts +++ b/libs/database/src/lib/entities/disturbance.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_disturbances", underscored: true, paranoid: true }) export class Disturbance extends Model { @@ -40,6 +40,20 @@ export class Disturbance extends Model { @Column(TEXT) description: string | null; + /** + * @deprecated This property is no longer in use and will be removed in future versions. + */ + @AllowNull + @Column(INTEGER.UNSIGNED) + oldId: number; + + /** + * @deprecated This property is no longer in use and will be removed in future versions. + */ + @AllowNull + @Column(STRING) + oldModel: string | null; + @Column(TINYINT) hidden: number | null; } diff --git a/libs/database/src/lib/entities/invasive.entity.ts b/libs/database/src/lib/entities/invasive.entity.ts index 8a3ca9e6..aecd83eb 100644 --- a/libs/database/src/lib/entities/invasive.entity.ts +++ b/libs/database/src/lib/entities/invasive.entity.ts @@ -1,5 +1,5 @@ import { AllowNull, AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; +import { BIGINT, INTEGER, STRING, TEXT, TINYINT, UUID, UUIDV4 } from "sequelize"; @Table({ tableName: "v2_invasives", underscored: true, paranoid: true }) export class Invasive extends Model { @@ -32,6 +32,20 @@ export class Invasive extends Model { @Column(TEXT) name: string | null; + /** + * @deprecated This property is no longer in use and will be removed in future versions. + */ + @AllowNull + @Column(INTEGER.UNSIGNED) + oldId: number; + + /** + * @deprecated This property is no longer in use and will be removed in future versions. + */ + @AllowNull + @Column(STRING) + oldModel: string | null; + @Column(TINYINT) hidden: number | null; } From 2fb4a3b02b24bc697784d19af002f80825fc5d27 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 30 Apr 2025 15:26:33 -0700 Subject: [PATCH 93/98] Add TLS strings to the redis config. --- libs/data-api/src/lib/data-api.module.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libs/data-api/src/lib/data-api.module.ts b/libs/data-api/src/lib/data-api.module.ts index 463f7ebd..f85225b3 100644 --- a/libs/data-api/src/lib/data-api.module.ts +++ b/libs/data-api/src/lib/data-api.module.ts @@ -9,10 +9,13 @@ import { RedisModule } from "@nestjs-modules/ioredis"; RedisModule.forRootAsync({ imports: [ConfigModule.forRoot({ isGlobal: true })], inject: [ConfigService], - useFactory: (configService: ConfigService) => ({ - type: "single", - url: `redis://${configService.get("REDIS_HOST")}:${configService.get("REDIS_PORT")}` - }) + useFactory: (configService: ConfigService) => { + const protocol = process.env["NODE_ENV"] === "development" ? "redis://" : "rediss://"; + return { + type: "single", + url: `${protocol}${configService.get("REDIS_HOST")}:${configService.get("REDIS_PORT")}` + }; + } }) ], providers: [DataApiService], From 879d2e6ff16bc19d7c56e2aba87bc767712ff972 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 30 Apr 2025 16:58:06 -0700 Subject: [PATCH 94/98] [TM-1931] Modify configurations for cpu / memory based on AWS recommendations. --- cdk/service-stack/lib/service-stack.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cdk/service-stack/lib/service-stack.ts b/cdk/service-stack/lib/service-stack.ts index 757327a0..f3e6bc32 100644 --- a/cdk/service-stack/lib/service-stack.ts +++ b/cdk/service-stack/lib/service-stack.ts @@ -10,6 +10,7 @@ import { } from "aws-cdk-lib/aws-ecs-patterns"; import { Role } from "aws-cdk-lib/aws-iam"; import { upperFirst } from "lodash"; +import { Dictionary } from "factory-girl-ts"; const extractFromEnv = (...names: string[]) => names.map(name => { @@ -22,11 +23,20 @@ type Mutable = { -readonly [P in keyof T]: T[P]; }; +// Recommendations for optimal pricing from AWs +const RIGHTSIZE_RECOMMENDATIONS: Dictionary> = { + "research-service": { + prod: { + cpu: 1024, + memoryLimitMiB: 2048 + } + } +}; + const customizeFargate = (service: string, env: string, props: Mutable) => { - if (service === "research-service" && ["prod", "staging"].includes(env)) { - // Beef up the research service in staging and prod - props.cpu = 2048; - props.memoryLimitMiB = 4096; + const recommendation = RIGHTSIZE_RECOMMENDATIONS[service]?.[env]; + if (recommendation != null) { + return { ...props, ...recommendation }; } return props; @@ -82,7 +92,9 @@ export class ServiceStack extends Stack { customizeFargate(service, env, { serviceName: `terramatch-${service}-${env}`, cluster, - cpu: 512, + // These are the recommended defaults by Amazon Rightsize in the billing console + cpu: 256, + memoryLimitMiB: 512, desiredCount: 1, enableExecuteCommand: true, taskImageOptions: { @@ -97,7 +109,6 @@ export class ServiceStack extends Stack { }, securityGroups: securityGroups, taskSubnets: { subnets: privateSubnets }, - memoryLimitMiB: 2048, assignPublicIp: false, publicLoadBalancer: false, loadBalancerName: `${service}-${env}` From 305ed3193906fdb6201c813d113b8590d2e50544 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 30 Apr 2025 17:10:36 -0700 Subject: [PATCH 95/98] [TM-1931] Audit fix --- cdk/service-stack/package-lock.json | 171 +++++++--------------------- 1 file changed, 38 insertions(+), 133 deletions(-) diff --git a/cdk/service-stack/package-lock.json b/cdk/service-stack/package-lock.json index 1c990253..973800c7 100644 --- a/cdk/service-stack/package-lock.json +++ b/cdk/service-stack/package-lock.json @@ -88,13 +88,14 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.25.7.tgz", - "integrity": "sha512-0xZJFNE5XMpENsgfHYTw8FbX4kv53mFLn2i3XPoq69LyhYSCBJtitaHx9QnsVTrsogI4Z3+HtEfZ2/GFPOtf5g==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "dependencies": { - "@babel/highlight": "^7.25.7", - "picocolors": "^1.0.0" + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" }, "engines": { "node": ">=6.9.0" @@ -224,18 +225,18 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.7.tgz", - "integrity": "sha512-CbkjYdsJNHFk8uqpEkpCvRs3YRp9tY6FmFY7wLMSYuGYkrdUi7r2lc4/wqsvlHoMznX3WJ9IP8giGPq68T/Y6g==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.7.tgz", - "integrity": "sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", "dev": true, "engines": { "node": ">=6.9.0" @@ -251,111 +252,25 @@ } }, "node_modules/@babel/helpers": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.25.7.tgz", - "integrity": "sha512-Sv6pASx7Esm38KQpF/U/OXLwPPrdGHNKoeblRxgZRLXnAtnkEe4ptJPDtAZM7fBLadbc1Q07kQpSiGQ0Jg6tRA==", - "dev": true, - "dependencies": { - "@babel/template": "^7.25.7", - "@babel/types": "^7.25.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.25.7.tgz", - "integrity": "sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.1.tgz", + "integrity": "sha512-FCvFTm0sWV8Fxhpp2McP5/W53GPllQ9QeQ7SiqGWjMf/LVG07lFa5+pgK05IRhVwtvafT22KF+ZSnM9I545CvQ==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.25.7", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" + "@babel/template": "^7.27.1", + "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/@babel/parser": { - "version": "7.25.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.25.8.tgz", - "integrity": "sha512-HcttkxzdPucv3nNFmfOOMfFf64KgdJVqm1KaCm25dPGMLElo9nsLvXeJECQg8UzPuBGLyTSA0ZzqCtDSzKTEoQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.1.tgz", + "integrity": "sha512-I0dZ3ZpCrJ1c04OqlNsQcKiZlsrXf/kkE4FXzID9rIOYICsAbA8mMDzhW/luRNAHdCNt7os/u8wenklZDlUVUQ==", "dev": true, "dependencies": { - "@babel/types": "^7.25.8" + "@babel/types": "^7.27.1" }, "bin": { "parser": "bin/babel-parser.js" @@ -587,14 +502,14 @@ } }, "node_modules/@babel/template": { - "version": "7.25.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.7.tgz", - "integrity": "sha512-wRwtAgI3bAS+JGU2upWNL9lSlDcRCqD05BZ1n3X2ONLH1WilFP6O1otQjeMK/1g0pvYcXC7b/qVUB1keofjtZA==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.1.tgz", + "integrity": "sha512-Fyo3ghWMqkHHpHQCoBs2VnYjR4iWFFjguTDEqA5WgZDOrFesVjMhMM2FSqTKSoUSDO1VQtavj8NFpdRBEvJTtg==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.25.7", - "@babel/parser": "^7.25.7", - "@babel/types": "^7.25.7" + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.1", + "@babel/types": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -619,14 +534,13 @@ } }, "node_modules/@babel/types": { - "version": "7.25.8", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.25.8.tgz", - "integrity": "sha512-JWtuCu8VQsMladxVz/P4HzHUGCAwpuqacmowgXFs5XjxIgKuNjnLokQzuVjlTvIzODaDmpjT3oxcC48vyk9EWg==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.1.tgz", + "integrity": "sha512-+EzkxvLNfiUeKMgy/3luqfsCWFRXLb7U6wNQTk60tovuckwB15B191tJWvpp4HjiQWdJkCxO3Wbvc6jlk3Xb2Q==", "dev": true, "dependencies": { - "@babel/helper-string-parser": "^7.25.7", - "@babel/helper-validator-identifier": "^7.25.7", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" }, "engines": { "node": ">=6.9.0" @@ -2014,9 +1928,9 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -3574,9 +3488,9 @@ "dev": true }, "node_modules/picocolors": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", - "integrity": "sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true }, "node_modules/picomatch": { @@ -3928,15 +3842,6 @@ "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", From 7a05b244aad4ce58f83caacdf51217448a4f97e8 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 30 Apr 2025 17:21:00 -0700 Subject: [PATCH 96/98] [TM-1931] Fix imports. --- cdk/service-stack/lib/service-stack.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdk/service-stack/lib/service-stack.ts b/cdk/service-stack/lib/service-stack.ts index f3e6bc32..e2d01ad5 100644 --- a/cdk/service-stack/lib/service-stack.ts +++ b/cdk/service-stack/lib/service-stack.ts @@ -10,7 +10,6 @@ import { } from "aws-cdk-lib/aws-ecs-patterns"; import { Role } from "aws-cdk-lib/aws-iam"; import { upperFirst } from "lodash"; -import { Dictionary } from "factory-girl-ts"; const extractFromEnv = (...names: string[]) => names.map(name => { @@ -24,7 +23,7 @@ type Mutable = { }; // Recommendations for optimal pricing from AWs -const RIGHTSIZE_RECOMMENDATIONS: Dictionary> = { +const RIGHTSIZE_RECOMMENDATIONS: Record> = { "research-service": { prod: { cpu: 1024, From 86f13872b27f59de3c808347b36215741839e428 Mon Sep 17 00:00:00 2001 From: Jose Carlos Laura Ramirez Date: Thu, 1 May 2025 12:25:37 -0400 Subject: [PATCH 97/98] [TM-2010] fix typo and add subjectKey as parameter for template rendering --- libs/common/src/lib/email/email.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index 412d220b..f4fda96f 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -26,7 +26,7 @@ export class EmailService { this.transporter = nodemailer.createTransport({ host: this.configService.get("MAIL_HOST"), port: this.configService.get("MAIL_PORT"), - secure: this.configService.get("MAIN_PORT") === 465, + secure: this.configService.get("MAIL_PORT") === 465, auth: { user: this.configService.get("MAIL_USERNAME"), pass: this.configService.get("MAIL_PASSWORD") @@ -82,7 +82,8 @@ export class EmailService { i18nKeys: Dictionary, { i18nReplacements, additionalValues }: I18nEmailOptions = {} ) { - if (i18nKeys["subject"] == null) throw new InternalServerErrorException("Email subject is required"); + if (i18nKeys["subject"] == null && i18nKeys["subjectKey"] == null) + throw new InternalServerErrorException("Email subject is required"); const { subject, ...translated } = await this.localizationService.translateKeys( i18nKeys, locale, From 78e1f301df5ba5aa197413a3629691ab80219815 Mon Sep 17 00:00:00 2001 From: Jose Carlos Laura Ramirez Date: Thu, 1 May 2025 12:43:18 -0400 Subject: [PATCH 98/98] [TM-2010] improve subject gather from params --- libs/common/src/lib/email/email.service.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/libs/common/src/lib/email/email.service.ts b/libs/common/src/lib/email/email.service.ts index f4fda96f..054ca6bc 100644 --- a/libs/common/src/lib/email/email.service.ts +++ b/libs/common/src/lib/email/email.service.ts @@ -82,9 +82,11 @@ export class EmailService { i18nKeys: Dictionary, { i18nReplacements, additionalValues }: I18nEmailOptions = {} ) { - if (i18nKeys["subject"] == null && i18nKeys["subjectKey"] == null) - throw new InternalServerErrorException("Email subject is required"); - const { subject, ...translated } = await this.localizationService.translateKeys( + if (!this.hasSubject(i18nKeys)) throw new InternalServerErrorException("Email subject is required"); + + const subjectKey = this.getSubjectKey(i18nKeys); + + const { [subjectKey]: subject, ...translated } = await this.localizationService.translateKeys( i18nKeys, locale, i18nReplacements ?? {} @@ -109,4 +111,12 @@ export class EmailService { const body = this.templateService.render(EMAIL_TEMPLATE, data); return { subject, body }; } + + private hasSubject(i18nKeys: Dictionary) { + return i18nKeys["subject"] != null || i18nKeys["subjectKey"] != null; + } + + private getSubjectKey(i18nKeys: Dictionary) { + return i18nKeys["subject"] == null ? "subjectKey" : "subject"; + } }