[RELEASE] Dazzling Diamond

.env.local.sample

@@ -15,6 +15,9 @@ DB_PASSWORD=wri
 REDIS_HOST=localhost
 REDIS_PORT=6379
 
+# Shared Data API key that may be used by all devs and AWS environments
+DATA_API_KEY=349c85f6-a915-4d3d-b337-0a0dafc0e5db
+
 JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt
 
 AWS_ENDPOINT==http://minio:9000
@@ -28,15 +31,17 @@ AIRTABLE_BASE_ID=
 SLACK_API_KEY=
 UDB_SLACK_CHANNEL=
 
-MAIL_HOST=email-smtp.eu-west-1.amazonaws.com
-MAIL_PORT=587
-MAIL_USERNAME=
-MAIL_PASSWORD=
-[email protected]
+# Points to the mail catcher running from the PHP docker-compose. Visit localhost:1080 to see "sent" emails.
+MAIL_HOST=localhost
+MAIL_PORT=1025
+[email protected]
 # If set, email is redirected from the original recipient to this list of addresses. The original
 # recipients are encoded in the X-Original-Recipients header
 MAIL_RECIPIENTS=
+# If set, email addresses on this list will be exempted from receiving updates to entity statuses
+ENTITY_UPDATE_DO_NOT_EMAIL=
 
 TRANSIFEX_TOKEN=1/dada8aab9ed6bba68805fcab6349f2e69d3b1367
 
 EMAIL_IMAGE_BASE_URL=https://api.terramatch.org/images
+APP_FRONT_END=http://localhost:3000

.github/workflows/deploy-service.yml

@@ -78,6 +78,7 @@ jobs:
           echo "AIRTABLE_API_KEY=\"${{ secrets.AIRTABLE_API_KEY }}\"" >> .env
           echo "SENTRY_DSN=\"${{ secrets.SENTRY_DSN }}\"" >> .env
           echo "SLACK_API_KEY=\"${{ secrets.SLACK_API_KEY }}\"" >> .env
+          echo "ENTITY_UPDATE_DO_NOT_EMAIL=\"${{ secrets.ENTITY_UPDATE_DO_NOT_EMAIL }}\"" >> .env
           : # Don't build the base image with NODE_ENV because it'll limit the packages that are installed
           docker build -t terramatch-microservices-base:nx-base .
           SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:${{ env.IMAGE_TAG }}

.github/workflows/pull-request.yml

@@ -46,7 +46,7 @@ jobs:
       # First run just the small database test to get the test database synced to the current schema
       # in a clean way.
       - name: Sync DB Schema
-        run: npx nx test database --no-cloud --skip-nx-cache
+        run: npx nx test database --no-cloud --skip-nx-cache libs/database/src/lib/database.module.spec.ts
 
       # All tests are run in band so that wipes of database tables from one test doesn't interfere
       # with other tests.

README.md

@@ -83,18 +83,37 @@ to take effect.
 Once this project is live in production, we can explore continuous deployment to at least staging and prod envs on the staging
 and main branches.
 
+# Environment
+
+The Environment for a given service deployment is configured in Github Actions secrets / variables. Some are repo-wide, and
+some apply only to a given environment. During the build process, the contents of the variables applied to .env are visible
+to the general public, so we need to be very careful about what is included there. Nothing sensitive (passwords, email
+addresses, API tokens, etc) may be included in Variables, and must instead be in Secrets
+
+- If you need to update a _non-secret_ ENV variable, add / update it in the given environment's ENV variable
+- If you need to add a _secret_ ENV variable, create the secret in Github actions, and then add a line to `deploy-service.yml`
+  to append that secret to the generated `.env` variable.
+- The current value of secrets in GitHub actions may not be read by anyone, including repository admins. If you need to
+  inspect the current value of a configured secret, the recommended approach is to access that deployed service's REPL, and
+  pull the value using the `ConfigService`:
+
+```
+> $(ConfigService).get("SUPER_SECRETE_ENV_VALUE");
+```
+
 # Creating a new service
 
 - In the root directory: `nx g @nx/nest:app apps/foo-service`
 - Set up the new `main.ts` similarly to existing services.
-  - Make sure swagger docs and the `/health` endpoint are implemented
+  - Make sure swagger docs are implemented
   - Pick a default local port that is unique from other services
   - Make sure the top of `main.ts` has these two lines:
   ```
   // eslint-disable-next-line @nx/enforce-module-boundaries
   import "../../../instrument-sentry";
   ```
   - Add the `SentryModule` and `SentryGlobalFilter` to your main `app.module.ts`. See an existing service for an example.
+  - Add the `HealthModule` to your main `app.module.ts`. You will likely need `CommonModule` as well.
 - Set up REPL access:
   - Copy `repl.ts` from an existing service (and modify to specify the new service's name)
   - Add the `build-repl` target to `project.json`, which an empty definition.

apps/entity-service/src/app.module.ts

@@ -1,17 +1,16 @@
 import { Module } from "@nestjs/common";
-import { DatabaseModule } from "@terramatch-microservices/database";
 import { CommonModule } from "@terramatch-microservices/common";
-import { HealthModule } from "./health/health.module";
 import { TreesController } from "./trees/trees.controller";
 import { TreeService } from "./trees/tree.service";
 import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup";
 import { APP_FILTER } from "@nestjs/core";
 import { EntitiesService } from "./entities/entities.service";
 import { EntitiesController } from "./entities/entities.controller";
 import { EntityAssociationsController } from "./entities/entity-associations.controller";
+import { HealthModule } from "@terramatch-microservices/common/health/health.module";
 
 @Module({
-  imports: [SentryModule.forRoot(), DatabaseModule, CommonModule, HealthModule],
+  imports: [SentryModule.forRoot(), CommonModule, HealthModule],
   controllers: [EntitiesController, EntityAssociationsController, TreesController],
   providers: [
     {

apps/entity-service/src/entities/dto/disturbance.dto.ts

@@ -0,0 +1,30 @@
+import { JsonApiDto } from "@terramatch-microservices/common/decorators";
+import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes";
+import { ApiProperty } from "@nestjs/swagger";
+import { Disturbance } from "@terramatch-microservices/database/entities/disturbance.entity";
+import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto";
+
+@JsonApiDto({ type: "disturbances" })
+export class DisturbanceDto extends AssociationDto<DisturbanceDto> {
+  constructor(disturbance: Disturbance, additional: AssociationDtoAdditionalProps) {
+    super({
+      ...pickApiProperties(disturbance, DisturbanceDto),
+      ...additional
+    });
+  }
+
+  @ApiProperty({ nullable: true })
+  collection: string | null;
+
+  @ApiProperty({ nullable: true })
+  type: string | null;
+
+  @ApiProperty({ nullable: true })
+  intensity: string | null;
+
+  @ApiProperty({ nullable: true })
+  extent: string | null;
+
+  @ApiProperty({ nullable: true })
+  description: string | null;
+}

apps/entity-service/src/entities/dto/entity-query.dto.ts

@@ -4,12 +4,16 @@ import { NumberPage } from "@terramatch-microservices/common/dto/page.dto";
 import {
   MAX_PAGE_SIZE,
   PROCESSABLE_ENTITIES,
-  ProcessableEntity,
+  PROCESSABLE_ASSOCIATIONS,
   POLYGON_STATUSES_FILTERS,
   PolygonStatusFilter
 } from "../entities.service";
 import { Type } from "class-transformer";
 
+export const VALID_SIDELOAD_TYPES = [...PROCESSABLE_ENTITIES, ...PROCESSABLE_ASSOCIATIONS] as const;
+
+export type SideloadType = (typeof VALID_SIDELOAD_TYPES)[number];
+
 class QuerySort {
   @ApiProperty({ name: "sort[field]", required: false })
   @IsOptional()
@@ -22,9 +26,13 @@ class QuerySort {
 }
 
 export class EntitySideload {
-  @IsIn(PROCESSABLE_ENTITIES)
-  @ApiProperty({ name: "entity", enum: PROCESSABLE_ENTITIES, description: "Entity type to sideload" })
-  entity: ProcessableEntity;
+  @IsIn(VALID_SIDELOAD_TYPES)
+  @ApiProperty({
+    name: "entity",
+    enum: VALID_SIDELOAD_TYPES,
+    description: "Entity or association type to sideload"
+  })
+  entity: SideloadType;
 
   @ApiProperty({ name: "pageSize", description: "The page size to include." })
   @IsInt()

apps/entity-service/src/entities/dto/entity-update.dto.ts

@@ -0,0 +1,87 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { ENTITY_STATUSES, REPORT_STATUSES, SITE_STATUSES } from "@terramatch-microservices/database/constants/status";
+import { IsArray, IsBoolean, IsIn, IsOptional, IsString } from "class-validator";
+import { JsonApiDataDto, JsonApiMultiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto";
+import { Type } from "class-transformer";
+
+export class EntityUpdateAttributes {
+  @IsOptional()
+  @IsIn(ENTITY_STATUSES)
+  @ApiProperty({
+    description: "Request to change to the status of the given entity",
+    nullable: true,
+    enum: ENTITY_STATUSES
+  })
+  status?: string | null;
+
+  @IsOptional()
+  @IsString()
+  @ApiProperty({ description: "Specific feedback for the PD", nullable: true })
+  feedback?: string | null;
+
+  @IsOptional()
+  @IsArray()
+  @Type(() => String)
+  @ApiProperty({
+    isArray: true,
+    type: String,
+    description: "The fields in the entity form that need attention from the PD",
+    nullable: true
+  })
+  feedbackFields?: string[] | null;
+}
+
+export class ProjectUpdateAttributes extends EntityUpdateAttributes {
+  @IsOptional()
+  @IsBoolean()
+  @ApiProperty({ description: "Update the isTest flag.", nullable: true })
+  isTest?: boolean;
+}
+
+export class SiteUpdateAttributes extends EntityUpdateAttributes {
+  @IsOptional()
+  @IsIn(SITE_STATUSES)
+  @ApiProperty({
+    description: "Request to change to the status of the given site",
+    nullable: true,
+    enum: SITE_STATUSES
+  })
+  status?: string | null;
+}
+
+export class ReportUpdateAttributes extends EntityUpdateAttributes {
+  @IsOptional()
+  @IsIn(REPORT_STATUSES)
+  @ApiProperty({
+    description: "Request to change to the status of the given report",
+    nullable: true,
+    enum: REPORT_STATUSES
+  })
+  status?: string | null;
+
+  @IsOptional()
+  @IsBoolean()
+  @ApiProperty({ description: "Update the nothingToReport flag.", nullable: true })
+  nothingToReport?: boolean;
+}
+
+export class ProjectUpdateData extends JsonApiDataDto({ type: "projects" }, ProjectUpdateAttributes) {}
+export class SiteUpdateData extends JsonApiDataDto({ type: "sites" }, SiteUpdateAttributes) {}
+export class NurseryUpdateData extends JsonApiDataDto({ type: "nurseries" }, EntityUpdateAttributes) {}
+export class ProjectReportUpdateData extends JsonApiDataDto({ type: "projectReports" }, ReportUpdateAttributes) {}
+export class SiteReportUpdateData extends JsonApiDataDto({ type: "siteReports" }, ReportUpdateAttributes) {}
+export class NurseryReportUpdateData extends JsonApiDataDto({ type: "nurseryReports" }, ReportUpdateAttributes) {}
+
+export type EntityUpdateData =
+  | ProjectUpdateAttributes
+  | SiteUpdateAttributes
+  | ReportUpdateAttributes
+  | EntityUpdateAttributes;
+export class EntityUpdateBody extends JsonApiMultiBodyDto([
+  ProjectUpdateData,
+  SiteUpdateData,
+  NurseryUpdateData,
+  ProjectReportUpdateData,
+  SiteReportUpdateData,
+  NurseryReportUpdateData
+] as const) {}

apps/entity-service/src/entities/dto/entity.dto.ts

@@ -1,12 +1,20 @@
 import { ApiProperty } from "@nestjs/swagger";
 import { HybridSupportDto } from "@terramatch-microservices/common/dto/hybrid-support.dto";
+import { JsonApiConstants } from "@terramatch-microservices/common/decorators/json-api-constants.decorator";
+import { ENTITY_TYPES } from "@terramatch-microservices/database/constants/entities";
 
 /**
  * A utility type for constructing the "extra props" type of a DTO based on what's in the dto, the
  * base type (e.g. the light DTO & base model for a full DTO)
  */
 export type AdditionalProps<DTO, BaseType> = Pick<DTO, keyof Omit<DTO, keyof BaseType>>;
 
+@JsonApiConstants
+export class SupportedEntities {
+  @ApiProperty({ example: ENTITY_TYPES })
+  ENTITY_TYPES: string[];
+}
+
 export abstract class EntityDto extends HybridSupportDto {
   /**
    * All EntityDtos must include UUID in the attributes for use in the react-admin pagination

apps/entity-service/src/entities/dto/invasive.dto.ts

@@ -0,0 +1,21 @@
+import { JsonApiDto } from "@terramatch-microservices/common/decorators";
+import { pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes";
+import { ApiProperty } from "@nestjs/swagger";
+import { Invasive } from "@terramatch-microservices/database/entities/invasive.entity";
+import { AssociationDto, AssociationDtoAdditionalProps } from "./association.dto";
+
+@JsonApiDto({ type: "invasives" })
+export class InvasiveDto extends AssociationDto<InvasiveDto> {
+  constructor(invasive: Invasive, additional: AssociationDtoAdditionalProps) {
+    super({
+      ...pickApiProperties(invasive, InvasiveDto),
+      ...additional
+    });
+  }
+
+  @ApiProperty({ nullable: true })
+  type: string | null;
+
+  @ApiProperty({ nullable: true })
+  name: string | null;
+}

apps/entity-service/src/entities/dto/nursery-report.dto.ts

@@ -89,6 +89,9 @@ export class NurseryReportLightDto extends EntityDto {
 
   @ApiProperty()
   createdAt: Date;
+
+  @ApiProperty({ nullable: true })
+  nothingToReport: boolean | null;
 }
 
 export type AdditionalNurseryReportLightProps = Pick<NurseryReportLightDto, "reportTitle">;
@@ -156,9 +159,6 @@ export class NurseryReportFullDto extends NurseryReportLightDto {
   @ApiProperty({ nullable: true })
   feedbackFields: string[] | null;
 
-  @ApiProperty()
-  nothingToReport: boolean;
-
   @ApiProperty({ nullable: true })
   completion: number | null;
 

apps/entity-service/src/entities/dto/project-report.dto.ts

@@ -71,6 +71,9 @@ export class ProjectReportLightDto extends EntityDto {
 
   @ApiProperty()
   updatedAt: Date;
+
+  @ApiProperty({ nullable: true })
+  pctSurvivalToDate: number | null;
 }
 
 export type AdditionalProjectReportFullProps = AdditionalProps<
@@ -188,9 +191,6 @@ export class ProjectReportFullDto extends ProjectReportLightDto {
   @ApiProperty({ nullable: true })
   significantChange: string | null;
 
-  @ApiProperty({ nullable: true })
-  pctSurvivalToDate: number | null;
-
   @ApiProperty({ nullable: true })
   survivalCalculation: string | null;
 
@@ -304,4 +304,52 @@ export class ProjectReportFullDto extends ProjectReportLightDto {
 
   @ApiProperty({ type: () => MediaDto, isArray: true })
   photos: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  baselineReportUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  localGovernanceOrderLetterUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  eventsMeetingsPhotos: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  localGovernanceProofOfPartnershipUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  topThreeSuccessesUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  directJobsUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  convergenceJobsUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  convergenceSchemesUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  livelihoodActivitiesUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  directLivelihoodImpactsUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  certifiedDatabaseUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  physicalAssetsPhotos: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  indirectCommunityPartnersUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  trainingCapacityBuildingUpload: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  trainingCapacityBuildingPhotos: MediaDto[];
+
+  @ApiProperty({ type: () => MediaDto, isArray: true })
+  financialReportUpload: MediaDto[];
 }