diff --git a/api/.env.template b/api/.env.template index ae4144653d..17e309d13b 100644 --- a/api/.env.template +++ b/api/.env.template @@ -14,13 +14,13 @@ CLOUDINARY_SECRET= CLOUDINARY_CLOUD_NAME=exygy # app secret APP_SECRET="some-long-secret-key" -# url for the proxy +# url for the proxy PROXY_URL= # the node env the app should be running as NODE_ENV=development # how long a generated multi-factor authentication code should be MFA_CODE_LENGTH=5 -# TTL for the mfa code, stored in milliseconds +# TTL for the mfa code, stored in milliseconds MFA_CODE_VALID=60000 # how long logins are locked after too many failed login attempts in milliseconds AUTH_LOCK_LOGIN_COOLDOWN=1800000 @@ -94,3 +94,5 @@ S3_BUCKET= S3_ACCESS_TOKEN= # Secret key for the service account to the s3 bucket S3_SECRET_TOKEN= +FAST_API_KEY= +FAST_API_URL= \ No newline at end of file diff --git a/api/prisma/migrations/20251119163459_add_ai_consent_fields/migration.sql b/api/prisma/migrations/20251119163459_add_ai_consent_fields/migration.sql new file mode 100644 index 0000000000..753ab27981 --- /dev/null +++ b/api/prisma/migrations/20251119163459_add_ai_consent_fields/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable + +ALTER TABLE "user_accounts" ADD COLUMN "ai_consent_given_at" TIMESTAMPTZ(6), +ADD COLUMN "has_consented_to_ai" BOOLEAN DEFAULT false; diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 0c0e8fa4ed..9da44cef87 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -989,6 +989,8 @@ model UserAccounts { failedLoginAttemptsCount Int @default(0) @map("failed_login_attempts_count") phoneNumberVerified Boolean? @default(false) @map("phone_number_verified") agreedToTermsOfService Boolean @default(false) @map("agreed_to_terms_of_service") + hasConsentedToAI Boolean? @default(false) @map("has_consented_to_ai") + aiConsentGivenAt DateTime? @map("ai_consent_given_at") @db.Timestamptz(6) hitConfirmationUrl DateTime? @map("hit_confirmation_url") @db.Timestamptz(6) activeAccessToken String? @map("active_access_token") @db.VarChar activeRefreshToken String? @map("active_refresh_token") @db.VarChar diff --git a/api/src/controllers/data-explorer.controller.ts b/api/src/controllers/data-explorer.controller.ts new file mode 100644 index 0000000000..78a86b3c38 --- /dev/null +++ b/api/src/controllers/data-explorer.controller.ts @@ -0,0 +1,66 @@ +import { + Controller, + Get, + Post, + Body, + Query, + Request, + UseGuards, + UseInterceptors, + UsePipes, + ValidationPipe, +} from '@nestjs/common'; +import { ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { Request as ExpressRequest } from 'express'; +import { defaultValidationPipeOptions } from '../utilities/default-validation-pipe-options'; +import { ValidationsGroupsEnum } from '../enums/shared/validation-groups-enum'; +import { ActivityLogInterceptor } from '../interceptors/activity-log.interceptor'; +import { PermissionTypeDecorator } from '../decorators/permission-type.decorator'; +import { ApiKeyGuard } from '../guards/api-key.guard'; +import { JwtAuthGuard } from '../guards/jwt.guard'; +import { DataExplorerService } from '../services/data-explorer.service'; +import { DataExplorerParams } from '../dtos/applications/data-explorer/params/data-explorer-params.dto'; +import { DataExplorerReport } from '../dtos/applications/data-explorer/products/data-explorer-report.dto'; +import { GenerateInsightParams } from '../dtos/applications/data-explorer/generate-insight-params.dto'; +import { GenerateInsightResponse } from '../dtos/applications/data-explorer/generate-insight-response.dto'; + +@Controller('data-explorer') +@ApiTags('data-explorer') +@UsePipes( + new ValidationPipe({ + ...defaultValidationPipeOptions, + groups: [ValidationsGroupsEnum.default, ValidationsGroupsEnum.partners], + }), +) +@UseGuards(ApiKeyGuard, JwtAuthGuard) +@PermissionTypeDecorator('application') +@UseInterceptors(ActivityLogInterceptor) +export class DataExplorerController { + constructor(private readonly dataExplorerService: DataExplorerService) {} + + @Get('generate-report') + @ApiOperation({ + summary: 'Generate a report', + operationId: 'generateReport', + }) + @ApiOkResponse({ type: DataExplorerReport }) + async generateReport( + @Request() req: ExpressRequest, + @Query() queryParams: DataExplorerParams, + ) { + return await this.dataExplorerService.generateReport(queryParams, req); + } + + @Post('generate-insight') + @ApiOperation({ + summary: 'Generate AI insights from data', + operationId: 'generateInsight', + }) + @ApiOkResponse({ type: GenerateInsightResponse }) + async generateInsight( + @Request() req: ExpressRequest, + @Body() body: GenerateInsightParams, + ) { + return await this.dataExplorerService.generateInsight(body, req); + } +} diff --git a/api/src/controllers/user.controller.ts b/api/src/controllers/user.controller.ts index 77d81b0b2e..90586a517d 100644 --- a/api/src/controllers/user.controller.ts +++ b/api/src/controllers/user.controller.ts @@ -48,6 +48,7 @@ import { AdminOrJurisdictionalAdminGuard } from '../guards/admin-or-jurisdiction import { ActivityLogInterceptor } from '../interceptors/activity-log.interceptor'; import { PermissionTypeDecorator } from '../decorators/permission-type.decorator'; import { UserFilterParams } from '../dtos/users/user-filter-params.dto'; +import { UserAiConsentDto } from '../dtos/users/user-ai-consent.dto'; import { UserCsvExporterService } from '../services/user-csv-export.service'; import { ExportLogInterceptor } from '../interceptors/export-log.interceptor'; import { RequestSingleUseCode } from '../dtos/single-use-code/request-single-use-code.dto'; @@ -125,6 +126,22 @@ export class UserController { return await this.userService.favoriteListings(userId); } + @Put('ai-consent') + @ApiOperation({ + summary: 'Update AI consent preference', + operationId: 'updateAiConsent', + }) + @ApiOkResponse({ type: User }) + @UseGuards(JwtAuthGuard) + @UseInterceptors(ActivityLogInterceptor) + async updateAIConsent( + @Request() req: ExpressRequest, + @Body() body: UserAiConsentDto, + ): Promise { + const user = mapTo(User, req['user']); + return await this.userService.updateAIConsent(user.id, body.hasConsented); + } + @Post() @ApiOperation({ summary: 'Creates a public only user', diff --git a/api/src/dtos/applications/data-explorer/generate-insight-params.dto.ts b/api/src/dtos/applications/data-explorer/generate-insight-params.dto.ts new file mode 100644 index 0000000000..3621517d1a --- /dev/null +++ b/api/src/dtos/applications/data-explorer/generate-insight-params.dto.ts @@ -0,0 +1,42 @@ +import { Expose, Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsString, IsObject, ValidateNested } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../enums/shared/validation-groups-enum'; +import { ReportProducts } from './products/data-explorer-report-products.dto'; + +export class GenerateInsightParams { + @Expose() + @ApiProperty({ + type: ReportProducts, + description: 'The current data object containing report products', + }) + @IsObject({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ groups: [ValidationsGroupsEnum.default] }) + @Type(() => ReportProducts) + data: ReportProducts; + + @Expose() + @ApiProperty({ + type: String, + example: 'What are the key trends in this data?', + description: 'The prompt to send to the AI for generating insights', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + prompt: string; + + @Expose() + @ApiPropertyOptional({ + type: String, + example: 'jurisdictionId', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + jurisdictionId?: string; + + @Expose() + @ApiPropertyOptional({ + type: String, + example: 'userId', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + userId?: string; +} diff --git a/api/src/dtos/applications/data-explorer/generate-insight-response.dto.ts b/api/src/dtos/applications/data-explorer/generate-insight-response.dto.ts new file mode 100644 index 0000000000..7954d5269e --- /dev/null +++ b/api/src/dtos/applications/data-explorer/generate-insight-response.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../enums/shared/validation-groups-enum'; +import { AbstractDTO } from '../../shared/abstract.dto'; + +export class GenerateInsightResponse extends AbstractDTO { + @Expose() + @ApiProperty({ + type: String, + example: 'The data shows significant trends...', + description: 'Markdown-formatted AI-generated insights', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + insight: string; +} diff --git a/api/src/dtos/applications/data-explorer/params/data-explorer-params.dto.ts b/api/src/dtos/applications/data-explorer/params/data-explorer-params.dto.ts new file mode 100644 index 0000000000..59a57c96ed --- /dev/null +++ b/api/src/dtos/applications/data-explorer/params/data-explorer-params.dto.ts @@ -0,0 +1,181 @@ +import { Expose, Type } from 'class-transformer'; +import { ApiPropertyOptional } from '@nestjs/swagger'; +import { + IsOptional, + IsString, + IsNumber, + IsArray, + Min, + Max, +} from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../enums/shared/validation-groups-enum'; + +export class DataExplorerParams { + @Expose() + @ApiPropertyOptional({ + type: String, + example: 'jurisdictionId', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + jurisdictionId?: string; + + @Expose() + @ApiPropertyOptional({ + type: String, + example: 'userId', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + userId?: string; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['1', '2', '3', '4+'], + description: 'Filter by household size categories', + }) + @IsArray() + @IsOptional() + householdSize?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 30000, + description: 'Minimum household income in USD', + }) + @IsNumber() + @Type(() => Number) + @Min(0) + @IsOptional() + minIncome?: number; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 100000, + description: 'Maximum household income in USD', + }) + @IsNumber() + @Type(() => Number) + @Min(0) + @IsOptional() + maxIncome?: number; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['0-30% AMI', '31-50% AMI'], + description: 'Area Median Income level categories', + }) + @IsArray() + @IsOptional() + amiLevels?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['yes', 'no'], + description: 'Housing voucher or subsidy status', + }) + @IsArray() + @IsOptional() + voucherStatuses?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['mobility', 'hearing'], + description: 'Accessibility accommodation types', + }) + @IsArray() + @IsOptional() + accessibilityTypes?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['Asian', 'White'], + description: 'Racial categories for filtering', + }) + @IsArray() + @IsOptional() + races?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['Hispanic or Latino', 'Not Hispanic or Latino'], + description: 'Ethnicity categories for filtering', + }) + @IsArray() + @IsOptional() + ethnicities?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['San Francisco County', 'Alameda County'], + description: 'Counties where applicants currently reside', + }) + @IsArray() + @IsOptional() + applicantResidentialCounties?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: ['San Francisco County', 'Alameda County'], + description: 'Counties where applicants work', + }) + @IsArray() + @IsOptional() + applicantWorkCounties?: string[]; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 18, + description: 'Minimum age of applicant', + }) + @IsNumber() + @Type(() => Number) + @Min(0) + @Max(150) + @IsOptional() + minAge?: number; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 65, + description: 'Maximum age of applicant', + }) + @IsNumber() + @Type(() => Number) + @Min(0) + @Max(150) + @IsOptional() + maxAge?: number; + + @Expose() + @ApiPropertyOptional({ + type: String, + example: '2025-01-01', + description: 'Start date for filtering applications (ISO 8601 format)', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + startDate?: string; + + @Expose() + @ApiPropertyOptional({ + type: String, + example: '2025-06-30', + description: 'End date for filtering applications (ISO 8601 format)', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + endDate?: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/data-explorer-report-products.dto.ts b/api/src/dtos/applications/data-explorer/products/data-explorer-report-products.dto.ts new file mode 100644 index 0000000000..4bc6a796fb --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/data-explorer-report-products.dto.ts @@ -0,0 +1,124 @@ +import { Expose, Type } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsArray, ValidateNested, IsObject } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../enums/shared/validation-groups-enum'; +import { AccessibilityFrequency } from './frequency/accessibility-frequency.dto'; +import { AgeFrequency } from './frequency/age-frequency.dto'; +import { EthnicityFrequency } from './frequency/ethnicity-frequency.dto'; +import { LanguageFrequency } from './frequency/language-frequency.dto'; +import { LocationFrequency } from './frequency/location-frequency.dto'; +import { RaceFrequency } from './frequency/race-frequency.dto'; +import { SubsidyFrequency } from './frequency/subsidy-frequency.dto'; + +export class ReportProducts { + @Expose() + @ApiProperty({ + type: 'object', + additionalProperties: { + type: 'object', + additionalProperties: { + type: 'number', + }, + }, + example: { + '1': { + '0-30 AMI': 45, + '31-50 AMI': 78, + '51-80 AMI': 92, + '81-120 AMI': 34, + }, + '2': { + '0-30 AMI': 67, + '31-50 AMI': 89, + '51-80 AMI': 112, + '81-120 AMI': 56, + }, + '3': { + '0-30 AMI': 82, + '31-50 AMI': 95, + '51-80 AMI': 78, + '81-120 AMI': 45, + }, + '4+': { + '0-30 AMI': 93, + '31-50 AMI': 88, + '51-80 AMI': 65, + '81-120 AMI': 40, + }, + }, + description: + 'Cross-tabulation of income bands by household size. Keys are household sizes, values are income band distributions.', + }) + @IsObject({ groups: [ValidationsGroupsEnum.default] }) + incomeHouseholdSizeCrossTab: Record>; + + @Expose() + @ApiProperty({ + type: [RaceFrequency], + description: 'Frequency distribution by race', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => RaceFrequency) + raceFrequencies: RaceFrequency[]; + + @Expose() + @ApiProperty({ + type: [EthnicityFrequency], + description: 'Frequency distribution by ethnicity', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => EthnicityFrequency) + ethnicityFrequencies: EthnicityFrequency[]; + + @Expose() + @ApiProperty({ + type: [SubsidyFrequency], + description: 'Frequency distribution by subsidy or voucher type', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => SubsidyFrequency) + subsidyOrVoucherTypeFrequencies: SubsidyFrequency[]; + + @Expose() + @ApiProperty({ + type: [AccessibilityFrequency], + description: 'Frequency distribution by accessibility type', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => AccessibilityFrequency) + accessibilityTypeFrequencies: AccessibilityFrequency[]; + + @Expose() + @ApiProperty({ + type: [AgeFrequency], + description: 'Frequency distribution by age range', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => AgeFrequency) + ageFrequencies: AgeFrequency[]; + + @Expose() + @ApiProperty({ + type: [LocationFrequency], + description: 'Frequency distribution by residential location', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => LocationFrequency) + residentialLocationFrequencies: LocationFrequency[]; + + @Expose() + @ApiProperty({ + type: [LanguageFrequency], + description: 'Frequency distribution by language preference', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @ValidateNested({ each: true, groups: [ValidationsGroupsEnum.default] }) + @Type(() => LanguageFrequency) + languageFrequencies: LanguageFrequency[]; +} diff --git a/api/src/dtos/applications/data-explorer/products/data-explorer-report.dto.ts b/api/src/dtos/applications/data-explorer/products/data-explorer-report.dto.ts new file mode 100644 index 0000000000..a3becdf173 --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/data-explorer-report.dto.ts @@ -0,0 +1,103 @@ +import { Expose, Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { + IsString, + IsNumber, + IsBoolean, + IsArray, + ValidateNested, + IsOptional, +} from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../enums/shared/validation-groups-enum'; +import { AbstractDTO } from '../../../shared/abstract.dto'; +import { ReportProducts } from './data-explorer-report-products.dto'; + +export class DataExplorerReport extends AbstractDTO { + @Expose() + @ApiProperty({ + type: String, + example: '01/01/2025 - 06/30/2025', + description: 'Date range for the report', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + dateRange: string; + + @Expose() + @ApiProperty({ + type: Number, + example: 1200, + description: 'Total number of processed applications', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + totalProcessedApplications: number; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 1200, + description: 'Total number of applicants', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + totalApplicants?: number; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 5, + description: 'Total number of listings', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + totalListings?: number; + + @Expose() + @ApiProperty({ + type: Boolean, + example: true, + description: + 'Whether the data passes k-anonymity requirements and has no errors', + }) + @IsBoolean({ groups: [ValidationsGroupsEnum.default] }) + validResponse: boolean; + + @Expose() + @ApiProperty({ + type: Boolean, + example: true, + description: + 'Whether there is sufficient data for analysis (alias for validResponse)', + }) + @IsBoolean({ groups: [ValidationsGroupsEnum.default] }) + isSufficient: boolean; + + @Expose() + @ApiProperty({ + type: Number, + example: 10, + description: 'K-anonymity score for the dataset', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + kAnonScore: number; + + @Expose() + @ApiProperty({ + type: ReportProducts, + description: + 'Report data products containing various frequency distributions', + }) + @ValidateNested({ groups: [ValidationsGroupsEnum.default] }) + @Type(() => ReportProducts) + products: ReportProducts; + + @Expose() + @ApiPropertyOptional({ + type: [String], + example: [], + description: 'Any errors encountered during report generation', + }) + @IsArray({ groups: [ValidationsGroupsEnum.default] }) + @IsString({ each: true, groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + reportErrors?: string[]; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/accessibility-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/accessibility-frequency.dto.ts new file mode 100644 index 0000000000..c57e623a58 --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/accessibility-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class AccessibilityFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'Wheelchair Accessible', + description: 'Accessibility type', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + accessibilityType: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/age-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/age-frequency.dto.ts new file mode 100644 index 0000000000..681e76e703 --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/age-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class AgeFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: '25-34', + description: 'Age range', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + age: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/data-explorer-report-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/data-explorer-report-frequency.dto.ts new file mode 100644 index 0000000000..5d5ad71cab --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/data-explorer-report-frequency.dto.ts @@ -0,0 +1,33 @@ +import { Expose, Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { + IsString, + IsNumber, + IsBoolean, + IsArray, + ValidateNested, + IsOptional, + IsObject, +} from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; + +export class FrequencyData { + @Expose() + @ApiProperty({ + type: Number, + example: 120, + description: 'Count of occurrences', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + count: number; + + @Expose() + @ApiPropertyOptional({ + type: Number, + example: 0.1, + description: 'Percentage of total', + }) + @IsNumber({}, { groups: [ValidationsGroupsEnum.default] }) + @IsOptional() + percentage?: number; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/ethnicity-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/ethnicity-frequency.dto.ts new file mode 100644 index 0000000000..83c0fc10ee --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/ethnicity-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class EthnicityFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'Hispanic or Latino', + description: 'Ethnicity category', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + ethnicity: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/language-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/language-frequency.dto.ts new file mode 100644 index 0000000000..ea6b603b81 --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/language-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class LanguageFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'English', + description: 'Language preference', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + language: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/location-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/location-frequency.dto.ts new file mode 100644 index 0000000000..11d5581e1b --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/location-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class LocationFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'Oakland', + description: 'Residential location', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + location: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/race-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/race-frequency.dto.ts new file mode 100644 index 0000000000..1af753b83f --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/race-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class RaceFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'Asian', + description: 'Race category', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + race: string; +} diff --git a/api/src/dtos/applications/data-explorer/products/frequency/subsidy-frequency.dto.ts b/api/src/dtos/applications/data-explorer/products/frequency/subsidy-frequency.dto.ts new file mode 100644 index 0000000000..c0247fcac8 --- /dev/null +++ b/api/src/dtos/applications/data-explorer/products/frequency/subsidy-frequency.dto.ts @@ -0,0 +1,16 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../../../../enums/shared/validation-groups-enum'; +import { FrequencyData } from './data-explorer-report-frequency.dto'; + +export class SubsidyFrequency extends FrequencyData { + @Expose() + @ApiProperty({ + type: String, + example: 'Section 8', + description: 'Subsidy or voucher type', + }) + @IsString({ groups: [ValidationsGroupsEnum.default] }) + subsidyType: string; +} diff --git a/api/src/dtos/users/user-ai-consent.dto.ts b/api/src/dtos/users/user-ai-consent.dto.ts new file mode 100644 index 0000000000..f5ea5f1561 --- /dev/null +++ b/api/src/dtos/users/user-ai-consent.dto.ts @@ -0,0 +1,15 @@ +import { Expose } from 'class-transformer'; +import { ApiProperty } from '@nestjs/swagger'; +import { IsBoolean } from 'class-validator'; +import { ValidationsGroupsEnum } from '../../enums/shared/validation-groups-enum'; + +export class UserAiConsentDto { + @Expose() + @ApiProperty({ + type: Boolean, + example: true, + description: 'Whether the user has consented to AI features', + }) + @IsBoolean({ groups: [ValidationsGroupsEnum.default] }) + hasConsented: boolean; +} diff --git a/api/src/dtos/users/user-update.dto.ts b/api/src/dtos/users/user-update.dto.ts index 5f0ed5b255..c6f47ec32a 100644 --- a/api/src/dtos/users/user-update.dto.ts +++ b/api/src/dtos/users/user-update.dto.ts @@ -32,6 +32,7 @@ export class UserUpdate extends OmitType(User, [ 'activeAccessToken', 'activeRefreshToken', 'jurisdictions', + 'aiConsentGivenAt', ]) { @Expose() @ApiPropertyOptional() diff --git a/api/src/dtos/users/user.dto.ts b/api/src/dtos/users/user.dto.ts index 21f122a5cb..ca7f383ad7 100644 --- a/api/src/dtos/users/user.dto.ts +++ b/api/src/dtos/users/user.dto.ts @@ -125,6 +125,17 @@ export class User extends AbstractDTO { @ApiProperty() agreedToTermsOfService: boolean; + @Expose() + @IsBoolean({ groups: [ValidationsGroupsEnum.default] }) + @ApiPropertyOptional() + hasConsentedToAI?: boolean; + + @Expose() + @IsDate({ groups: [ValidationsGroupsEnum.default] }) + @Type(() => Date) + @ApiPropertyOptional() + aiConsentGivenAt?: Date; + @Expose() @IsDate({ groups: [ValidationsGroupsEnum.default] }) @Type(() => Date) diff --git a/api/src/modules/app.module.ts b/api/src/modules/app.module.ts index 58da240cb7..867f8804a9 100644 --- a/api/src/modules/app.module.ts +++ b/api/src/modules/app.module.ts @@ -23,6 +23,7 @@ import { ThrottleGuard } from '../guards/throttler.guard'; import { ScriptRunnerModule } from './script-runner.module'; import { LotteryModule } from './lottery.module'; import { FeatureFlagModule } from './feature-flag.module'; +import { DataExplorerModule } from './data-explorer.module'; import { CronJobModule } from './cron-job.module'; @Module({ @@ -45,6 +46,7 @@ import { CronJobModule } from './cron-job.module'; ScriptRunnerModule, LotteryModule, FeatureFlagModule, + DataExplorerModule, CronJobModule, ThrottlerModule.forRoot([ { @@ -82,6 +84,7 @@ import { CronJobModule } from './cron-job.module'; ScriptRunnerModule, LotteryModule, FeatureFlagModule, + DataExplorerModule, ], }) export class AppModule {} diff --git a/api/src/modules/data-explorer.module.ts b/api/src/modules/data-explorer.module.ts new file mode 100644 index 0000000000..eff1f458e8 --- /dev/null +++ b/api/src/modules/data-explorer.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { PermissionModule } from './permission.module'; +import { DataExplorerController } from '../controllers/data-explorer.controller'; +import { DataExplorerService } from '../services/data-explorer.service'; + +@Module({ + imports: [PermissionModule], + controllers: [DataExplorerController], + providers: [DataExplorerService], + exports: [DataExplorerService], +}) +export class DataExplorerModule {} diff --git a/api/src/services/data-explorer.service.ts b/api/src/services/data-explorer.service.ts new file mode 100644 index 0000000000..936d7b6942 --- /dev/null +++ b/api/src/services/data-explorer.service.ts @@ -0,0 +1,402 @@ +import { + BadRequestException, + Injectable, + NotFoundException, + ForbiddenException, +} from '@nestjs/common'; +import { Request as ExpressRequest } from 'express'; + +import { mapTo } from '../utilities/mapTo'; + +import { PermissionService } from './permission.service'; +import { User } from '../dtos/users/user.dto'; +import { permissionActions } from '../enums/permissions/permission-actions-enum'; +import { DataExplorerParams } from '../dtos/applications/data-explorer/params/data-explorer-params.dto'; +import { DataExplorerReport } from '../dtos/applications/data-explorer/products/data-explorer-report.dto'; +import { GenerateInsightParams } from '../dtos/applications/data-explorer/generate-insight-params.dto'; +import { GenerateInsightResponse } from '../dtos/applications/data-explorer/generate-insight-response.dto'; +import axios from 'axios'; +/* + this is the service for calling the FastAPI housing-reports endpoint + this simply passes along the parameters to the FastAPI endpoint + and returns the response from the FastAPI endpoint + We place this call here so we can use the permission service to check if the user is allowed to access the data reports +*/ +@Injectable() +export class DataExplorerService { + constructor(private permissionService: PermissionService) {} + + /* + this will call the FastAPI endpoint to generate a report + and return the report data + it will also check if the user has permission to access the report data + if the user does not have permission, it will throw a ForbiddenException + if the user is not authenticated, it will throw a ForbiddenException + */ + async generateReport( + params: DataExplorerParams, + req: ExpressRequest, + ): Promise { + const user = mapTo(User, req['user']); + if (!user) { + throw new ForbiddenException(); + } + + await this.authorizeAction( + user, + permissionActions.read, + params.jurisdictionId, + ); + + const reportData = await this.getReportDataFastAPI(params); + if (!reportData) { + console.error('No report data returned from API'); + throw new NotFoundException('No report data found'); + } + + const mappedData = mapTo(DataExplorerReport, reportData); + + // Fix: class-transformer strips out complex nested objects with dynamic keys + // Manually preserve the incomeHouseholdSizeCrossTab data + if ( + reportData.products?.incomeHouseholdSizeCrossTab && + mappedData.products + ) { + mappedData.products.incomeHouseholdSizeCrossTab = + reportData.products.incomeHouseholdSizeCrossTab; + } + + // Preserve all frequency arrays that might get stripped by class-transformer + if (reportData.products && mappedData.products) { + mappedData.products.raceFrequencies = reportData.products.raceFrequencies; + mappedData.products.ethnicityFrequencies = + reportData.products.ethnicityFrequencies; + mappedData.products.subsidyOrVoucherTypeFrequencies = + reportData.products.subsidyOrVoucherTypeFrequencies; + mappedData.products.accessibilityTypeFrequencies = + reportData.products.accessibilityTypeFrequencies; + mappedData.products.ageFrequencies = reportData.products.ageFrequencies; + mappedData.products.residentialLocationFrequencies = + reportData.products.residentialLocationFrequencies; + mappedData.products.languageFrequencies = + reportData.products.languageFrequencies; + } + + // Ensure isSufficient matches validResponse + if (mappedData.validResponse !== undefined) { + mappedData.isSufficient = mappedData.validResponse; + } + + return mappedData; + } + + async getReportDataFastAPI( + params?: DataExplorerParams, + ): Promise { + try { + const API_BASE_URL = process.env.FAST_API_URL; + if (!process.env.FAST_API_KEY || !process.env.FAST_API_URL) { + throw new BadRequestException( + 'FastAPI key or URL is not configured in environment variables', + ); + } + + // Helper function to filter out placeholder values like "all", "any", empty strings, and zero + const filterPlaceholders = ( + arr: any[] | undefined | null, + ): any[] | null => { + if (!arr || !Array.isArray(arr)) return null; + const filtered = arr.filter( + (val) => val !== 'all' && val !== 'any' && val !== '' && val !== 0, + ); + return filtered.length > 0 ? filtered : null; + }; + + // Helper to check if a value is meaningful (not null, not 0, not empty string) + const isMeaningful = (val: any): boolean => { + return val !== null && val !== undefined && val !== '' && val !== 0; + }; + + // Build filter object from params (exclude jurisdictionId and userId) + // Map flat structure to nested structure expected by FastAPI + let filters = null; + if (params) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { jurisdictionId, userId, ...filterParams } = params; + + // Only build filters if there are actual filter parameters + if (Object.keys(filterParams).length > 0) { + const householdSizeArray = filterPlaceholders( + filterParams.householdSize, + ); + const races = filterPlaceholders(filterParams.races); + const ethnicities = filterPlaceholders(filterParams.ethnicities); + const accessibilityTypes = filterPlaceholders( + filterParams.accessibilityTypes, + ); + const residentialCounties = filterPlaceholders( + filterParams.applicantResidentialCounties, + ); + const workCounties = filterPlaceholders( + filterParams.applicantWorkCounties, + ); + + // Convert household size array to HouseholdSize object {min, max} + let householdSize = null; + if (householdSizeArray && householdSizeArray.length > 0) { + const numericSizes = householdSizeArray + .map((s) => (typeof s === 'string' ? parseInt(s, 10) : s)) + .filter((s) => !isNaN(s)); + if (numericSizes.length > 0) { + householdSize = { + min: Math.min(...numericSizes), + max: Math.max(...numericSizes), + }; + } + } + + // Convert income to HouseholdIncome object {min, max} + let householdIncome = null; + if ( + isMeaningful(filterParams.minIncome) || + isMeaningful(filterParams.maxIncome) + ) { + householdIncome = { + min: isMeaningful(filterParams.minIncome) + ? filterParams.minIncome + : null, + max: isMeaningful(filterParams.maxIncome) + ? filterParams.maxIncome + : null, + }; + } + + // income_vouchers is a boolean - true if any voucher filters are present + const hasVoucherFilters = filterPlaceholders( + filterParams.voucherStatuses, + ); + const incomeVouchers = + hasVoucherFilters && hasVoucherFilters.length > 0 ? true : null; + + // Convert age filters to array format expected by FastAPI + let ageArray = null; + if ( + isMeaningful(filterParams.minAge) || + isMeaningful(filterParams.maxAge) + ) { + // Create age range strings + const minAge = filterParams.minAge; + const maxAge = filterParams.maxAge; + if (minAge && maxAge) { + ageArray = [`${minAge}-${maxAge}`]; + } else if (minAge) { + ageArray = [`${minAge}+`]; + } else if (maxAge) { + ageArray = [`0-${maxAge}`]; + } + } + + // Build Household object - only include if at least one field is present + let household = null; + if ( + householdSize || + householdIncome || + incomeVouchers || + accessibilityTypes + ) { + household = { + household_size: householdSize, + household_income: householdIncome, + income_vouchers: incomeVouchers, + accessibility: accessibilityTypes, + }; + } + + // Build Demographics object - only include if at least one field is present + let demographics = null; + if (races || ethnicities || ageArray) { + demographics = { + race: races, + ethnicity: ethnicities, + age: ageArray, + }; + } + + // Build Geography object - only include if at least one field is present + let geography = null; + if (residentialCounties || workCounties) { + geography = { + cities: residentialCounties, + census_tracts: null, + zip_codes: workCounties, + }; + } + + filters = { + date_range: + isMeaningful(filterParams.startDate) || + isMeaningful(filterParams.endDate) + ? { + start_date: filterParams.startDate, + end_date: filterParams.endDate, + } + : null, + household: household, + demographics: demographics, + geography: geography, + }; + } + } + + const response = await axios.post( + `${API_BASE_URL}/api/v1/secure/generate-report`, + filters, + { + headers: { + 'X-API-Key': process.env.FAST_API_KEY, + }, + }, + ); + + // Map FastAPI response to NestJS DataExplorerReport structure + const fastApiData = response.data; + + // Helper to extract value array from metric objects + const extractMetricValue = (metric: any): any[] => { + if (!metric) return []; + // If metric has a 'value' property, return it; otherwise return the metric itself + return metric.value || metric; + }; + + // Transform the response to match our DTO structure + const transformedData = { + dateRange: fastApiData.date_range || 'N/A', + totalProcessedApplications: + fastApiData.total_processed_applications || 0, + totalApplicants: fastApiData.total_applicants, + totalListings: fastApiData.total_listings, + validResponse: fastApiData.is_sufficient || false, + isSufficient: fastApiData.is_sufficient || false, + kAnonScore: fastApiData.k_anonymity_score || 0, + reportErrors: fastApiData.report_errors || [], + products: { + // Extract the value from the cross-tab metric + incomeHouseholdSizeCrossTab: + fastApiData.metrics?.income_household_size_cross_tab?.value || + fastApiData.metrics?.income_household_size_cross_tab || + {}, + // Use race_frequency_inclusive by default (could also use exclusive) + raceFrequencies: extractMetricValue( + fastApiData.metrics?.race_frequency_inclusive, + ), + ethnicityFrequencies: extractMetricValue( + fastApiData.metrics?.ethnicity_frequency, + ), + subsidyOrVoucherTypeFrequencies: extractMetricValue( + fastApiData.metrics?.voucher_usage_frequency, + ), + accessibilityTypeFrequencies: extractMetricValue( + fastApiData.metrics?.accessibility_frequency, + ), + ageFrequencies: extractMetricValue( + fastApiData.metrics?.age_frequency, + ), + residentialLocationFrequencies: extractMetricValue( + fastApiData.metrics?.city_frequency, + ), + languageFrequencies: extractMetricValue( + fastApiData.metrics?.language_frequency, + ), + }, + }; + + return transformedData as DataExplorerReport; + } catch (error) { + console.error('Error fetching report data from FastAPI:', error); + if (axios.isAxiosError(error)) { + // Log detailed validation errors if it's a 422 + if (error.response?.status === 422 && error.response?.data?.detail) { + console.error( + 'FastAPI validation errors:', + JSON.stringify(error.response.data.detail, null, 2), + ); + } + } + throw new NotFoundException('No report data found'); + } + } + + async authorizeAction( + user: User, + action: permissionActions, + jurisdictionId: string, + ): Promise { + await this.permissionService.canOrThrow(user, 'application', action, { + jurisdictionId: jurisdictionId, + }); + } + + /* + this will call the FastAPI endpoint to generate an AI insight + and return the markdown response + it will also check if the user has permission to access the data + if the user does not have permission, it will throw a ForbiddenException + if the user is not authenticated, it will throw a ForbiddenException + */ + async generateInsight( + params: GenerateInsightParams, + req: ExpressRequest, + ): Promise { + const user = mapTo(User, req['user']); + if (!user) { + throw new ForbiddenException(); + } + + if (params.jurisdictionId) { + await this.authorizeAction( + user, + permissionActions.read, + params.jurisdictionId, + ); + } + + const insightData = await this.getInsightFromFastAPI(params); + if (!insightData) { + console.error('No insight data returned from API'); + throw new NotFoundException('No insight data found'); + } + + return mapTo(GenerateInsightResponse, insightData); + } + + async getInsightFromFastAPI( + params: GenerateInsightParams, + ): Promise { + try { + const API_BASE_URL = process.env.FAST_API_URL; + if (!process.env.FAST_API_KEY || !process.env.FAST_API_URL) { + throw new BadRequestException( + 'FastAPI key or URL is not configured in environment variables', + ); + } + const response = await axios.post( + `${API_BASE_URL}/api/v1/secure/generate-insight`, + { + data: params.data, + prompt: params.prompt, + }, + { + headers: { + 'X-API-Key': process.env.FAST_API_KEY, + 'Content-Type': 'application/json', + }, + }, + ); + + return response.data as GenerateInsightResponse; + } catch (error) { + console.error('Error calling FastAPI generate-insight:', error); + throw new NotFoundException('Failed to generate insight'); + } + } +} diff --git a/api/src/services/user.service.ts b/api/src/services/user.service.ts index 32c201a944..94ac564807 100644 --- a/api/src/services/user.service.ts +++ b/api/src/services/user.service.ts @@ -340,6 +340,22 @@ export class UserService { return mapTo(User, res); } + /* + this will update the AI consent status for a user + */ + async updateAIConsent(userId: string, hasConsented: boolean): Promise { + const updatedUser = await this.prisma.userAccounts.update({ + include: views.full, + where: { id: userId }, + data: { + hasConsentedToAI: hasConsented, + aiConsentGivenAt: hasConsented ? new Date() : null, + }, + }); + + return mapTo(User, updatedUser); + } + /* resends a confirmation email or errors if no user matches the incoming email if forPublic is true then we resend a confirmation for a public site user diff --git a/api/test/unit/services/data-explorer.service.spec.ts b/api/test/unit/services/data-explorer.service.spec.ts new file mode 100644 index 0000000000..0f5153ad79 --- /dev/null +++ b/api/test/unit/services/data-explorer.service.spec.ts @@ -0,0 +1,288 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { DataExplorerService } from '../../../src/services/data-explorer.service'; +import { PermissionService } from '../../../src/services/permission.service'; +import { DataExplorerReport } from '../../../src/dtos/applications/data-explorer/products/data-explorer-report.dto'; +import { GenerateInsightResponse } from '../../../src/dtos/applications/data-explorer/generate-insight-response.dto'; +import { ReportProducts } from '../../../src/dtos/applications/data-explorer/products/data-explorer-report-products.dto'; + +describe('DataExplorerService', () => { + let service: DataExplorerService; + + const canOrThrowMock = jest.fn(); + + const mockReportData = { + id: 'mock-report-id', + createdAt: new Date('2023-01-01'), + updatedAt: new Date('2023-01-01'), + dateRange: '01/01/2023 - 12/31/2023', + totalProcessedApplications: 100, + totalApplicants: 95, + totalListings: 5, + validResponse: true, + isSufficient: true, + kAnonScore: 10, + products: { + incomeHouseholdSizeCrossTab: { + '1': { + '0-30 AMI': 45, + '31-50 AMI': 78, + '51-80 AMI': 92, + '81-120 AMI': 34, + }, + '2': { + '0-30 AMI': 67, + '31-50 AMI': 89, + '51-80 AMI': 112, + '81-120 AMI': 56, + }, + '3': { + '0-30 AMI': 82, + '31-50 AMI': 95, + '51-80 AMI': 78, + '81-120 AMI': 45, + }, + '4+': { + '0-30 AMI': 93, + '31-50 AMI': 88, + '51-80 AMI': 65, + '81-120 AMI': 40, + }, + }, + raceFrequencies: [ + { count: 120, percentage: 25.5, race: 'Asian' }, + { count: 85, percentage: 18.1, race: 'Black or African American' }, + { count: 200, percentage: 42.6, race: 'White' }, + { count: 65, percentage: 13.8, race: 'Other' }, + ], + ethnicityFrequencies: [ + { count: 150, percentage: 31.9, ethnicity: 'Hispanic or Latino' }, + { + count: 320, + percentage: 68.1, + ethnicity: 'Not Hispanic or Latino', + }, + ], + subsidyOrVoucherTypeFrequencies: [ + { count: 75, percentage: 16.0, subsidyType: 'Section 8' }, + { count: 25, percentage: 5.3, subsidyType: 'VASH' }, + { count: 370, percentage: 78.7, subsidyType: 'None' }, + ], + accessibilityTypeFrequencies: [ + { + count: 45, + percentage: 9.6, + accessibilityType: 'Wheelchair Accessible', + }, + { + count: 30, + percentage: 6.4, + accessibilityType: 'Hearing Impaired', + }, + { + count: 20, + percentage: 4.3, + accessibilityType: 'Vision Impaired', + }, + { count: 375, percentage: 79.8, accessibilityType: 'None' }, + ], + ageFrequencies: [ + { count: 85, percentage: 18.1, age: '18-24' }, + { count: 120, percentage: 25.5, age: '25-34' }, + { count: 110, percentage: 23.4, age: '35-44' }, + { count: 90, percentage: 19.1, age: '45-54' }, + { count: 65, percentage: 13.8, age: '55+' }, + ], + residentialLocationFrequencies: [ + { count: 180, percentage: 38.3, location: 'Oakland' }, + { count: 120, percentage: 25.5, location: 'Berkeley' }, + { count: 95, percentage: 20.2, location: 'San Francisco' }, + { count: 75, percentage: 16.0, location: 'Other' }, + ], + languageFrequencies: [ + { count: 350, percentage: 74.5, language: 'English' }, + { count: 85, percentage: 18.1, language: 'Spanish' }, + { count: 25, percentage: 5.3, language: 'Chinese' }, + { count: 10, percentage: 2.1, language: 'Other' }, + ], + }, + reportErrors: [], + } as DataExplorerReport; + + beforeAll(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DataExplorerService, + { + provide: PermissionService, + useValue: { + canOrThrow: canOrThrowMock, + }, + }, + ], + }).compile(); + + service = module.get(DataExplorerService); + }); + + describe('generateReport', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + it('should throw a forbidden exception if no user is present in the request', async () => { + await expect( + service.generateReport({ jurisdictionId: 'test-jurisdiction' }, { + user: null, + } as any), + ).rejects.toThrow('Forbidden'); + expect(canOrThrowMock).not.toHaveBeenCalled(); + }); + it('should throw a forbidden exception if user is not authorized', async () => { + canOrThrowMock.mockRejectedValueOnce(new Error('Forbidden')); + await expect( + service.generateReport({ jurisdictionId: 'test-jurisdiction' }, { + user: { id: 'test-user' }, + } as any), + ).rejects.toThrow('Forbidden'); + expect(canOrThrowMock).toHaveBeenCalled(); + }); + it('should return report data if user is authorized', async () => { + jest + .spyOn(service, 'getReportDataFastAPI') + .mockResolvedValue(mockReportData as DataExplorerReport); + canOrThrowMock.mockResolvedValueOnce(true); + + const result = await service.generateReport( + { jurisdictionId: 'test-jurisdiction' }, + { user: { id: 'test-user' } } as any, + ); + + expect(result).toEqual(mockReportData); + expect(canOrThrowMock).toHaveBeenCalledWith( + expect.objectContaining({ id: 'test-user' }), + 'application', + 'read', + { jurisdictionId: 'test-jurisdiction' }, + ); + }); + }); + + describe('generateInsight', () => { + const mockInsightData = { + id: 'mock-insight-id', + createdAt: new Date('2023-01-01'), + updatedAt: new Date('2023-01-01'), + insight: '# Analysis Results\n\nThe data shows significant trends...', + } as GenerateInsightResponse; + + const mockProductsData: ReportProducts = { + incomeHouseholdSizeCrossTab: { + '1': { '0-30 AMI': 45 }, + }, + raceFrequencies: [{ count: 120, percentage: 25.5, race: 'Asian' }], + ethnicityFrequencies: [ + { count: 150, percentage: 31.9, ethnicity: 'Hispanic or Latino' }, + ], + subsidyOrVoucherTypeFrequencies: [ + { count: 75, percentage: 16.0, subsidyType: 'Section 8' }, + ], + accessibilityTypeFrequencies: [ + { + count: 45, + percentage: 9.6, + accessibilityType: 'Wheelchair Accessible', + }, + ], + ageFrequencies: [{ count: 85, percentage: 18.1, age: '18-24' }], + residentialLocationFrequencies: [ + { count: 180, percentage: 38.3, location: 'Oakland' }, + ], + languageFrequencies: [ + { count: 350, percentage: 74.5, language: 'English' }, + ], + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should throw a forbidden exception if no user is present in the request', async () => { + await expect( + service.generateInsight( + { + data: mockProductsData, + prompt: 'Test prompt', + jurisdictionId: 'test-jurisdiction', + }, + { + user: null, + } as any, + ), + ).rejects.toThrow('Forbidden'); + expect(canOrThrowMock).not.toHaveBeenCalled(); + }); + + it('should throw a forbidden exception if user is not authorized', async () => { + canOrThrowMock.mockRejectedValueOnce(new Error('Forbidden')); + await expect( + service.generateInsight( + { + data: mockProductsData, + prompt: 'Test prompt', + jurisdictionId: 'test-jurisdiction', + }, + { user: { id: 'test-user' } } as any, + ), + ).rejects.toThrow('Forbidden'); + expect(canOrThrowMock).toHaveBeenCalled(); + }); + + it('should return insight data if user is authorized', async () => { + jest + .spyOn(service, 'getInsightFromFastAPI') + .mockResolvedValue(mockInsightData as GenerateInsightResponse); + canOrThrowMock.mockResolvedValueOnce(true); + + const result = await service.generateInsight( + { + data: mockProductsData, + prompt: 'Test prompt', + jurisdictionId: 'test-jurisdiction', + }, + { user: { id: 'test-user' } } as any, + ); + + expect(result).toEqual(mockInsightData); + expect(canOrThrowMock).toHaveBeenCalledWith( + expect.objectContaining({ id: 'test-user' }), + 'application', + 'read', + { jurisdictionId: 'test-jurisdiction' }, + ); + }); + + it('should not call authorization if no jurisdictionId is provided', async () => { + jest + .spyOn(service, 'getInsightFromFastAPI') + .mockResolvedValue(mockInsightData as GenerateInsightResponse); + + const result = await service.generateInsight( + { + data: mockProductsData, + prompt: 'Test prompt', + }, + { user: { id: 'test-user' } } as any, + ); + + expect(result).toEqual(mockInsightData); + expect(canOrThrowMock).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/shared-helpers/src/auth/AuthContext.ts b/shared-helpers/src/auth/AuthContext.ts index 5759480e0e..3eeecc668f 100644 --- a/shared-helpers/src/auth/AuthContext.ts +++ b/shared-helpers/src/auth/AuthContext.ts @@ -34,6 +34,7 @@ import { SuccessDTO, LotteryService, LanguagesEnum, + DataExplorerService, FeatureFlagsService, } from "../types/backend-swagger" import { getListingRedirectUrl } from "../utilities/getListingRedirectUrl" @@ -42,6 +43,7 @@ import { useRouter } from "next/router" type ContextProps = { amiChartsService: AmiChartsService applicationsService: ApplicationsService + dataExplorerService: DataExplorerService applicationFlaggedSetsService: ApplicationFlaggedSetsService listingsService: ListingsService jurisdictionsService: JurisdictionsService @@ -221,6 +223,7 @@ export const AuthProvider: FunctionComponent = ({ child const contextValues: ContextProps = { amiChartsService: new AmiChartsService(), applicationsService: new ApplicationsService(), + dataExplorerService: new DataExplorerService(), applicationFlaggedSetsService: new ApplicationFlaggedSetsService(), listingsService: new ListingsService(), jurisdictionsService: new JurisdictionsService(), diff --git a/shared-helpers/src/types/backend-swagger.ts b/shared-helpers/src/types/backend-swagger.ts index de89dfbf3d..031ea34523 100644 --- a/shared-helpers/src/types/backend-swagger.ts +++ b/shared-helpers/src/types/backend-swagger.ts @@ -1955,6 +1955,28 @@ export class UserService { axios(configs, resolve, reject) }) } + /** + * Update AI consent preference + */ + updateAiConsent( + params: { + /** requestBody */ + body?: UserAiConsent + } = {} as any, + options: IRequestOptions = {} + ): Promise { + return new Promise((resolve, reject) => { + let url = basePath + "/user/ai-consent" + + const configs: IRequestConfig = getConfigs("put", "application/json", url, options) + + let data = params.body + + configs.data = data + + axios(configs, resolve, reject) + }) + } /** * Invite partner user */ @@ -2998,6 +3020,99 @@ export class LotteryService { } } +export class DataExplorerService { + /** + * Generate a report + */ + generateReport( + params: { + /** */ + jurisdictionId?: string + /** */ + userId?: string + /** Filter by household size categories */ + householdSize?: any | null[] + /** Minimum household income in USD */ + minIncome?: number + /** Maximum household income in USD */ + maxIncome?: number + /** Area Median Income level categories */ + amiLevels?: any | null[] + /** Housing voucher or subsidy status */ + voucherStatuses?: any | null[] + /** Accessibility accommodation types */ + accessibilityTypes?: any | null[] + /** Racial categories for filtering */ + races?: any | null[] + /** Ethnicity categories for filtering */ + ethnicities?: any | null[] + /** Counties where applicants currently reside */ + applicantResidentialCounties?: any | null[] + /** Counties where applicants work */ + applicantWorkCounties?: any | null[] + /** Minimum age of applicant */ + minAge?: number + /** Maximum age of applicant */ + maxAge?: number + /** Start date for filtering applications (ISO 8601 format) */ + startDate?: string + /** End date for filtering applications (ISO 8601 format) */ + endDate?: string + } = {} as any, + options: IRequestOptions = {} + ): Promise { + return new Promise((resolve, reject) => { + let url = basePath + "/data-explorer/generate-report" + + const configs: IRequestConfig = getConfigs("get", "application/json", url, options) + configs.params = { + jurisdictionId: params["jurisdictionId"], + userId: params["userId"], + householdSize: params["householdSize"], + minIncome: params["minIncome"], + maxIncome: params["maxIncome"], + amiLevels: params["amiLevels"], + voucherStatuses: params["voucherStatuses"], + accessibilityTypes: params["accessibilityTypes"], + races: params["races"], + ethnicities: params["ethnicities"], + applicantResidentialCounties: params["applicantResidentialCounties"], + applicantWorkCounties: params["applicantWorkCounties"], + minAge: params["minAge"], + maxAge: params["maxAge"], + startDate: params["startDate"], + endDate: params["endDate"], + } + + /** 适配ios13，get请求不允许带body */ + + axios(configs, resolve, reject) + }) + } + /** + * Generate AI insights from data + */ + generateInsight( + params: { + /** requestBody */ + body?: GenerateInsightParams + } = {} as any, + options: IRequestOptions = {} + ): Promise { + return new Promise((resolve, reject) => { + let url = basePath + "/data-explorer/generate-insight" + + const configs: IRequestConfig = getConfigs("post", "application/json", url, options) + + let data = params.body + + configs.data = data + + axios(configs, resolve, reject) + }) + } +} + export interface SuccessDTO { /** */ success: boolean @@ -7276,6 +7391,12 @@ export interface User { /** */ agreedToTermsOfService: boolean + /** */ + hasConsentedToAI?: boolean + + /** */ + aiConsentGivenAt?: Date + /** */ hitConfirmationURL?: Date @@ -7302,6 +7423,11 @@ export interface PaginatedUser { meta: PaginationMeta } +export interface UserAiConsent { + /** Whether the user has consented to AI features */ + hasConsented: boolean +} + export interface UserCreate { /** */ firstName: string @@ -7327,6 +7453,9 @@ export interface UserCreate { /** */ agreedToTermsOfService: boolean + /** */ + hasConsentedToAI?: boolean + /** */ favoriteListings?: IdDTO[] @@ -7385,6 +7514,9 @@ export interface UserInvite { /** */ language?: LanguagesEnum + /** */ + hasConsentedToAI?: boolean + /** */ favoriteListings?: IdDTO[] @@ -7450,6 +7582,9 @@ export interface UserUpdate { /** */ agreedToTermsOfService: boolean + /** */ + hasConsentedToAI?: boolean + /** */ favoriteListings?: IdDTO[] @@ -7674,6 +7809,175 @@ export interface PublicLotteryTotal { multiselectQuestionId?: string } +export interface RaceFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Race category */ + race: string +} + +export interface EthnicityFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Ethnicity category */ + ethnicity: string +} + +export interface SubsidyFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Subsidy or voucher type */ + subsidyType: string +} + +export interface AccessibilityFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Accessibility type */ + accessibilityType: string +} + +export interface AgeFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Age range */ + age: string +} + +export interface LocationFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Residential location */ + location: string +} + +export interface LanguageFrequency { + /** Count of occurrences */ + count: number + + /** Percentage of total */ + percentage?: number + + /** Language preference */ + language: string +} + +export interface ReportProducts { + /** Cross-tabulation of income bands by household size. Keys are household sizes, values are income band distributions. */ + incomeHouseholdSizeCrossTab: object + + /** Frequency distribution by race */ + raceFrequencies: RaceFrequency[] + + /** Frequency distribution by ethnicity */ + ethnicityFrequencies: EthnicityFrequency[] + + /** Frequency distribution by subsidy or voucher type */ + subsidyOrVoucherTypeFrequencies: SubsidyFrequency[] + + /** Frequency distribution by accessibility type */ + accessibilityTypeFrequencies: AccessibilityFrequency[] + + /** Frequency distribution by age range */ + ageFrequencies: AgeFrequency[] + + /** Frequency distribution by residential location */ + residentialLocationFrequencies: LocationFrequency[] + + /** Frequency distribution by language preference */ + languageFrequencies: LanguageFrequency[] +} + +export interface DataExplorerReport { + /** */ + id: string + + /** */ + createdAt: Date + + /** */ + updatedAt: Date + + /** Date range for the report */ + dateRange: string + + /** Total number of processed applications */ + totalProcessedApplications: number + + /** Total number of applicants */ + totalApplicants?: number + + /** Total number of listings */ + totalListings?: number + + /** Whether the data passes k-anonymity requirements and has no errors */ + validResponse: boolean + + /** Whether there is sufficient data for analysis (alias for validResponse) */ + isSufficient: boolean + + /** K-anonymity score for the dataset */ + kAnonScore: number + + /** Report data products containing various frequency distributions */ + products: CombinedProductsTypes + + /** Any errors encountered during report generation */ + reportErrors?: string[] +} + +export interface GenerateInsightParams { + /** The current data object containing report products */ + data: CombinedDataTypes + + /** The prompt to send to the AI for generating insights */ + prompt: string + + /** */ + jurisdictionId?: string + + /** */ + userId?: string +} + +export interface GenerateInsightResponse { + /** */ + id: string + + /** */ + createdAt: Date + + /** */ + updatedAt: Date + + /** Markdown-formatted AI-generated insights */ + insight: string +} + export enum FilterAvailabilityEnum { "closedWaitlist" = "closedWaitlist", "comingSoon" = "comingSoon", @@ -8103,3 +8407,5 @@ export enum MfaType { "sms" = "sms", "email" = "email", } +export type CombinedProductsTypes = ReportProducts +export type CombinedDataTypes = ReportProducts diff --git a/sites/partners/next.config.js b/sites/partners/next.config.js index 4e59dbc474..446811503c 100644 --- a/sites/partners/next.config.js +++ b/sites/partners/next.config.js @@ -53,6 +53,8 @@ module.exports = withBundleAnalyzer( process.env.APPLICATION_EXPORT_AS_SPREADSHEET === "TRUE" ? "TRUE" : "", useSecureDownloadPathway: process.env.USE_SECURE_DOWNLOAD_PATHWAY === "TRUE" ? "TRUE" : "", limitClosedListingActions: process.env.LIMIT_CLOSED_LISTING_ACTIONS === "TRUE" ? "TRUE" : "", + geminiAPIKey: process.env.GEMINI_KEY, + enableHousingReports: (process.env.ENABLE_HOUSING_REPORTS || "TRUE") === "TRUE", // Enable housing reports by default }, i18n: { locales: process.env.LANGUAGES ? process.env.LANGUAGES.split(",") : ["en"], diff --git a/sites/partners/package.json b/sites/partners/package.json index 78f7ea9985..6205337d12 100644 --- a/sites/partners/package.json +++ b/sites/partners/package.json @@ -56,6 +56,7 @@ "react-dom": "19.2.3", "react-google-recaptcha-v3": "^1.10.1", "react-hook-form": "^6.15.5", + "recharts": "^2.15.3", "react-map-gl": "^8.1.0", "swr": "^2.1.2", "tailwindcss": "2.2.10" diff --git a/sites/partners/src/components/explore/AIInsightsPanel.tsx b/sites/partners/src/components/explore/AIInsightsPanel.tsx new file mode 100644 index 0000000000..9a3aefd734 --- /dev/null +++ b/sites/partners/src/components/explore/AIInsightsPanel.tsx @@ -0,0 +1,154 @@ +import { Button } from "@bloom-housing/ui-seeds" +import Markdown from "markdown-to-jsx" + +interface AiInsightsPanelProps { + insight?: string + isLoading?: boolean + error?: string | null + onRegenerate?: () => void +} + +const defaultMarkdownContent = ` +# Executive summary + +Significant housing-jobs mismatches exist among East Bay affordable housing applicants, with 52.7% commuting between different cities for work. + +## Key numbers: + +- **Cross-city commuters:** 7,005 applicants (52.7%) +- **Oakland residential dominance:** 5,129 applicants (38.6%) +- **Central Corridor employment concentration:** 3,663 workers (52.2%) + +# Data summary + +## Highest Demand: + +Central Corridor (Oakland-Berkeley-Alameda) with 5,997 applicants (45.4%) + +## Distribution patterns: + +- **Residential:** Central Corridor (45.1%), Peninsula Transition (10.0%), Southern Cluster (11.4%) +- **Employment:** Central Corridor (52.2%), Southern Cluster (22.0%), Peninsula Transition (10.0%) +- **Commute:** 52.7% work outside their residential city, 79.5% live in BART-accessible areas + +# Cross-analysis: + +- Same-city living and working: 6,288 applicants (47.3%) +- Cross-cluster commuting: 4,278 applicants (32.2%) +- Largest commute flow: 1,278 from Central Corridor to Southern Cluster +- BART accessibility: 10,574 applicants (79.5%) +` + +export const AiInsightsPanel = ({ + insight, + isLoading, + error, + onRegenerate, +}: AiInsightsPanelProps) => { + const markdownContent = insight || defaultMarkdownContent + + if (isLoading) { + return ( +
+
+

Generating insights...

+
+ ) + } + + if (error) { + return ( +
+
+ + + +
+

{error}

+ {onRegenerate && } +
+ ) + } + + return ( +
+ {/* Header with regenerate button */} + {onRegenerate && insight && ( +
+ +
+ )} + + {/* Markdown Content */} +
+ + {markdownContent} + +
+ + {/* Chat Interface */} + {/*
+
+
+

+ + Ask for more insights +

+
+ +
+
*/} +
+ ) +} diff --git a/sites/partners/src/components/explore/ChatInterface.tsx b/sites/partners/src/components/explore/ChatInterface.tsx new file mode 100644 index 0000000000..5b562646a4 --- /dev/null +++ b/sites/partners/src/components/explore/ChatInterface.tsx @@ -0,0 +1,142 @@ +import React, { useState, useRef, useEffect } from "react" +import { Button } from "@bloom-housing/ui-seeds" +import { chatWithAI } from "../../lib/ai/conversational-ai" +import { defaultReport as reportData } from "../../lib/explore/data-explorer" +// import Markdown from "react-markdown" + +export interface Message { + id: string + content: string + isUser: boolean + timestamp: Date +} + +const ChatInterface: React.FC = () => { + const [messages, setMessages] = useState([]) + const [inputValue, setInputValue] = useState("") + const [isLoading, setIsLoading] = useState(false) + const messagesEndRef = useRef(null) + const textareaRef = useRef(null) + + const adjustTextareaHeight = () => { + if (textareaRef.current) { + textareaRef.current.style.height = "auto" + textareaRef.current.style.height = `${Math.min(textareaRef.current.scrollHeight, 120)}px` + } + } + + useEffect(() => { + adjustTextareaHeight() + }, [inputValue]) + + const handleSendMessage = async () => { + if (!inputValue.trim() || isLoading) return + + const userMessage: Message = { + id: Date.now().toString(), + content: inputValue.trim(), + isUser: true, + timestamp: new Date(), + } + + const updatedMessages = [...messages, userMessage] + setMessages((prev) => [...prev, userMessage]) + setInputValue("") + setIsLoading(true) + + try { + const aiResponse = await chatWithAI(userMessage.content, updatedMessages, reportData.products) + + const aiMessage: Message = { + id: (Date.now() + 1).toString(), + content: aiResponse, + isUser: false, + timestamp: new Date(), + } + + setMessages((prev) => [...prev, aiMessage]) + } catch (error) { + console.error("Error sending message:", error) + const errorMessage: Message = { + id: (Date.now() + 1).toString(), + content: "Sorry, there was an error processing your message.", + isUser: false, + timestamp: new Date(), + } + setMessages((prev) => [...prev, errorMessage]) + } finally { + setIsLoading(false) + } + } + + const handleKeyPress = async (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault() + await handleSendMessage() + } + } + + console.log("Current messages:", messages) + + return ( +
+ {/* Conversation Log */} +
+ {messages.length === 0 && ( +
Start a conversation
+ )} + {messages.map((message) => ( +
+
+ {message.content} +
+
+ ))} + {isLoading && ( +
+
+
Typing...
+
+
+ )} +
+
+ + {/* Chat Bar */} +
+
+