diff --git a/packages/corsair/core/constants.ts b/packages/corsair/core/constants.ts index b30f1180f..bd6841a6e 100644 --- a/packages/corsair/core/constants.ts +++ b/packages/corsair/core/constants.ts @@ -184,6 +184,7 @@ export const BaseProviders = [ 'tavilymcp', 'teams', 'telegram', + 'textrazor', 'ticktick', 'todoist', 'toggl', @@ -385,6 +386,7 @@ export const ProviderDisplayNames = { tavilymcp: 'Tavily MCP', teams: 'Teams', telegram: 'Telegram', + textrazor: 'TextRazor', ticktick: 'TickTick', todoist: 'Todoist', toggl: 'Toggl', @@ -593,6 +595,7 @@ export type AllProviders = | 'tavilymcp' | 'teams' | 'telegram' + | 'textrazor' | 'ticktick' | 'todoist' | 'toggl' diff --git a/packages/textrazor/api.test.ts b/packages/textrazor/api.test.ts new file mode 100644 index 000000000..1066b0e8a --- /dev/null +++ b/packages/textrazor/api.test.ts @@ -0,0 +1,442 @@ +import { logEventFromContext } from 'corsair/core'; +import { request } from 'corsair/http'; +import { + AccountEndpoints, + AnalysisEndpoints, + ClassifierEndpoints, + DictionaryEndpoints, +} from './endpoints'; +import { + TextrazorEndpointInputSchemas, + TextrazorEndpointOutputSchemas, +} from './endpoints/types'; +import { errorHandlers } from './error-handlers'; +import { + assertTextrazorOk, + TEXTRAZOR_API_BASE, + TextrazorAPIError, + toFormBody, +} from './index'; + +jest.mock('corsair/core', () => ({ + logEventFromContext: jest.fn().mockResolvedValue(undefined), + AuthMissingError: class AuthMissingError extends Error { + constructor(plugin: string, authType: string) { + super(`Missing ${authType} auth for ${plugin}`); + this.name = 'AuthMissingError'; + } + }, +})); + +jest.mock('corsair/http', () => { + class MockApiError extends Error { + status: number; + statusText: string; + body: unknown; + retryAfter: number | undefined; + + constructor( + status: number, + message: string, + statusText = '', + body: unknown = undefined, + retryAfter: number | undefined = undefined, + ) { + super(message); + this.name = 'ApiError'; + this.status = status; + this.statusText = statusText; + this.body = body; + this.retryAfter = retryAfter; + } + } + + return { ApiError: MockApiError, request: jest.fn() }; +}); + +const requestMock = request as unknown as jest.Mock; + +function call(fn: unknown, ctx: unknown, input?: unknown): Promise { + return (fn as (c: unknown, i: unknown) => Promise)(ctx, input); +} + +function createContext() { + return { + key: 'test-key', + db: { + accounts: { upsertByEntityId: jest.fn().mockResolvedValue(undefined) }, + dictionaries: { + upsertByEntityId: jest.fn().mockResolvedValue(undefined), + }, + dictionaryEntries: { + upsertByEntityId: jest.fn().mockResolvedValue(undefined), + }, + categories: { upsertByEntityId: jest.fn().mockResolvedValue(undefined) }, + entities: { upsertByEntityId: jest.fn().mockResolvedValue(undefined) }, + }, + }; +} + +const entityApple = { + id: 0, + matchingTokens: [0, 1], + entityId: 'Apple Inc.', + confidenceScore: 10.2, + wikiLink: 'http://en.wikipedia.org/wiki/Apple_Inc.', + matchedText: 'Apple Inc.', + relevanceScore: 0.9, + entityEnglishId: 'Apple Inc.', + startingPos: 0, + endingPos: 10, + wikidataId: 'Q312', + wikidataTypes: ['Q4830453/business'], + type: ['Organisation', 'Company'], +}; + +describe('TextRazor form encoding', () => { + it('encodes extractors as a comma-separated form field', () => { + expect( + toFormBody({ + extractors: ['entities', 'topics'], + text: 'hello', + 'entities.allowOverlap': true, + }), + ).toBe( + 'extractors=entities%2Ctopics&text=hello&entities.allowOverlap=true', + ); + }); + + it('omits empty arrays and undefined values', () => { + expect( + toFormBody({ extractors: [], text: undefined, url: 'https://x.test' }), + ).toBe('url=https%3A%2F%2Fx.test'); + }); +}); + +describe('TextRazor schemas', () => { + it('requires exactly one of text or url', () => { + const schema = TextrazorEndpointInputSchemas.analyzeContent; + expect( + schema.safeParse({ extractors: ['entities'], text: 'hi' }).success, + ).toBe(true); + expect( + schema.safeParse({ + extractors: ['entities'], + url: 'https://example.com', + }).success, + ).toBe(true); + expect(schema.safeParse({ extractors: ['entities'] }).success).toBe(false); + expect( + schema.safeParse({ + extractors: ['entities'], + text: 'hi', + url: 'https://example.com', + }).success, + ).toBe(false); + }); + + it('parses a live classify payload shape', () => { + TextrazorEndpointOutputSchemas.classifyText.parse({ + ok: true, + time: 0.01671, + response: { + language: 'eng', + languageIsReliable: true, + entities: [entityApple], + categories: [ + { + id: 0, + classifierId: 'textrazor_iab', + categoryId: 'IAB17', + label: 'Sports', + score: 0.66, + }, + ], + }, + }); + }); + + it('parses a live account payload, including planDailyRequestsIncluded', () => { + TextrazorEndpointOutputSchemas.getAccount.parse({ + ok: true, + response: { + plan: 'FREE', + concurrentRequestLimit: 2, + concurrentRequestsUsed: 1, + planDailyRequestsIncluded: 500, + requestsUsedToday: 0, + }, + }); + }); + + it('accepts an empty dictionary list envelope', () => { + TextrazorEndpointOutputSchemas.listDictionaries.parse({ + ok: true, + time: 0.09491, + }); + }); + + it('validates dictionary and classifier list payloads', () => { + expect( + TextrazorEndpointOutputSchemas.listDictionaries.safeParse({ + ok: true, + response: [{ id: 'test_ents', matchType: 'token' }], + }).success, + ).toBe(true); + expect( + TextrazorEndpointOutputSchemas.listDictionaries.safeParse({ + ok: true, + response: { dictionaries: [{ id: 'test_ents' }] }, + }).success, + ).toBe(true); + expect( + TextrazorEndpointOutputSchemas.listDictionaries.safeParse({ + ok: true, + response: 'not-a-list', + }).success, + ).toBe(false); + + expect( + TextrazorEndpointOutputSchemas.listDictionaryEntries.safeParse({ + ok: true, + response: [{ id: 'DEV1', text: 'Bjarne Stroustrup' }], + limit: 20, + offset: 0, + }).success, + ).toBe(true); + expect( + TextrazorEndpointOutputSchemas.listDictionaryEntries.safeParse({ + ok: true, + response: 12, + }).success, + ).toBe(false); + + expect( + TextrazorEndpointOutputSchemas.listClassifierCategories.safeParse({ + ok: true, + response: [{ categoryId: '100', label: 'Golf' }], + }).success, + ).toBe(true); + expect( + TextrazorEndpointOutputSchemas.listClassifierCategories.safeParse({ + ok: true, + response: { categories: [{ categoryId: '100' }] }, + }).success, + ).toBe(true); + expect( + TextrazorEndpointOutputSchemas.listClassifierCategories.safeParse({ + ok: true, + response: true, + }).success, + ).toBe(false); + expect( + TextrazorEndpointOutputSchemas.listDictionaries.safeParse({ + ok: true, + response: [{}], + }).success, + ).toBe(false); + expect( + TextrazorEndpointOutputSchemas.listDictionaryEntries.safeParse({ + ok: true, + response: [{ text: 'no-id' }], + }).success, + ).toBe(false); + expect( + TextrazorEndpointOutputSchemas.listClassifierCategories.safeParse({ + ok: true, + response: [{ label: 'Golf' }], + }).success, + ).toBe(false); + }); +}); + +describe('TextRazor endpoint routing', () => { + beforeEach(() => { + requestMock.mockReset(); + requestMock.mockResolvedValue({ ok: true }); + (logEventFromContext as unknown as jest.Mock).mockReset(); + }); + + it('sends X-TextRazor-Key and never a bearer token', async () => { + requestMock.mockResolvedValue({ + ok: true, + response: { entities: [entityApple] }, + }); + await call(AnalysisEndpoints.analyzeContent, createContext(), { + text: 'Apple Inc. announced a partnership.', + extractors: ['entities', 'topics'], + }); + + const [config, options] = requestMock.mock.calls[0] as [ + { BASE: string; TOKEN?: string; HEADERS: Record }, + { method: string; url: string; body: string; mediaType: string }, + ]; + expect(config.BASE).toBe(TEXTRAZOR_API_BASE); + expect(config.TOKEN).toBeUndefined(); + expect(config.HEADERS['X-TextRazor-Key']).toBe('test-key'); + expect(options.method).toBe('POST'); + expect(options.url).toBe('/'); + expect(options.mediaType).toBe('application/x-www-form-urlencoded'); + expect(options.body).toContain('extractors=entities%2Ctopics'); + expect(options.body).toContain('text=Apple'); + }); + + it('filters extracted entities by score thresholds', async () => { + requestMock.mockResolvedValue({ + ok: true, + response: { + entities: [ + { ...entityApple, relevanceScore: 0.2, confidenceScore: 1 }, + { + ...entityApple, + entityId: 'OpenAI', + relevanceScore: 0.9, + confidenceScore: 8, + }, + ], + }, + }); + const result = (await call( + AnalysisEndpoints.extractEntities, + createContext(), + { + text: 'Apple Inc. announced a partnership with OpenAI.', + minRelevanceScore: 0.5, + minConfidenceScore: 2, + }, + )) as { response?: { entities?: Array<{ entityId?: string }> } }; + expect(result.response?.entities).toEqual([ + expect.objectContaining({ entityId: 'OpenAI' }), + ]); + }); + + it('GETs account/', async () => { + requestMock.mockResolvedValue({ + ok: true, + response: { plan: 'FREE', planDailyRequestsIncluded: 500 }, + }); + await call(AccountEndpoints.get, createContext(), {}); + const [, options] = requestMock.mock.calls[0] as [ + unknown, + { method: string; url: string }, + ]; + expect(options.method).toBe('GET'); + expect(options.url).toBe('account/'); + }); + + it('creates, lists, pages, and deletes dictionaries', async () => { + const ctx = createContext(); + await call(DictionaryEndpoints.create, ctx, { + id: 'test_ents', + matchType: 'token', + caseInsensitive: true, + language: 'eng', + }); + await call(DictionaryEndpoints.list, ctx, {}); + await call(DictionaryEndpoints.get, ctx, { id: 'test_ents' }); + await call(DictionaryEndpoints.listEntries, ctx, { + id: 'test_ents', + limit: 20, + offset: 0, + }); + await call(DictionaryEndpoints.addEntries, ctx, { + id: 'test_ents', + entries: [{ text: 'Bjarne Stroustrup', id: 'DEV2' }], + }); + await call(DictionaryEndpoints.getEntry, ctx, { + id: 'test_ents', + entryId: 'DEV2', + }); + await call(DictionaryEndpoints.deleteEntry, ctx, { + id: 'test_ents', + entryId: 'DEV2', + }); + await call(DictionaryEndpoints.delete, ctx, { id: 'test_ents' }); + + const urls = requestMock.mock.calls.map( + (callArgs) => + (callArgs[1] as { method: string; url: string }).method + + ' ' + + (callArgs[1] as { url: string }).url, + ); + expect(urls).toEqual([ + 'PUT entities/test_ents', + 'GET entities/', + 'GET entities/test_ents', + 'GET entities/test_ents/_all', + 'POST entities/test_ents/', + 'GET entities/test_ents/DEV2', + 'DELETE entities/test_ents/DEV2', + 'DELETE entities/test_ents', + ]); + const listEntries = requestMock.mock.calls[3] as [ + unknown, + { query: { limit: number; offset: number } }, + ]; + expect(listEntries[1].query).toEqual({ limit: 20, offset: 0 }); + }); + + it('creates, pages, and deletes custom classifiers', async () => { + const ctx = createContext(); + await call(ClassifierEndpoints.put, ctx, { + id: 'sport', + categories: [ + { categoryId: '100', label: 'Golf', query: "concept('sport>golf')" }, + ], + }); + await call(ClassifierEndpoints.listCategories, ctx, { + id: 'sport', + limit: 20, + offset: 0, + }); + await call(ClassifierEndpoints.getCategory, ctx, { + id: 'sport', + categoryId: '100', + }); + await call(ClassifierEndpoints.deleteCategory, ctx, { + id: 'sport', + categoryId: '100', + }); + await call(ClassifierEndpoints.delete, ctx, { id: 'sport' }); + + const urls = requestMock.mock.calls.map( + (callArgs) => + (callArgs[1] as { method: string; url: string }).method + + ' ' + + (callArgs[1] as { url: string }).url, + ); + expect(urls).toEqual([ + 'PUT categories/sport', + 'GET categories/sport/_all', + 'GET categories/sport/100', + 'DELETE categories/sport/100', + 'DELETE categories/sport', + ]); + }); + + it('throws when TextRazor returns ok:false', () => { + expect(() => + assertTextrazorOk({ ok: false, error: 'bad document' }), + ).toThrow(TextrazorAPIError); + }); + + it('does not retry auth failures', async () => { + const error = new TextrazorAPIError('Unauthorized'); + (error as { status?: number }).status = 401; + const matched = errorHandlers.AUTH_ERROR.match(error); + expect(matched).toBe(true); + await expect(errorHandlers.AUTH_ERROR.handler()).resolves.toEqual({ + maxRetries: 0, + }); + }); + + it('retries server errors', async () => { + const error = new TextrazorAPIError('Internal Server Error'); + (error as { status?: number }).status = 500; + expect(errorHandlers.SERVER_ERROR.match(error)).toBe(true); + await expect(errorHandlers.SERVER_ERROR.handler()).resolves.toEqual({ + maxRetries: 2, + retryStrategy: 'exponential_backoff', + }); + }); +}); diff --git a/packages/textrazor/client.ts b/packages/textrazor/client.ts new file mode 100644 index 000000000..b07c15d70 --- /dev/null +++ b/packages/textrazor/client.ts @@ -0,0 +1,155 @@ +import type { ApiRequestOptions, OpenAPIConfig } from 'corsair/http'; +import { ApiError, request } from 'corsair/http'; + +export class TextrazorAPIError extends Error { + public readonly status?: number; + public readonly statusText?: string; + public readonly body?: unknown; + public readonly retryAfter?: number; + public readonly rateLimitReset?: number; + public readonly rateLimitRemaining?: number; + public readonly rateLimitLimit?: number; + + constructor(message: string, options?: { cause?: Error; body?: unknown }) { + super(message, options?.cause ? { cause: options.cause } : undefined); + this.name = 'TextrazorAPIError'; + this.body = options?.body; + + if (options?.cause instanceof ApiError) { + this.status = options.cause.status; + this.statusText = options.cause.statusText; + this.body = this.body ?? options.cause.body; + this.retryAfter = options.cause.retryAfter; + this.rateLimitReset = options.cause.rateLimitReset; + this.rateLimitRemaining = options.cause.rateLimitRemaining; + this.rateLimitLimit = options.cause.rateLimitLimit; + } + } +} + +/** @see https://www.textrazor.com/docs/rest */ +export const TEXTRAZOR_API_BASE = 'https://api.textrazor.com'; + +const TEXTRAZOR_ERRORS = { + 400: 'Bad Request', + 401: 'Unauthorized', + 413: 'Request too large', + 429: 'Too Many Requests', + 500: 'Internal Server Error', +}; + +export function appendFormValue( + params: URLSearchParams, + key: string, + value: unknown, +): void { + if (value === undefined || value === null) return; + if (Array.isArray(value)) { + if (value.length === 0) return; + params.append(key, value.map(String).join(',')); + return; + } + if (typeof value === 'boolean') { + params.append(key, value ? 'true' : 'false'); + return; + } + params.append(key, String(value)); +} + +export function toFormBody(fields: Record): string { + const params = new URLSearchParams(); + for (const [key, value] of Object.entries(fields)) { + appendFormValue(params, key, value); + } + return params.toString(); +} + +function parseBody(body: unknown): T { + if (typeof body !== 'string') { + return body as T; + } + const trimmed = body.trim(); + if (trimmed.length === 0) { + return {} as T; + } + try { + return JSON.parse(trimmed) as T; + } catch { + return body as T; + } +} + +function buildConfig(apiKey: string): OpenAPIConfig { + return { + BASE: TEXTRAZOR_API_BASE, + VERSION: '1.0.0', + WITH_CREDENTIALS: false, + CREDENTIALS: 'omit', + HEADERS: { + Accept: 'application/json', + 'X-TextRazor-Key': apiKey, + }, + }; +} + +async function send( + config: OpenAPIConfig, + requestOptions: ApiRequestOptions, +): Promise { + try { + const raw = await request(config, requestOptions); + return parseBody(raw); + } catch (error) { + if (error instanceof Error) { + throw new TextrazorAPIError(error.message, { cause: error }); + } + throw new TextrazorAPIError('Unknown TextRazor API error'); + } +} + +export async function makeTextrazorRequest( + endpoint: string, + apiKey: string, + options: { + method?: 'GET' | 'POST' | 'PUT' | 'DELETE'; + form?: Record; + json?: unknown; + query?: Record; + } = {}, +): Promise { + const method = options.method ?? 'GET'; + const config = buildConfig(apiKey); + const form = + options.form !== undefined ? toFormBody(options.form) : undefined; + const requestOptions: ApiRequestOptions = { + method, + url: endpoint, + query: options.query, + errors: TEXTRAZOR_ERRORS, + body: form ?? options.json, + mediaType: + options.form !== undefined + ? 'application/x-www-form-urlencoded' + : options.json !== undefined + ? 'application/json' + : undefined, + }; + + return send(config, requestOptions); +} + +type Envelope = { + ok?: boolean; + error?: string; + message?: string; +}; + +export function assertTextrazorOk(body: T): T { + if (body.ok === false) { + throw new TextrazorAPIError( + body.error || body.message || 'TextRazor request failed', + { body }, + ); + } + return body; +} diff --git a/packages/textrazor/endpoints/account.ts b/packages/textrazor/endpoints/account.ts new file mode 100644 index 000000000..b62cc5e36 --- /dev/null +++ b/packages/textrazor/endpoints/account.ts @@ -0,0 +1,32 @@ +import type { TextrazorEndpoints } from '../index'; +import { textrazorCall } from './call'; +import { GetAccountInputSchema, GetAccountOutputSchema } from './types'; + +export const get: TextrazorEndpoints['getAccount'] = async (ctx, input) => { + const parsed = GetAccountInputSchema.parse(input ?? {}); + const result = await textrazorCall( + ctx, + 'textrazor.account.get', + 'account/', + 'GET', + parsed, + ); + const output = GetAccountOutputSchema.parse(result); + const account = output.response; + if (account) { + try { + await ctx.db.accounts.upsertByEntityId('current', { + id: 'current', + plan: account.plan, + concurrentRequestLimit: account.concurrentRequestLimit, + concurrentRequestsUsed: account.concurrentRequestsUsed, + planDailyRequestsIncluded: account.planDailyRequestsIncluded, + requestsUsedToday: account.requestsUsedToday, + fetchedAt: new Date(), + }); + } catch (error) { + console.warn('[textrazor] Failed to cache account:', error); + } + } + return output; +}; diff --git a/packages/textrazor/endpoints/analysis.ts b/packages/textrazor/endpoints/analysis.ts new file mode 100644 index 000000000..8bef9f02c --- /dev/null +++ b/packages/textrazor/endpoints/analysis.ts @@ -0,0 +1,121 @@ +import { logEventFromContext } from 'corsair/core'; +import { assertTextrazorOk, makeTextrazorRequest } from '../client'; +import type { TextrazorContext, TextrazorEndpoints } from '../index'; +import { analysisForm } from './call'; +import type { AnalysisResponse } from './types'; +import { + AnalyzeContentInputSchema, + AnalyzeContentOutputSchema, + ClassifyTextInputSchema, + ClassifyTextOutputSchema, + ExtractEntitiesInputSchema, + ExtractEntitiesOutputSchema, +} from './types'; + +type AnalysisIn = { + text?: string; + url?: string; + extractors?: string[]; + minRelevanceScore?: number; + minConfidenceScore?: number; +}; + +async function runAnalysis( + ctx: TextrazorContext, + event: string, + form: Record, + input: AnalysisIn, + parse: (raw: unknown) => AnalysisResponse, +): Promise { + const raw = await makeTextrazorRequest('/', ctx.key, { + method: 'POST', + form, + }); + const result = parse(assertTextrazorOk(raw)); + + if (result.response?.entities) { + const minRel = input.minRelevanceScore; + const minConf = input.minConfidenceScore; + if (minRel !== undefined || minConf !== undefined) { + result.response.entities = result.response.entities.filter((entity) => { + if (minRel !== undefined && (entity.relevanceScore ?? 0) < minRel) { + return false; + } + if (minConf !== undefined && (entity.confidenceScore ?? 0) < minConf) { + return false; + } + return true; + }); + } + + for (const entity of result.response.entities) { + const entityId = entity.entityId ?? entity.matchedText; + if (!entityId) continue; + try { + await ctx.db.entities.upsertByEntityId(entityId, { + id: entityId, + entityId: entity.entityId ?? null, + matchedText: entity.matchedText, + confidenceScore: entity.confidenceScore, + relevanceScore: entity.relevanceScore, + wikiLink: entity.wikiLink ?? null, + wikidataId: entity.wikidataId ?? null, + fetchedAt: new Date(), + }); + } catch (error) { + console.warn('[textrazor] Failed to cache entity:', error); + } + } + } + + await logEventFromContext(ctx, event, input, 'completed'); + return result; +} + +export const analyzeContent: TextrazorEndpoints['analyzeContent'] = async ( + ctx, + input, +) => { + const parsed = AnalyzeContentInputSchema.parse(input); + return runAnalysis( + ctx, + 'textrazor.analysis.analyzeContent', + analysisForm(parsed), + parsed, + (raw) => AnalyzeContentOutputSchema.parse(raw), + ); +}; + +export const classifyText: TextrazorEndpoints['classifyText'] = async ( + ctx, + input, +) => { + const parsed = ClassifyTextInputSchema.parse(input); + return runAnalysis( + ctx, + 'textrazor.analysis.classifyText', + analysisForm({ + ...parsed, + extractors: parsed.extractors ?? ['entities'], + }), + parsed, + (raw) => ClassifyTextOutputSchema.parse(raw), + ); +}; + +export const extractEntities: TextrazorEndpoints['extractEntities'] = async ( + ctx, + input, +) => { + const parsed = ExtractEntitiesInputSchema.parse(input); + return runAnalysis( + ctx, + 'textrazor.analysis.extractEntities', + analysisForm({ + ...parsed, + extractors: parsed.extractors ?? ['entities'], + }), + parsed, + (raw) => ExtractEntitiesOutputSchema.parse(raw), + ); +}; diff --git a/packages/textrazor/endpoints/call.ts b/packages/textrazor/endpoints/call.ts new file mode 100644 index 000000000..b28d21da0 --- /dev/null +++ b/packages/textrazor/endpoints/call.ts @@ -0,0 +1,78 @@ +import { logEventFromContext } from 'corsair/core'; +import { assertTextrazorOk, makeTextrazorRequest } from '../client'; +import type { TextrazorContext } from '../index'; + +export async function textrazorCall( + ctx: TextrazorContext, + event: string, + path: string, + method: 'GET' | 'POST' | 'PUT' | 'DELETE', + input: object, + options: { + form?: Record; + json?: unknown; + query?: Record; + } = {}, +): Promise { + const result = assertTextrazorOk( + await makeTextrazorRequest(path, ctx.key, { + method, + form: options.form, + json: options.json, + query: options.query, + }), + ); + await logEventFromContext( + ctx, + event, + input as Record, + 'completed', + ); + return result; +} + +export function analysisForm(input: { + text?: string; + url?: string; + extractors?: string[]; + classifiers?: string[]; + classifierMaxCategories?: number; + cleanupMode?: string; + cleanupReturnCleaned?: boolean; + cleanupReturnRaw?: boolean; + cleanupUseMetadata?: boolean; + cleanupCleanHtmlPrecision?: number; + cleanupCleanHtmlUseTitle?: boolean; + downloadRunJavascript?: boolean; + downloadUserAgent?: string; + entitiesAllowOverlap?: boolean; + entitiesDictionaries?: string[]; + entitiesFilterDbpediaTypes?: string[]; + entitiesFilterFreebaseTypes?: string[]; + entitiesIncludeAddressPlaces?: boolean; + languageOverride?: string; + rules?: string; +}): Record { + return { + text: input.text, + url: input.url, + extractors: input.extractors, + classifiers: input.classifiers, + 'classifier.maxCategories': input.classifierMaxCategories, + 'cleanup.mode': input.cleanupMode, + 'cleanup.returnCleaned': input.cleanupReturnCleaned, + 'cleanup.returnRaw': input.cleanupReturnRaw, + 'cleanup.useMetadata': input.cleanupUseMetadata, + 'cleanup.cleanHTML.precision': input.cleanupCleanHtmlPrecision, + 'cleanup.cleanHTML.useTitle': input.cleanupCleanHtmlUseTitle, + 'download.runJavascript': input.downloadRunJavascript, + 'download.userAgent': input.downloadUserAgent, + 'entities.allowOverlap': input.entitiesAllowOverlap, + 'entities.dictionaries': input.entitiesDictionaries, + 'entities.filterDbpediaTypes': input.entitiesFilterDbpediaTypes, + 'entities.filterFreebaseTypes': input.entitiesFilterFreebaseTypes, + 'entities.includeAddressPlaces': input.entitiesIncludeAddressPlaces, + languageOverride: input.languageOverride, + rules: input.rules, + }; +} diff --git a/packages/textrazor/endpoints/classifiers.ts b/packages/textrazor/endpoints/classifiers.ts new file mode 100644 index 000000000..73f4be2f2 --- /dev/null +++ b/packages/textrazor/endpoints/classifiers.ts @@ -0,0 +1,111 @@ +import type { TextrazorEndpoints } from '../index'; +import { textrazorCall } from './call'; +import { + DeleteClassifierCategoryInputSchema, + DeleteClassifierCategoryOutputSchema, + DeleteClassifierInputSchema, + DeleteClassifierOutputSchema, + GetClassifierCategoryInputSchema, + GetClassifierCategoryOutputSchema, + ListClassifierCategoriesInputSchema, + ListClassifierCategoriesOutputSchema, + PutClassifierInputSchema, + PutClassifierOutputSchema, +} from './types'; + +function classifierPath(id: string): string { + return `categories/${encodeURIComponent(id)}`; +} + +export const put: TextrazorEndpoints['putClassifier'] = async (ctx, input) => { + const parsed = PutClassifierInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.classifiers.put', + classifierPath(parsed.id), + 'PUT', + parsed, + { json: parsed.categories }, + ); + const output = PutClassifierOutputSchema.parse(result); + for (const category of parsed.categories) { + try { + await ctx.db.categories.upsertByEntityId( + `${parsed.id}:${category.categoryId}`, + { + id: `${parsed.id}:${category.categoryId}`, + categoryId: category.categoryId, + label: category.label, + query: category.query, + classifierId: parsed.id, + fetchedAt: new Date(), + }, + ); + } catch (error) { + console.warn('[textrazor] Failed to cache category:', error); + } + } + return output; +}; + +export const remove: TextrazorEndpoints['deleteClassifier'] = async ( + ctx, + input, +) => { + const parsed = DeleteClassifierInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.classifiers.delete', + classifierPath(parsed.id), + 'DELETE', + parsed, + ); + return DeleteClassifierOutputSchema.parse(result); +}; + +export const listCategories: TextrazorEndpoints['listClassifierCategories'] = + async (ctx, input) => { + const parsed = ListClassifierCategoriesInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.classifiers.listCategories', + `${classifierPath(parsed.id)}/_all`, + 'GET', + parsed, + { + query: { + limit: parsed.limit, + offset: parsed.offset, + }, + }, + ); + return ListClassifierCategoriesOutputSchema.parse(result); + }; + +export const getCategory: TextrazorEndpoints['getClassifierCategory'] = async ( + ctx, + input, +) => { + const parsed = GetClassifierCategoryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.classifiers.getCategory', + `${classifierPath(parsed.id)}/${encodeURIComponent(parsed.categoryId)}`, + 'GET', + parsed, + ); + return GetClassifierCategoryOutputSchema.parse(result); +}; + +export const deleteCategory: TextrazorEndpoints['deleteClassifierCategory'] = + async (ctx, input) => { + const parsed = DeleteClassifierCategoryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.classifiers.deleteCategory', + `${classifierPath(parsed.id)}/${encodeURIComponent(parsed.categoryId)}`, + 'DELETE', + parsed, + ); + return DeleteClassifierCategoryOutputSchema.parse(result); + }; diff --git a/packages/textrazor/endpoints/dictionaries.ts b/packages/textrazor/endpoints/dictionaries.ts new file mode 100644 index 000000000..6aa11f79f --- /dev/null +++ b/packages/textrazor/endpoints/dictionaries.ts @@ -0,0 +1,167 @@ +import type { TextrazorEndpoints } from '../index'; +import { textrazorCall } from './call'; +import { + AddDictionaryEntriesInputSchema, + AddDictionaryEntriesOutputSchema, + CreateDictionaryInputSchema, + CreateDictionaryOutputSchema, + DeleteDictionaryEntryInputSchema, + DeleteDictionaryEntryOutputSchema, + DeleteDictionaryInputSchema, + DeleteDictionaryOutputSchema, + GetDictionaryEntryInputSchema, + GetDictionaryEntryOutputSchema, + GetDictionaryInputSchema, + GetDictionaryOutputSchema, + ListDictionariesInputSchema, + ListDictionariesOutputSchema, + ListDictionaryEntriesInputSchema, + ListDictionaryEntriesOutputSchema, +} from './types'; + +function dictionaryPath(id: string): string { + return `entities/${encodeURIComponent(id)}`; +} + +export const create: TextrazorEndpoints['createDictionary'] = async ( + ctx, + input, +) => { + const parsed = CreateDictionaryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.create', + dictionaryPath(parsed.id), + 'PUT', + parsed, + { + json: { + matchType: parsed.matchType, + caseInsensitive: parsed.caseInsensitive, + language: parsed.language, + }, + }, + ); + const output = CreateDictionaryOutputSchema.parse(result); + try { + await ctx.db.dictionaries.upsertByEntityId(parsed.id, { + id: parsed.id, + matchType: parsed.matchType, + caseInsensitive: parsed.caseInsensitive, + language: parsed.language, + fetchedAt: new Date(), + }); + } catch (error) { + console.warn('[textrazor] Failed to cache dictionary:', error); + } + return output; +}; + +export const list: TextrazorEndpoints['listDictionaries'] = async ( + ctx, + input, +) => { + const parsed = ListDictionariesInputSchema.parse(input ?? {}); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.list', + 'entities/', + 'GET', + parsed, + ); + return ListDictionariesOutputSchema.parse(result); +}; + +export const get: TextrazorEndpoints['getDictionary'] = async (ctx, input) => { + const parsed = GetDictionaryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.get', + dictionaryPath(parsed.id), + 'GET', + parsed, + ); + return GetDictionaryOutputSchema.parse(result); +}; + +export const remove: TextrazorEndpoints['deleteDictionary'] = async ( + ctx, + input, +) => { + const parsed = DeleteDictionaryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.delete', + dictionaryPath(parsed.id), + 'DELETE', + parsed, + ); + return DeleteDictionaryOutputSchema.parse(result); +}; + +export const listEntries: TextrazorEndpoints['listDictionaryEntries'] = async ( + ctx, + input, +) => { + const parsed = ListDictionaryEntriesInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.listEntries', + `${dictionaryPath(parsed.id)}/_all`, + 'GET', + parsed, + { + query: { + limit: parsed.limit, + offset: parsed.offset, + }, + }, + ); + return ListDictionaryEntriesOutputSchema.parse(result); +}; + +export const addEntries: TextrazorEndpoints['addDictionaryEntries'] = async ( + ctx, + input, +) => { + const parsed = AddDictionaryEntriesInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.addEntries', + `${dictionaryPath(parsed.id)}/`, + 'POST', + parsed, + { json: parsed.entries }, + ); + return AddDictionaryEntriesOutputSchema.parse(result); +}; + +export const getEntry: TextrazorEndpoints['getDictionaryEntry'] = async ( + ctx, + input, +) => { + const parsed = GetDictionaryEntryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.getEntry', + `${dictionaryPath(parsed.id)}/${encodeURIComponent(parsed.entryId)}`, + 'GET', + parsed, + ); + return GetDictionaryEntryOutputSchema.parse(result); +}; + +export const deleteEntry: TextrazorEndpoints['deleteDictionaryEntry'] = async ( + ctx, + input, +) => { + const parsed = DeleteDictionaryEntryInputSchema.parse(input); + const result = await textrazorCall( + ctx, + 'textrazor.dictionaries.deleteEntry', + `${dictionaryPath(parsed.id)}/${encodeURIComponent(parsed.entryId)}`, + 'DELETE', + parsed, + ); + return DeleteDictionaryEntryOutputSchema.parse(result); +}; diff --git a/packages/textrazor/endpoints/index.ts b/packages/textrazor/endpoints/index.ts new file mode 100644 index 000000000..a56d9725f --- /dev/null +++ b/packages/textrazor/endpoints/index.ts @@ -0,0 +1,35 @@ +import * as Account from './account'; +import * as Analysis from './analysis'; +import * as Classifiers from './classifiers'; +import * as Dictionaries from './dictionaries'; + +export const AnalysisEndpoints = { + analyzeContent: Analysis.analyzeContent, + classifyText: Analysis.classifyText, + extractEntities: Analysis.extractEntities, +}; + +export const AccountEndpoints = { + get: Account.get, +}; + +export const DictionaryEndpoints = { + create: Dictionaries.create, + list: Dictionaries.list, + get: Dictionaries.get, + delete: Dictionaries.remove, + listEntries: Dictionaries.listEntries, + addEntries: Dictionaries.addEntries, + getEntry: Dictionaries.getEntry, + deleteEntry: Dictionaries.deleteEntry, +}; + +export const ClassifierEndpoints = { + put: Classifiers.put, + delete: Classifiers.remove, + listCategories: Classifiers.listCategories, + getCategory: Classifiers.getCategory, + deleteCategory: Classifiers.deleteCategory, +}; + +export * from './types'; diff --git a/packages/textrazor/endpoints/types.ts b/packages/textrazor/endpoints/types.ts new file mode 100644 index 000000000..d3aba1a85 --- /dev/null +++ b/packages/textrazor/endpoints/types.ts @@ -0,0 +1,494 @@ +import { z } from 'zod'; + +const EXTRACTORS = [ + 'entities', + 'topics', + 'words', + 'phrases', + 'dependency-trees', + 'relations', + 'entailments', + 'senses', + 'spelling', +] as const; + +export const ExtractorSchema = z.enum(EXTRACTORS); + +const CleanupModeSchema = z.enum(['raw', 'stripTags', 'cleanHTML']); +const MatchTypeSchema = z.enum(['token', 'stem']); + +export const EntitySchema = z + .object({ + id: z.number().optional(), + entityId: z.string().nullable().optional(), + entityEnglishId: z.string().nullable().optional(), + matchedText: z.string().optional(), + matchingTokens: z.array(z.number()).optional(), + startingPos: z.number().optional(), + endingPos: z.number().optional(), + confidenceScore: z.number().optional(), + relevanceScore: z.number().optional(), + type: z.array(z.string()).optional(), + freebaseTypes: z.array(z.string()).optional(), + freebaseId: z.string().nullable().optional(), + wikiLink: z.string().nullable().optional(), + wikidataId: z.string().nullable().optional(), + wikidataTypes: z.array(z.string()).optional(), + customEntityId: z.string().optional(), + sourceId: z.string().optional(), + data: z.unknown().optional(), + crunchbaseId: z.string().optional(), + lei: z.string().optional(), + figi: z.string().optional(), + permid: z.string().optional(), + unit: z.string().optional(), + }) + .loose(); + +export const TopicSchema = z + .object({ + id: z.number().optional(), + label: z.string().optional(), + score: z.number().optional(), + wikiLink: z.string().nullable().optional(), + wikidataId: z.string().nullable().optional(), + }) + .loose(); + +export const ScoredCategorySchema = z + .object({ + id: z.number().optional(), + categoryId: z.string().optional(), + label: z.string().optional(), + score: z.number().optional(), + classifierId: z.string().optional(), + }) + .loose(); + +export const WordSchema = z + .object({ + position: z.number().optional(), + startingPos: z.number().optional(), + endingPos: z.number().optional(), + token: z.string().optional(), + stem: z.string().optional(), + lemma: z.string().optional(), + partOfSpeech: z.string().optional(), + parentPosition: z.number().nullable().optional(), + relationToParent: z.string().nullable().optional(), + senses: z.array(z.unknown()).optional(), + spellingSuggestions: z.array(z.unknown()).optional(), + }) + .loose(); + +export const SentenceSchema = z + .object({ + position: z.number().optional(), + words: z.array(WordSchema).optional(), + }) + .loose(); + +export const NounPhraseSchema = z + .object({ + wordPositions: z.array(z.number()).optional(), + }) + .loose(); + +export const RelationParamSchema = z + .object({ + wordPositions: z.array(z.number()).optional(), + relation: z.string().optional(), + }) + .loose(); + +export const RelationSchema = z + .object({ + wordPositions: z.array(z.number()).optional(), + params: z.array(RelationParamSchema).optional(), + }) + .loose(); + +export const PropertySchema = z + .object({ + wordPositions: z.array(z.number()).optional(), + propertyPositions: z.array(z.number()).optional(), + }) + .loose(); + +export const EntailmentSchema = z + .object({ + wordPositions: z.array(z.number()).optional(), + score: z.number().optional(), + priorScore: z.number().optional(), + contextScore: z.number().optional(), + entailedTree: z.unknown().optional(), + }) + .loose(); + +export const AnalysisPayloadSchema = z + .object({ + language: z.string().optional(), + languageIsReliable: z.boolean().optional(), + cleanedText: z.string().optional(), + rawText: z.string().optional(), + customAnnotationOutput: z.unknown().optional(), + matchingRules: z.array(z.string()).optional(), + entities: z.array(EntitySchema).optional(), + topics: z.array(TopicSchema).optional(), + coarseTopics: z.array(TopicSchema).optional(), + categories: z.array(ScoredCategorySchema).optional(), + sentences: z.array(SentenceSchema).optional(), + nounPhrases: z.array(NounPhraseSchema).optional(), + relations: z.array(RelationSchema).optional(), + properties: z.array(PropertySchema).optional(), + entailments: z.array(EntailmentSchema).optional(), + }) + .loose(); + +export const AnalysisResponseSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: AnalysisPayloadSchema.optional(), + }) + .loose(); + +const documentSource = { + text: z.string().min(1).optional(), + url: z.string().url().optional(), +}; + +function requireTextOrUrl>( + schema: T, +) { + return schema.refine((v) => Boolean(v.text) !== Boolean(v.url), { + message: 'Provide exactly one of text or url', + }); +} + +const analysisOptions = { + ...documentSource, + cleanupMode: CleanupModeSchema.optional(), + cleanupReturnCleaned: z.boolean().optional(), + cleanupReturnRaw: z.boolean().optional(), + cleanupUseMetadata: z.boolean().optional(), + cleanupCleanHtmlPrecision: z + .union([z.literal(1), z.literal(2), z.literal(3)]) + .optional(), + cleanupCleanHtmlUseTitle: z.boolean().optional(), + downloadRunJavascript: z.boolean().optional(), + downloadUserAgent: z.string().optional(), + entitiesAllowOverlap: z.boolean().optional(), + entitiesDictionaries: z.array(z.string().min(1)).optional(), + entitiesFilterDbpediaTypes: z.array(z.string().min(1)).optional(), + entitiesFilterFreebaseTypes: z.array(z.string().min(1)).optional(), + entitiesIncludeAddressPlaces: z.boolean().optional(), + languageOverride: z.string().min(2).optional(), + rules: z.string().optional(), + classifiers: z.array(z.string().min(1)).optional(), + classifierMaxCategories: z.number().int().positive().optional(), +}; + +export const AnalyzeContentInputSchema = requireTextOrUrl( + z.object({ + ...analysisOptions, + extractors: z.array(ExtractorSchema).min(1), + }), +); + +export const ClassifyTextInputSchema = requireTextOrUrl( + z.object({ + ...analysisOptions, + extractors: z.array(ExtractorSchema).optional(), + classifiers: z.array(z.string().min(1)).min(1), + }), +); + +export const ExtractEntitiesInputSchema = requireTextOrUrl( + z.object({ + ...analysisOptions, + extractors: z.array(ExtractorSchema).optional(), + minRelevanceScore: z.number().min(0).max(1).optional(), + minConfidenceScore: z.number().min(0).optional(), + }), +); + +export type AnalysisResponse = z.output; + +export const AnalyzeContentOutputSchema = AnalysisResponseSchema; +export const ClassifyTextOutputSchema = AnalysisResponseSchema; +export const ExtractEntitiesOutputSchema = AnalysisResponseSchema; + +export const AccountSchema = z + .object({ + plan: z.string().optional(), + concurrentRequestLimit: z.number().optional(), + concurrentRequestsUsed: z.number().optional(), + planDailyRequestsIncluded: z.number().optional(), + requestsUsedToday: z.number().optional(), + }) + .loose(); + +export const GetAccountInputSchema = z.object({}); +export const GetAccountOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: AccountSchema.optional(), + }) + .loose(); + +export const DictionarySchema = z + .object({ + id: z.string().min(1), + matchType: MatchTypeSchema.optional(), + caseInsensitive: z.boolean().optional(), + language: z.string().optional(), + }) + .loose(); + +export const DictionaryEntrySchema = z + .object({ + id: z.string().min(1), + text: z.string().optional(), + data: z.record(z.string(), z.array(z.string())).optional(), + }) + .loose(); + +export const CreateDictionaryInputSchema = z.object({ + id: z.string().min(1), + matchType: MatchTypeSchema.optional(), + caseInsensitive: z.boolean().optional(), + language: z.string().optional(), +}); +export const CreateDictionaryOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: DictionarySchema.optional(), + }) + .loose(); + +export const ListDictionariesInputSchema = z.object({}); +export const ListDictionariesOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: z + .union([ + z.array(DictionarySchema), + z + .object({ + dictionaries: z.array(DictionarySchema), + }) + .loose(), + ]) + .optional(), + dictionaries: z.array(DictionarySchema).optional(), + }) + .loose(); + +export const GetDictionaryInputSchema = z.object({ + id: z.string().min(1), +}); +export const GetDictionaryOutputSchema = CreateDictionaryOutputSchema; + +export const DeleteDictionaryInputSchema = z.object({ + id: z.string().min(1), +}); +export const DeleteDictionaryOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + }) + .loose(); + +export const ListDictionaryEntriesInputSchema = z.object({ + id: z.string().min(1), + limit: z.number().int().positive().optional(), + offset: z.number().int().min(0).optional(), +}); +export const ListDictionaryEntriesOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: z + .union([ + z.array(DictionaryEntrySchema), + z + .object({ + entries: z.array(DictionaryEntrySchema), + }) + .loose(), + ]) + .optional(), + limit: z.number().optional(), + offset: z.number().optional(), + total: z.number().optional(), + }) + .loose(); + +export const AddDictionaryEntriesInputSchema = z.object({ + id: z.string().min(1), + entries: z + .array( + z.object({ + id: z.string().optional(), + text: z.string().min(1), + data: z.record(z.string(), z.array(z.string())).optional(), + }), + ) + .min(1), +}); +export const AddDictionaryEntriesOutputSchema = DeleteDictionaryOutputSchema; + +export const GetDictionaryEntryInputSchema = z.object({ + id: z.string().min(1), + entryId: z.string().min(1), +}); +export const GetDictionaryEntryOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: DictionaryEntrySchema.optional(), + }) + .loose(); + +export const DeleteDictionaryEntryInputSchema = GetDictionaryEntryInputSchema; +export const DeleteDictionaryEntryOutputSchema = DeleteDictionaryOutputSchema; + +export const ClassifierCategoryInputSchema = z.object({ + categoryId: z.string().min(1), + label: z.string().optional(), + query: z.string().min(1), +}); + +export const PutClassifierInputSchema = z.object({ + id: z.string().min(1), + categories: z.array(ClassifierCategoryInputSchema).min(1), +}); +export const PutClassifierOutputSchema = DeleteDictionaryOutputSchema; + +export const DeleteClassifierInputSchema = z.object({ + id: z.string().min(1), +}); +export const DeleteClassifierOutputSchema = DeleteDictionaryOutputSchema; + +export const ListClassifierCategoriesInputSchema = z.object({ + id: z.string().min(1), + limit: z.number().int().positive().optional(), + offset: z.number().int().min(0).optional(), +}); +export const ClassifierCategorySchema = z + .object({ + categoryId: z.string().min(1), + label: z.string().optional(), + query: z.string().optional(), + }) + .loose(); + +export const ListClassifierCategoriesOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: z + .union([ + z.array(ClassifierCategorySchema), + z + .object({ + categories: z.array(ClassifierCategorySchema), + }) + .loose(), + ]) + .optional(), + limit: z.number().optional(), + offset: z.number().optional(), + total: z.number().optional(), + }) + .loose(); + +export const GetClassifierCategoryInputSchema = z.object({ + id: z.string().min(1), + categoryId: z.string().min(1), +}); +export const GetClassifierCategoryOutputSchema = z + .object({ + ok: z.boolean().optional(), + time: z.union([z.number(), z.string()]).optional(), + error: z.string().optional(), + message: z.string().optional(), + response: ClassifierCategorySchema.optional(), + }) + .loose(); + +export const DeleteClassifierCategoryInputSchema = + GetClassifierCategoryInputSchema; +export const DeleteClassifierCategoryOutputSchema = + DeleteDictionaryOutputSchema; + +export const TextrazorEndpointInputSchemas = { + analyzeContent: AnalyzeContentInputSchema, + classifyText: ClassifyTextInputSchema, + extractEntities: ExtractEntitiesInputSchema, + getAccount: GetAccountInputSchema, + createDictionary: CreateDictionaryInputSchema, + listDictionaries: ListDictionariesInputSchema, + getDictionary: GetDictionaryInputSchema, + deleteDictionary: DeleteDictionaryInputSchema, + listDictionaryEntries: ListDictionaryEntriesInputSchema, + addDictionaryEntries: AddDictionaryEntriesInputSchema, + getDictionaryEntry: GetDictionaryEntryInputSchema, + deleteDictionaryEntry: DeleteDictionaryEntryInputSchema, + putClassifier: PutClassifierInputSchema, + deleteClassifier: DeleteClassifierInputSchema, + listClassifierCategories: ListClassifierCategoriesInputSchema, + getClassifierCategory: GetClassifierCategoryInputSchema, + deleteClassifierCategory: DeleteClassifierCategoryInputSchema, +}; + +export const TextrazorEndpointOutputSchemas = { + analyzeContent: AnalyzeContentOutputSchema, + classifyText: ClassifyTextOutputSchema, + extractEntities: ExtractEntitiesOutputSchema, + getAccount: GetAccountOutputSchema, + createDictionary: CreateDictionaryOutputSchema, + listDictionaries: ListDictionariesOutputSchema, + getDictionary: GetDictionaryOutputSchema, + deleteDictionary: DeleteDictionaryOutputSchema, + listDictionaryEntries: ListDictionaryEntriesOutputSchema, + addDictionaryEntries: AddDictionaryEntriesOutputSchema, + getDictionaryEntry: GetDictionaryEntryOutputSchema, + deleteDictionaryEntry: DeleteDictionaryEntryOutputSchema, + putClassifier: PutClassifierOutputSchema, + deleteClassifier: DeleteClassifierOutputSchema, + listClassifierCategories: ListClassifierCategoriesOutputSchema, + getClassifierCategory: GetClassifierCategoryOutputSchema, + deleteClassifierCategory: DeleteClassifierCategoryOutputSchema, +}; + +export type TextrazorEndpointInputs = { + [K in keyof typeof TextrazorEndpointInputSchemas]: z.input< + (typeof TextrazorEndpointInputSchemas)[K] + >; +}; +export type TextrazorEndpointOutputs = { + [K in keyof typeof TextrazorEndpointOutputSchemas]: z.output< + (typeof TextrazorEndpointOutputSchemas)[K] + >; +}; diff --git a/packages/textrazor/error-handlers.ts b/packages/textrazor/error-handlers.ts new file mode 100644 index 000000000..d01fe2dd2 --- /dev/null +++ b/packages/textrazor/error-handlers.ts @@ -0,0 +1,74 @@ +import type { CorsairErrorHandler } from 'corsair/core'; +import type { TextrazorAPIError } from './client'; + +function getStatus(error: Error): number | undefined { + return (error as Partial).status; +} + +function getRetryAfter(error: Error): number | undefined { + return (error as Partial).retryAfter; +} + +export const errorHandlers = { + VALIDATION_ERROR: { + match: (error: Error) => error.name === 'ZodError', + handler: async () => ({ maxRetries: 0 }), + }, + RATE_LIMIT_ERROR: { + match: (error: Error) => { + if (getStatus(error) === 429) return true; + const msg = error.message.toLowerCase(); + return msg.includes('429') || msg.includes('rate limit'); + }, + handler: async (error: Error) => ({ + maxRetries: 3, + retryStrategy: 'exponential_backoff' as const, + headersRetryAfterMs: getRetryAfter(error), + }), + }, + AUTH_ERROR: { + match: (error: Error) => { + if (getStatus(error) === 401) return true; + const msg = error.message.toLowerCase(); + return ( + msg.includes('unauthorized') || + msg.includes('invalid api key') || + msg.includes('used up its quota') + ); + }, + handler: async () => ({ maxRetries: 0 }), + }, + NOT_FOUND_ERROR: { + match: (error: Error) => { + if (getStatus(error) === 404) return true; + const msg = error.message.toLowerCase(); + return msg.includes('404') || msg.includes('not found'); + }, + handler: async () => ({ maxRetries: 0 }), + }, + BAD_REQUEST_ERROR: { + match: (error: Error) => { + const status = getStatus(error); + if (status === 400 || status === 413) return true; + const msg = error.message.toLowerCase(); + return msg.includes('bad request') || msg.includes('request too large'); + }, + handler: async () => ({ maxRetries: 0 }), + }, + SERVER_ERROR: { + match: (error: Error) => { + const status = getStatus(error); + if (status !== undefined && status >= 500) return true; + const msg = error.message.toLowerCase(); + return msg.includes('503') || msg.includes('server error'); + }, + handler: async () => ({ + maxRetries: 2, + retryStrategy: 'exponential_backoff' as const, + }), + }, + DEFAULT: { + match: () => true, + handler: async () => ({ maxRetries: 0 }), + }, +} satisfies CorsairErrorHandler; diff --git a/packages/textrazor/index.ts b/packages/textrazor/index.ts new file mode 100644 index 000000000..39a0a6aa0 --- /dev/null +++ b/packages/textrazor/index.ts @@ -0,0 +1,346 @@ +import type { + AuthTypes, + BindEndpoints, + CorsairEndpoint, + CorsairErrorHandler, + CorsairPlugin, + CorsairPluginContext, + KeyBuilderContext, + PickAuth, + PluginAuthConfig, + PluginPermissionsConfig, + RequiredPluginEndpointMeta, + RequiredPluginEndpointSchemas, +} from 'corsair/core'; +import { AuthMissingError } from 'corsair/core'; +import { + AccountEndpoints, + AnalysisEndpoints, + ClassifierEndpoints, + DictionaryEndpoints, +} from './endpoints'; +import type { + TextrazorEndpointInputs, + TextrazorEndpointOutputs, +} from './endpoints/types'; +import { + TextrazorEndpointInputSchemas, + TextrazorEndpointOutputSchemas, +} from './endpoints/types'; +import { errorHandlers } from './error-handlers'; +import { TextrazorSchema } from './schema'; + +export type TextrazorPluginOptions = { + authType?: PickAuth<'api_key'>; + key?: string; + hooks?: InternalTextrazorPlugin['hooks']; + errorHandlers?: CorsairErrorHandler; + permissions?: PluginPermissionsConfig; +}; + +export type TextrazorContext = CorsairPluginContext< + typeof TextrazorSchema, + TextrazorPluginOptions +>; + +export type TextrazorKeyBuilderContext = + KeyBuilderContext; + +export type TextrazorBoundEndpoints = BindEndpoints< + typeof textrazorEndpointsNested +>; + +type TextrazorEndpoint = + CorsairEndpoint< + TextrazorContext, + TextrazorEndpointInputs[K], + TextrazorEndpointOutputs[K] + >; + +export type TextrazorEndpoints = { + analyzeContent: TextrazorEndpoint<'analyzeContent'>; + classifyText: TextrazorEndpoint<'classifyText'>; + extractEntities: TextrazorEndpoint<'extractEntities'>; + getAccount: TextrazorEndpoint<'getAccount'>; + createDictionary: TextrazorEndpoint<'createDictionary'>; + listDictionaries: TextrazorEndpoint<'listDictionaries'>; + getDictionary: TextrazorEndpoint<'getDictionary'>; + deleteDictionary: TextrazorEndpoint<'deleteDictionary'>; + listDictionaryEntries: TextrazorEndpoint<'listDictionaryEntries'>; + addDictionaryEntries: TextrazorEndpoint<'addDictionaryEntries'>; + getDictionaryEntry: TextrazorEndpoint<'getDictionaryEntry'>; + deleteDictionaryEntry: TextrazorEndpoint<'deleteDictionaryEntry'>; + putClassifier: TextrazorEndpoint<'putClassifier'>; + deleteClassifier: TextrazorEndpoint<'deleteClassifier'>; + listClassifierCategories: TextrazorEndpoint<'listClassifierCategories'>; + getClassifierCategory: TextrazorEndpoint<'getClassifierCategory'>; + deleteClassifierCategory: TextrazorEndpoint<'deleteClassifierCategory'>; +}; + +const textrazorEndpointsNested = { + analysis: { + analyzeContent: AnalysisEndpoints.analyzeContent, + classifyText: AnalysisEndpoints.classifyText, + extractEntities: AnalysisEndpoints.extractEntities, + }, + account: { + get: AccountEndpoints.get, + }, + dictionaries: { + create: DictionaryEndpoints.create, + list: DictionaryEndpoints.list, + get: DictionaryEndpoints.get, + delete: DictionaryEndpoints.delete, + listEntries: DictionaryEndpoints.listEntries, + addEntries: DictionaryEndpoints.addEntries, + getEntry: DictionaryEndpoints.getEntry, + deleteEntry: DictionaryEndpoints.deleteEntry, + }, + classifiers: { + put: ClassifierEndpoints.put, + delete: ClassifierEndpoints.delete, + listCategories: ClassifierEndpoints.listCategories, + getCategory: ClassifierEndpoints.getCategory, + deleteCategory: ClassifierEndpoints.deleteCategory, + }, +} as const; + +export const textrazorEndpointSchemas = { + 'analysis.analyzeContent': { + input: TextrazorEndpointInputSchemas.analyzeContent, + output: TextrazorEndpointOutputSchemas.analyzeContent, + }, + 'analysis.classifyText': { + input: TextrazorEndpointInputSchemas.classifyText, + output: TextrazorEndpointOutputSchemas.classifyText, + }, + 'analysis.extractEntities': { + input: TextrazorEndpointInputSchemas.extractEntities, + output: TextrazorEndpointOutputSchemas.extractEntities, + }, + 'account.get': { + input: TextrazorEndpointInputSchemas.getAccount, + output: TextrazorEndpointOutputSchemas.getAccount, + }, + 'dictionaries.create': { + input: TextrazorEndpointInputSchemas.createDictionary, + output: TextrazorEndpointOutputSchemas.createDictionary, + }, + 'dictionaries.list': { + input: TextrazorEndpointInputSchemas.listDictionaries, + output: TextrazorEndpointOutputSchemas.listDictionaries, + }, + 'dictionaries.get': { + input: TextrazorEndpointInputSchemas.getDictionary, + output: TextrazorEndpointOutputSchemas.getDictionary, + }, + 'dictionaries.delete': { + input: TextrazorEndpointInputSchemas.deleteDictionary, + output: TextrazorEndpointOutputSchemas.deleteDictionary, + }, + 'dictionaries.listEntries': { + input: TextrazorEndpointInputSchemas.listDictionaryEntries, + output: TextrazorEndpointOutputSchemas.listDictionaryEntries, + }, + 'dictionaries.addEntries': { + input: TextrazorEndpointInputSchemas.addDictionaryEntries, + output: TextrazorEndpointOutputSchemas.addDictionaryEntries, + }, + 'dictionaries.getEntry': { + input: TextrazorEndpointInputSchemas.getDictionaryEntry, + output: TextrazorEndpointOutputSchemas.getDictionaryEntry, + }, + 'dictionaries.deleteEntry': { + input: TextrazorEndpointInputSchemas.deleteDictionaryEntry, + output: TextrazorEndpointOutputSchemas.deleteDictionaryEntry, + }, + 'classifiers.put': { + input: TextrazorEndpointInputSchemas.putClassifier, + output: TextrazorEndpointOutputSchemas.putClassifier, + }, + 'classifiers.delete': { + input: TextrazorEndpointInputSchemas.deleteClassifier, + output: TextrazorEndpointOutputSchemas.deleteClassifier, + }, + 'classifiers.listCategories': { + input: TextrazorEndpointInputSchemas.listClassifierCategories, + output: TextrazorEndpointOutputSchemas.listClassifierCategories, + }, + 'classifiers.getCategory': { + input: TextrazorEndpointInputSchemas.getClassifierCategory, + output: TextrazorEndpointOutputSchemas.getClassifierCategory, + }, + 'classifiers.deleteCategory': { + input: TextrazorEndpointInputSchemas.deleteClassifierCategory, + output: TextrazorEndpointOutputSchemas.deleteClassifierCategory, + }, +} satisfies RequiredPluginEndpointSchemas; + +const textrazorEndpointMeta = { + 'analysis.analyzeContent': { + riskLevel: 'read', + description: + 'Analyze text or a URL with one or more TextRazor extractors in a single call', + }, + 'analysis.classifyText': { + riskLevel: 'read', + description: + 'Classify text or a URL against built-in or custom TextRazor classifiers', + }, + 'analysis.extractEntities': { + riskLevel: 'read', + description: + 'Extract named entities from text or a URL, optionally filtering by relevance and confidence', + }, + 'account.get': { + riskLevel: 'read', + description: + 'Get the current TextRazor plan, concurrency limits, and daily usage', + }, + 'dictionaries.create': { + riskLevel: 'write', + description: 'Create a custom entity dictionary', + }, + 'dictionaries.list': { + riskLevel: 'read', + description: 'List custom entity dictionaries on the account', + }, + 'dictionaries.get': { + riskLevel: 'read', + description: 'Get a custom entity dictionary by id', + }, + 'dictionaries.delete': { + riskLevel: 'destructive', + description: 'Delete a custom entity dictionary and all of its entries', + }, + 'dictionaries.listEntries': { + riskLevel: 'read', + description: 'List dictionary entries with limit and offset pagination', + }, + 'dictionaries.addEntries': { + riskLevel: 'write', + description: 'Add or overwrite entries in a custom entity dictionary', + }, + 'dictionaries.getEntry': { + riskLevel: 'read', + description: 'Get a dictionary entry by id', + }, + 'dictionaries.deleteEntry': { + riskLevel: 'destructive', + description: 'Delete a dictionary entry by id', + }, + 'classifiers.put': { + riskLevel: 'write', + description: 'Create or update a custom classifier from JSON categories', + }, + 'classifiers.delete': { + riskLevel: 'destructive', + description: 'Delete a custom classifier and all of its categories', + }, + 'classifiers.listCategories': { + riskLevel: 'read', + description: + 'List categories for a custom classifier with limit and offset pagination', + }, + 'classifiers.getCategory': { + riskLevel: 'read', + description: 'Get a category from a custom classifier by id', + }, + 'classifiers.deleteCategory': { + riskLevel: 'destructive', + description: 'Delete a category from a custom classifier', + }, +} as const satisfies RequiredPluginEndpointMeta< + typeof textrazorEndpointsNested +>; + +function mergeErrorHandlers( + builtIn: CorsairErrorHandler, + overrides?: CorsairErrorHandler, +): CorsairErrorHandler { + const { DEFAULT: builtInDefault, ...builtInRest } = builtIn; + const { DEFAULT: overrideDefault, ...overrideRest } = overrides ?? {}; + return { + ...builtInRest, + ...overrideRest, + DEFAULT: overrideDefault ?? builtInDefault, + }; +} + +const defaultAuthType: AuthTypes = 'api_key' as const; + +export const textrazorAuthConfig = { + api_key: { + account: ['one'] as const, + }, +} as const satisfies PluginAuthConfig; + +export type BaseTextrazorPlugin = + CorsairPlugin< + 'textrazor', + typeof TextrazorSchema, + typeof textrazorEndpointsNested, + {}, + T, + typeof defaultAuthType, + typeof textrazorAuthConfig + >; + +export type InternalTextrazorPlugin = + BaseTextrazorPlugin; + +export type ExternalTextrazorPlugin = + BaseTextrazorPlugin; + +export function textrazor( + incomingOptions: TextrazorPluginOptions & T = {} as TextrazorPluginOptions & + T, +): ExternalTextrazorPlugin { + const options = { + ...incomingOptions, + authType: incomingOptions.authType ?? defaultAuthType, + }; + return { + id: 'textrazor', + authConfig: textrazorAuthConfig, + schema: TextrazorSchema, + options, + hooks: options.hooks, + endpoints: textrazorEndpointsNested, + webhooks: {}, + endpointMeta: textrazorEndpointMeta, + endpointSchemas: textrazorEndpointSchemas, + pluginWebhookMatcher: () => false, + errorHandlers: mergeErrorHandlers(errorHandlers, options.errorHandlers), + keyBuilder: async (ctx: TextrazorKeyBuilderContext, source) => { + if (source === 'endpoint' && options.key) { + return options.key; + } + + if (source === 'endpoint' && ctx.authType === 'api_key') { + const key = await ctx.keys.get_api_key(); + if (!key) { + throw new AuthMissingError('textrazor', 'api_key'); + } + return key; + } + + throw new AuthMissingError('textrazor', 'api_key'); + }, + } satisfies InternalTextrazorPlugin; +} + +export { + assertTextrazorOk, + TEXTRAZOR_API_BASE, + TextrazorAPIError, + toFormBody, +} from './client'; +export type { + TextrazorEndpointInputs, + TextrazorEndpointOutputs, +} from './endpoints/types'; +export { + TextrazorEndpointInputSchemas, + TextrazorEndpointOutputSchemas, +} from './endpoints/types'; diff --git a/packages/textrazor/jest.config.cjs b/packages/textrazor/jest.config.cjs new file mode 100644 index 000000000..d2cd6c11a --- /dev/null +++ b/packages/textrazor/jest.config.cjs @@ -0,0 +1,57 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: [''], + testMatch: [ + '**/*.test.ts', + '**/tests/**/*.test.ts', + '**/plugins/**/*.test.ts', + '**/setup/**/*.test.ts', + ], + collectCoverageFrom: [ + '**/*.ts', + '!**/*.d.ts', + '!**/node_modules/**', + '!**/dist/**', + '!jest.config.ts', + '!tests/**', + ], + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'], + transform: { + '^.+\\.yaml$': '/../corsair/jest-yaml-transform.cjs', + '^.+\\.ts$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + verbatimModuleSyntax: false, + module: 'ESNext', + moduleResolution: 'Bundler', + types: ['node', 'jest'], + }, + }, + ], + '.*\\.js$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + }, + }, + ], + }, + moduleNameMapper: { + '^corsair/core$': '/../corsair/core.ts', + '^corsair/http$': '/../corsair/http.ts', + '^corsair$': '/../corsair/index.ts', + '^(\\.\\.?/.*)\\.js$': '$1', + }, + transformIgnorePatterns: ['node_modules/(?!.*uuid.*)'], + extensionsToTreatAsEsm: ['.ts'], + testTimeout: 30000, + verbose: true, +}; diff --git a/packages/textrazor/live.test.ts b/packages/textrazor/live.test.ts new file mode 100644 index 000000000..5b3271b93 --- /dev/null +++ b/packages/textrazor/live.test.ts @@ -0,0 +1,133 @@ +import { makeTextrazorRequest } from './client'; + +jest.mock('corsair/core', () => ({ + logEventFromContext: jest.fn().mockResolvedValue(undefined), +})); + +import { + AccountEndpoints, + AnalysisEndpoints, + ClassifierEndpoints, + DictionaryEndpoints, +} from './endpoints'; + +const LIVE_KEY = process.env.TEXTRAZOR_API_KEY ?? ''; +const describeLive = LIVE_KEY ? describe : describe.skip; + +function ctx() { + return { + key: LIVE_KEY, + db: { + accounts: { upsertByEntityId: async () => undefined }, + dictionaries: { upsertByEntityId: async () => undefined }, + dictionaryEntries: { upsertByEntityId: async () => undefined }, + categories: { upsertByEntityId: async () => undefined }, + entities: { upsertByEntityId: async () => undefined }, + }, + }; +} + +function call(fn: unknown, input?: unknown): Promise { + return (fn as (c: unknown, i: unknown) => Promise)(ctx(), input); +} + +describeLive('TextRazor live API', () => { + it('gets account usage from GET /account/', async () => { + const account = await call<{ + ok?: boolean; + response?: { plan?: string; planDailyRequestsIncluded?: number }; + }>(AccountEndpoints.get, {}); + expect(account.ok).toBe(true); + expect(account.response?.plan).toEqual(expect.any(String)); + expect(account.response?.planDailyRequestsIncluded).toEqual( + expect.any(Number), + ); + }); + + it('analyzes, classifies, and extracts entities', async () => { + const analyzed = await call<{ + ok?: boolean; + response?: { entities?: unknown[]; topics?: unknown[] }; + }>(AnalysisEndpoints.analyzeContent, { + text: 'Apple Inc. announced a partnership with OpenAI in California.', + extractors: ['entities', 'topics'], + }); + expect(analyzed.ok).toBe(true); + expect((analyzed.response?.entities ?? []).length).toBeGreaterThan(0); + + const classified = await call<{ + ok?: boolean; + response?: { categories?: Array<{ classifierId?: string }> }; + }>(AnalysisEndpoints.classifyText, { + text: 'The football match ended with a last-minute goal from the striker.', + classifiers: ['textrazor_iab'], + }); + expect(classified.ok).toBe(true); + expect(classified.response?.categories?.[0]?.classifierId).toBe( + 'textrazor_iab', + ); + + const extracted = await call<{ + ok?: boolean; + response?: { entities?: unknown[] }; + }>(AnalysisEndpoints.extractEntities, { + text: 'Apple Inc. is based in California.', + minRelevanceScore: 0.1, + }); + expect(extracted.ok).toBe(true); + expect((extracted.response?.entities ?? []).length).toBeGreaterThan(0); + }); + + it('manages a custom dictionary end to end', async () => { + const id = `corsair_test_${Date.now()}`; + try { + await call(DictionaryEndpoints.create, { + id, + matchType: 'token', + caseInsensitive: true, + language: 'eng', + }); + await call(DictionaryEndpoints.addEntries, { + id, + entries: [{ text: 'Corsair Test Entity', id: 'DEV1' }], + }); + const listed = await makeTextrazorRequest<{ ok?: boolean }>( + 'entities/', + LIVE_KEY, + { method: 'GET' }, + ); + expect(listed.ok).toBe(true); + const entry = await call<{ + ok?: boolean; + response?: { text?: string }; + }>(DictionaryEndpoints.getEntry, { id, entryId: 'DEV1' }); + expect(entry.ok).toBe(true); + await call(DictionaryEndpoints.deleteEntry, { id, entryId: 'DEV1' }); + } finally { + await call(DictionaryEndpoints.delete, { id }); + } + }); + + it('manages a custom classifier end to end', async () => { + const id = `corsair_clf_${Date.now()}`; + try { + await call(ClassifierEndpoints.put, { + id, + categories: [ + { + categoryId: '100', + label: 'Golf', + query: "concept('sport>golf')", + }, + ], + }); + const listed = await call<{ ok?: boolean }>( + ClassifierEndpoints.listCategories, + { id, limit: 20, offset: 0 }, + ); + expect(listed.ok).toBe(true); + } finally { + await call(ClassifierEndpoints.delete, { id }); + } + }); +}); diff --git a/packages/textrazor/package.json b/packages/textrazor/package.json new file mode 100644 index 000000000..7acb01532 --- /dev/null +++ b/packages/textrazor/package.json @@ -0,0 +1,44 @@ +{ + "name": "@corsair-dev/textrazor", + "version": "0.1.0", + "description": "TextRazor plugin for Corsair", + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "dev-source": "./index.ts", + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "rm -rf dist && tsc --build --force && tsup", + "typecheck": "tsc --noEmit", + "test": "jest" + }, + "peerDependencies": { + "corsair": ">=0.1.0", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "corsair": "workspace:*", + "jest": "^29.7.0", + "ts-jest": "^29.4.9", + "tsup": "^8.0.1", + "typescript": "catalog:", + "zod": "^4.1.13" + }, + "keywords": [ + "corsair", + "textrazor", + "plugin" + ], + "author": "", + "license": "Apache-2.0", + "files": [ + "dist" + ] +} diff --git a/packages/textrazor/plugin.test.ts b/packages/textrazor/plugin.test.ts new file mode 100644 index 000000000..59d25fb6f --- /dev/null +++ b/packages/textrazor/plugin.test.ts @@ -0,0 +1,128 @@ +import { textrazor, textrazorEndpointSchemas } from './index'; + +jest.mock('corsair/core', () => { + class AuthMissingError extends Error { + constructor(plugin: string, authType: string) { + super(`Missing ${authType} auth for ${plugin}`); + this.name = 'AuthMissingError'; + } + } + + return { AuthMissingError, logEventFromContext: jest.fn() }; +}); + +const EXPECTED_OPERATIONS = [ + 'account.get', + 'analysis.analyzeContent', + 'analysis.classifyText', + 'analysis.extractEntities', + 'classifiers.delete', + 'classifiers.deleteCategory', + 'classifiers.getCategory', + 'classifiers.listCategories', + 'classifiers.put', + 'dictionaries.addEntries', + 'dictionaries.create', + 'dictionaries.delete', + 'dictionaries.deleteEntry', + 'dictionaries.get', + 'dictionaries.getEntry', + 'dictionaries.list', + 'dictionaries.listEntries', +]; + +function keyBuilderOf(plugin: { keyBuilder?: unknown }) { + const keyBuilder = plugin.keyBuilder; + if (typeof keyBuilder !== 'function') { + throw new Error('keyBuilder is not registered'); + } + return keyBuilder as (ctx: unknown, source: string) => Promise; +} + +function flattenEndpoints(plugin: ReturnType): string[] { + const groups = plugin.endpoints as unknown as Record< + string, + Record + >; + return Object.entries(groups) + .flatMap(([group, ops]) => Object.keys(ops).map((op) => `${group}.${op}`)) + .sort(); +} + +describe('textrazor plugin registration', () => { + const plugin = textrazor(); + + it('exposes the TextRazor operations', () => { + expect(flattenEndpoints(plugin)).toEqual(EXPECTED_OPERATIONS); + }); + + it('registers every endpoint as a callable function', () => { + const groups = plugin.endpoints as unknown as Record< + string, + Record + >; + for (const ops of Object.values(groups)) { + for (const [name, fn] of Object.entries(ops)) { + expect(typeof fn).toBe('function'); + expect(name).not.toHaveLength(0); + } + } + }); + + it('has an input and output schema for every endpoint', () => { + expect(Object.keys(textrazorEndpointSchemas).sort()).toEqual( + EXPECTED_OPERATIONS, + ); + for (const [name, schemas] of Object.entries(textrazorEndpointSchemas)) { + expect(schemas.input).toBeDefined(); + expect(schemas.output).toBeDefined(); + expect(typeof schemas.input.parse).toBe('function'); + expect(typeof schemas.output.parse).toBe('function'); + expect(name).not.toHaveLength(0); + } + }); + + it('has metadata with a risk level and description for every endpoint', () => { + const meta = plugin.endpointMeta as unknown as Record< + string, + { riskLevel: string; description: string } + >; + expect(Object.keys(meta).sort()).toEqual(EXPECTED_OPERATIONS); + for (const entry of Object.values(meta)) { + expect(['read', 'write', 'destructive']).toContain(entry.riskLevel); + expect(entry.description.length).toBeGreaterThan(0); + } + }); + + it('declares api_key auth and registers no webhooks', () => { + expect(plugin.id).toBe('textrazor'); + expect(plugin.authConfig).toHaveProperty('api_key'); + expect(plugin.authConfig).not.toHaveProperty('oauth_2'); + expect(plugin.options?.authType).toBe('api_key'); + expect(plugin.webhooks).toEqual({}); + expect(plugin.pluginWebhookMatcher?.({ headers: {} } as never)).toBe(false); + }); + + it('resolves a statically configured key without touching the key store', async () => { + const configured = textrazor({ key: 'static-key' }); + const ctx = { + authType: 'api_key', + keys: { + get_api_key: async () => { + throw new Error('key store should not be consulted'); + }, + }, + }; + await expect(keyBuilderOf(configured)(ctx, 'endpoint')).resolves.toBe( + 'static-key', + ); + }); + + it('throws AuthMissingError when no key is configured or stored', async () => { + const ctx = { + authType: 'api_key', + keys: { get_api_key: async () => undefined }, + }; + await expect(keyBuilderOf(plugin)(ctx, 'endpoint')).rejects.toThrow(); + }); +}); diff --git a/packages/textrazor/schema.test.ts b/packages/textrazor/schema.test.ts new file mode 100644 index 000000000..b765f16fc --- /dev/null +++ b/packages/textrazor/schema.test.ts @@ -0,0 +1,27 @@ +import { TextrazorSchema } from './schema'; + +describe('Textrazor schema', () => { + it('declares a semver version', () => { + expect(TextrazorSchema.version).toBeDefined(); + expect(TextrazorSchema.version).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it('declares an entities map', () => { + expect(typeof TextrazorSchema.entities).toBe('object'); + expect(TextrazorSchema.entities).not.toBeNull(); + expect(Array.isArray(Object.keys(TextrazorSchema.entities))).toBe(true); + for (const entity of Object.values(TextrazorSchema.entities)) { + expect(entity).toBeDefined(); + } + expect(Object.keys(TextrazorSchema.entities).sort()).toEqual([ + 'accounts', + 'categories', + 'dictionaries', + 'dictionaryEntries', + 'entities', + ]); + }); +}); + +// Per .github/PLUGIN_PR_RULES.md (R2), every implemented endpoint +// needs a corresponding test. diff --git a/packages/textrazor/schema/database.ts b/packages/textrazor/schema/database.ts new file mode 100644 index 000000000..8724cc798 --- /dev/null +++ b/packages/textrazor/schema/database.ts @@ -0,0 +1,58 @@ +import { z } from 'zod'; + +/** @see https://www.textrazor.com/docs/rest — Account Object */ +export const TextrazorAccount = z.object({ + id: z.string(), + plan: z.string().optional(), + concurrentRequestLimit: z.number().optional(), + concurrentRequestsUsed: z.number().optional(), + planDailyRequestsIncluded: z.number().optional(), + requestsUsedToday: z.number().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +/** @see https://www.textrazor.com/docs/rest — Dictionary Object */ +export const TextrazorDictionary = z.object({ + id: z.string(), + matchType: z.string().optional(), + caseInsensitive: z.boolean().optional(), + language: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +/** @see https://www.textrazor.com/docs/rest — DictionaryEntry Object */ +export const TextrazorDictionaryEntry = z.object({ + id: z.string(), + text: z.string().optional(), + data: z.record(z.string(), z.array(z.string())).optional(), + dictionaryId: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +/** @see https://www.textrazor.com/docs/rest — Category Object */ +export const TextrazorCategory = z.object({ + id: z.string(), + categoryId: z.string().optional(), + label: z.string().optional(), + query: z.string().optional(), + classifierId: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +/** @see https://www.textrazor.com/docs/rest — Entity Object */ +export const TextrazorEntity = z.object({ + id: z.string(), + entityId: z.string().nullable().optional(), + matchedText: z.string().optional(), + confidenceScore: z.number().optional(), + relevanceScore: z.number().optional(), + wikiLink: z.string().nullable().optional(), + wikidataId: z.string().nullable().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type TextrazorAccount = z.infer; +export type TextrazorDictionary = z.infer; +export type TextrazorDictionaryEntry = z.infer; +export type TextrazorCategory = z.infer; +export type TextrazorEntity = z.infer; diff --git a/packages/textrazor/schema/index.ts b/packages/textrazor/schema/index.ts new file mode 100644 index 000000000..23750a827 --- /dev/null +++ b/packages/textrazor/schema/index.ts @@ -0,0 +1,26 @@ +import { + TextrazorAccount, + TextrazorCategory, + TextrazorDictionary, + TextrazorDictionaryEntry, + TextrazorEntity, +} from './database'; + +export const TextrazorSchema = { + version: '1.0.0', + entities: { + accounts: TextrazorAccount, + dictionaries: TextrazorDictionary, + dictionaryEntries: TextrazorDictionaryEntry, + categories: TextrazorCategory, + entities: TextrazorEntity, + }, +} as const; + +export type { + TextrazorAccount, + TextrazorCategory, + TextrazorDictionary, + TextrazorDictionaryEntry, + TextrazorEntity, +} from './database'; diff --git a/packages/textrazor/tsconfig.json b/packages/textrazor/tsconfig.json new file mode 100644 index 000000000..15e507a13 --- /dev/null +++ b/packages/textrazor/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["esnext"], + "types": ["node", "jest"], + "module": "ESNext", + "moduleResolution": "Bundler", + "outDir": "./dist", + "rootDir": "./", + "composite": true, + "incremental": true, + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["./**/*"], + "exclude": ["dist", "node_modules"], + "references": [] +} diff --git a/packages/textrazor/tsup.config.ts b/packages/textrazor/tsup.config.ts new file mode 100644 index 000000000..3ec221e23 --- /dev/null +++ b/packages/textrazor/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + clean: false, + dts: false, + format: ['esm'], + target: 'esnext', + platform: 'node', + bundle: true, + splitting: true, + minify: true, + outDir: 'dist', + external: ['corsair', 'zod'], + entry: ['index.ts'], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0922e035b..ea48b498b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4828,6 +4828,30 @@ importers: specifier: 4.4.3 version: 4.4.3 + packages/textrazor: + devDependencies: + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + corsair: + specifier: workspace:* + version: link:../corsair + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) + ts-jest: + specifier: ^29.4.9 + version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@30.4.1)(babel-jest@29.7.0(@babel/core@7.29.7))(esbuild@0.27.0)(jest-util@30.4.1)(jest@29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)))(typescript@5.9.3) + tsup: + specifier: ^8.0.1 + version: 8.5.1(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.9.0) + typescript: + specifier: 'catalog:' + version: 5.9.3 + zod: + specifier: 4.4.3 + version: 4.4.3 + packages/ticktick: devDependencies: '@types/jest':