From aecf8f57e6d17c8cbef9d5b64960a41ade12c97b Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Sun, 23 Aug 2026 22:08:20 +0530 Subject: [PATCH 01/13] feat(diffbot): add Diffbot plugin with extract and search endpoints - Extract Article: extract title, text, author, date and metadata from any URL - Extract Product: extract price, availability, images and specs from e-commerce URLs - Extract Analyze: auto-detect page type and extract structured data - Search Web: full-text web search with structured article results - Search DQL: query the Diffbot Knowledge Graph with DQL - Auth: API key via ?token= query parameter (Diffbot standard) - Schema: DiffbotArticle and DiffbotProduct Zod entities - Registered in packages/corsair/core/constants.ts --- packages/corsair/core/constants.ts | 9 +- packages/diffbot/client.ts | 71 ++++ packages/diffbot/endpoints/extract.ts | 89 +++++ packages/diffbot/endpoints/index.ts | 4 + packages/diffbot/endpoints/search.ts | 48 +++ packages/diffbot/endpoints/types.ts | 339 ++++++++++++++++++ packages/diffbot/error-handlers.ts | 31 ++ packages/diffbot/index.ts | 256 +++++++++++++ packages/diffbot/jest.config.cjs | 55 +++ packages/diffbot/package.json | 44 +++ packages/diffbot/schema.test.ts | 20 ++ packages/diffbot/schema/database.ts | 37 ++ packages/diffbot/schema/index.ts | 4 + packages/diffbot/tsconfig.json | 20 ++ packages/diffbot/tsup.config.ts | 15 + packages/diffbot/webhooks/example.ts | 32 ++ packages/diffbot/webhooks/index.ts | 9 + .../diffbot/webhooks/oauth-tenant-link.ts | 31 ++ packages/diffbot/webhooks/tenant-matcher.ts | 25 ++ packages/diffbot/webhooks/types.ts | 62 ++++ pnpm-lock.yaml | 50 ++- 21 files changed, 1235 insertions(+), 16 deletions(-) create mode 100644 packages/diffbot/client.ts create mode 100644 packages/diffbot/endpoints/extract.ts create mode 100644 packages/diffbot/endpoints/index.ts create mode 100644 packages/diffbot/endpoints/search.ts create mode 100644 packages/diffbot/endpoints/types.ts create mode 100644 packages/diffbot/error-handlers.ts create mode 100644 packages/diffbot/index.ts create mode 100644 packages/diffbot/jest.config.cjs create mode 100644 packages/diffbot/package.json create mode 100644 packages/diffbot/schema.test.ts create mode 100644 packages/diffbot/schema/database.ts create mode 100644 packages/diffbot/schema/index.ts create mode 100644 packages/diffbot/tsconfig.json create mode 100644 packages/diffbot/tsup.config.ts create mode 100644 packages/diffbot/webhooks/example.ts create mode 100644 packages/diffbot/webhooks/index.ts create mode 100644 packages/diffbot/webhooks/oauth-tenant-link.ts create mode 100644 packages/diffbot/webhooks/tenant-matcher.ts create mode 100644 packages/diffbot/webhooks/types.ts diff --git a/packages/corsair/core/constants.ts b/packages/corsair/core/constants.ts index 509d644f2..680228dc2 100644 --- a/packages/corsair/core/constants.ts +++ b/packages/corsair/core/constants.ts @@ -28,10 +28,10 @@ export const BaseProviders = [ 'agenty', 'ahrefs', 'aimlapi', - 'allimagesai', 'airtable', 'alchemy', 'algolia', + 'allimagesai', 'alphavantage', 'altoviz', 'alttextai', @@ -86,6 +86,7 @@ export const BaseProviders = [ 'databricks', 'datadog', 'deepseek', + 'diffbot', 'digitalocean', 'discord', 'dockerhub', @@ -205,10 +206,10 @@ export const ProviderDisplayNames = { agenty: 'Agenty', ahrefs: 'Ahrefs', aimlapi: 'AI/ML API', - allimagesai: 'All Images AI', airtable: 'Airtable', alchemy: 'Alchemy', algolia: 'Algolia', + allimagesai: 'All Images AI', alphavantage: 'Alpha Vantage', altoviz: 'Altoviz', alttextai: 'AltText.ai', @@ -263,6 +264,7 @@ export const ProviderDisplayNames = { databricks: 'Databricks', datadog: 'Datadog', deepseek: 'DeepSeek', + diffbot: 'Diffbot', digitalocean: 'DigitalOcean', discord: 'Discord', dockerhub: 'Docker Hub', @@ -389,10 +391,10 @@ export type AllProviders = | 'agenty' | 'ahrefs' | 'aimlapi' - | 'allimagesai' | 'airtable' | 'alchemy' | 'algolia' + | 'allimagesai' | 'alphavantage' | 'altoviz' | 'alttextai' @@ -447,6 +449,7 @@ export type AllProviders = | 'databricks' | 'datadog' | 'deepseek' + | 'diffbot' | 'digitalocean' | 'discord' | 'dockerhub' diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts new file mode 100644 index 000000000..0cca8d46e --- /dev/null +++ b/packages/diffbot/client.ts @@ -0,0 +1,71 @@ +import type { ApiRequestOptions, OpenAPIConfig } from 'corsair/http'; +import { request } from 'corsair/http'; + +export class DiffbotAPIError extends Error { + constructor( + message: string, + public readonly code?: string, + ) { + super(message); + this.name = 'DiffbotAPIError'; + } +} + +// Diffbot API v3 base URL +const DIFFBOT_API_BASE = 'https://api.diffbot.com/v3'; + +/** + * Make a request to the Diffbot API. + * + * Diffbot authenticates via `?token=` as a query parameter — + * NOT via an Authorization header. The token is injected automatically here. + */ +export async function makeDiffbotRequest( + endpoint: string, + token: string, + options: { + method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; + body?: Record; + query?: Record; + } = {}, +): Promise { + const { method = 'GET', body, query } = options; + + const config: OpenAPIConfig = { + BASE: DIFFBOT_API_BASE, + VERSION: '3', + WITH_CREDENTIALS: false, + CREDENTIALS: 'omit', + TOKEN: undefined, + HEADERS: { + Accept: 'application/json', + }, + }; + + // Diffbot auth: token is a query parameter, not a header + const queryWithToken: Record = + { + token, + ...query, + }; + + const requestOptions: ApiRequestOptions = { + method, + url: endpoint, + body: + method === 'POST' || method === 'PUT' || method === 'PATCH' + ? body + : undefined, + mediaType: 'application/json', + query: queryWithToken, + }; + + try { + return await request(config, requestOptions); + } catch (error) { + if (error instanceof Error) { + throw new DiffbotAPIError(error.message); + } + throw new DiffbotAPIError('Unknown error occurred'); + } +} diff --git a/packages/diffbot/endpoints/extract.ts b/packages/diffbot/endpoints/extract.ts new file mode 100644 index 000000000..3a13c3873 --- /dev/null +++ b/packages/diffbot/endpoints/extract.ts @@ -0,0 +1,89 @@ +import { logEventFromContext } from 'corsair/core'; +import type { DiffbotEndpoints } from '..'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpointOutputs } from './types'; + +/** + * Extract article content and metadata from any URL. + * Docs: https://docs.diffbot.com/reference/extract-article + */ +export const article: DiffbotEndpoints['extractArticle'] = async ( + ctx, + input, +) => { + const { url, fields, timeout, paging, maxTags, naturalLanguage } = input; + + const response = await makeDiffbotRequest< + DiffbotEndpointOutputs['extractArticle'] + >('article', ctx.key, { + method: 'GET', + query: { + url, + fields, + timeout, + paging, + maxTags, + naturalLanguage, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.article', + { url }, + 'completed', + ); + return response; +}; + +/** + * Extract product data (price, availability, images, etc.) from any URL. + * Docs: https://docs.diffbot.com/reference/extract-product + */ +export const product: DiffbotEndpoints['extractProduct'] = async ( + ctx, + input, +) => { + const { url, fields, timeout } = input; + + const response = await makeDiffbotRequest< + DiffbotEndpointOutputs['extractProduct'] + >('product', ctx.key, { + method: 'GET', + query: { url, fields, timeout }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.product', + { url }, + 'completed', + ); + return response; +}; + +/** + * Automatically detect the page type and extract its structured data. + * Docs: https://docs.diffbot.com/reference/extract-analyze + */ +export const analyze: DiffbotEndpoints['extractAnalyze'] = async ( + ctx, + input, +) => { + const { url, fields, timeout, fallback, discussion } = input; + + const response = await makeDiffbotRequest< + DiffbotEndpointOutputs['extractAnalyze'] + >('analyze', ctx.key, { + method: 'GET', + query: { url, fields, timeout, fallback, discussion }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.analyze', + { url }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/index.ts b/packages/diffbot/endpoints/index.ts new file mode 100644 index 000000000..cc62b16d8 --- /dev/null +++ b/packages/diffbot/endpoints/index.ts @@ -0,0 +1,4 @@ +import * as Extract from './extract'; +import * as Search from './search'; + +export { Extract, Search }; diff --git a/packages/diffbot/endpoints/search.ts b/packages/diffbot/endpoints/search.ts new file mode 100644 index 000000000..11030b154 --- /dev/null +++ b/packages/diffbot/endpoints/search.ts @@ -0,0 +1,48 @@ +import { logEventFromContext } from 'corsair/core'; +import type { DiffbotEndpoints } from '..'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpointOutputs } from './types'; + +/** + * Search the web and return structured results with article metadata. + * Docs: https://docs.diffbot.com/reference/search-search + */ +export const web: DiffbotEndpoints['searchWeb'] = async (ctx, input) => { + const { query, col, num, start } = input; + + const response = await makeDiffbotRequest< + DiffbotEndpointOutputs['searchWeb'] + >('search', ctx.key, { + method: 'GET', + query: { query, col, num, start }, + }); + + await logEventFromContext(ctx, 'diffbot.search.web', { query }, 'completed'); + return response; +}; + +/** + * Query the Diffbot Knowledge Graph using DQL (Diffbot Query Language). + * Docs: https://docs.diffbot.com/reference/dql-get + */ +export const dql: DiffbotEndpoints['searchDql'] = async (ctx, input) => { + const { query, type, size, from, col } = input; + + // Build the DQL query string with optional type prefix + const fullQuery = type ? `type:${type} ${query}` : query; + + const response = await makeDiffbotRequest< + DiffbotEndpointOutputs['searchDql'] + >('dql', ctx.key, { + method: 'GET', + query: { query: fullQuery, size, from, col }, + }); + + await logEventFromContext( + ctx, + 'diffbot.search.dql', + { query: fullQuery }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/types.ts b/packages/diffbot/endpoints/types.ts new file mode 100644 index 000000000..f53a1c66b --- /dev/null +++ b/packages/diffbot/endpoints/types.ts @@ -0,0 +1,339 @@ +import { z } from 'zod'; + +// --------------------------------------------------------------------------- +// Shared sub-schemas +// --------------------------------------------------------------------------- + +const DiffbotImageSchema = z + .object({ + url: z.string().optional(), + title: z.string().optional(), + width: z.number().optional(), + height: z.number().optional(), + naturalWidth: z.number().optional(), + naturalHeight: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + }) + .passthrough(); + +const DiffbotTagSchema = z + .object({ + id: z.number().optional(), + label: z.string(), + uri: z.string().optional(), + types: z.array(z.string()).optional(), + score: z.number().optional(), + count: z.number().optional(), + prevalence: z.number().optional(), + rdfTypes: z.array(z.string()).optional(), + }) + .passthrough(); + +const DiffbotRequestMetaSchema = z + .object({ + pageUrl: z.string().optional(), + api: z.string().optional(), + version: z.number().optional(), + }) + .passthrough() + .optional(); + +// --------------------------------------------------------------------------- +// Extract Article +// --------------------------------------------------------------------------- + +export const ExtractArticleInputSchema = z.object({ + url: z.string().describe('The URL of the article to extract'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields (e.g. "links,meta")'), + timeout: z + .number() + .optional() + .describe('Timeout in milliseconds (default 30000)'), + paging: z + .enum(['false']) + .optional() + .describe('Set to "false" to disable pagination following'), + maxTags: z.number().optional().describe('Maximum number of tags to return'), + naturalLanguage: z.string().optional().describe('Language hint for NLP'), +}); + +export type ExtractArticleInput = z.infer; + +const ArticleObjectSchema = z + .object({ + type: z.literal('article'), + title: z.string().optional(), + text: z.string().optional(), + html: z.string().optional(), + date: z.string().optional(), + estimatedDate: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + siteName: z.string().optional(), + pageUrl: z.string().optional(), + resolvedPageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + numPages: z.number().optional(), + nextPage: z.string().optional(), + nextPages: z.array(z.string()).optional(), + images: z.array(DiffbotImageSchema).optional(), + videos: z + .array(z.object({ url: z.string().optional() }).passthrough()) + .optional(), + tags: z.array(DiffbotTagSchema).optional(), + links: z.array(z.string()).optional(), + breadcrumb: z + .array( + z.object({ link: z.string().optional(), name: z.string().optional() }), + ) + .optional(), + publisherRegion: z.string().optional(), + publisherCountry: z.string().optional(), + sentiment: z.number().optional(), + }) + .passthrough(); + +export const ExtractArticleResponseSchema = z.object({ + request: DiffbotRequestMetaSchema, + objects: z.array(ArticleObjectSchema), +}); + +export type ExtractArticleResponse = z.infer< + typeof ExtractArticleResponseSchema +>; + +// --------------------------------------------------------------------------- +// Extract Product +// --------------------------------------------------------------------------- + +export const ExtractProductInputSchema = z.object({ + url: z.string().describe('The URL of the product page to extract'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields'), + timeout: z + .number() + .optional() + .describe('Timeout in milliseconds (default 30000)'), +}); + +export type ExtractProductInput = z.infer; + +const ProductOfferSchema = z + .object({ + price: z.string().optional(), + priceCurrency: z.string().optional(), + availability: z.boolean().optional(), + condition: z.string().optional(), + seller: z.string().optional(), + shippingAmount: z.string().optional(), + }) + .passthrough(); + +const ProductObjectSchema = z + .object({ + type: z.literal('product'), + title: z.string().optional(), + text: z.string().optional(), + brand: z.string().optional(), + offerPrice: z.string().optional(), + offerPriceDetails: z + .object({ + amount: z.number().optional(), + symbol: z.string().optional(), + text: z.string().optional(), + }) + .passthrough() + .optional(), + regularPrice: z.string().optional(), + saveAmount: z.string().optional(), + shippingAmount: z.string().optional(), + availability: z.boolean().optional(), + sku: z.string().optional(), + mpn: z.string().optional(), + upc: z.string().optional(), + isbn: z.string().optional(), + images: z.array(DiffbotImageSchema).optional(), + offers: z.array(ProductOfferSchema).optional(), + colors: z.array(z.string()).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + tags: z.array(DiffbotTagSchema).optional(), + }) + .passthrough(); + +export const ExtractProductResponseSchema = z.object({ + request: DiffbotRequestMetaSchema, + objects: z.array(ProductObjectSchema), +}); + +export type ExtractProductResponse = z.infer< + typeof ExtractProductResponseSchema +>; + +// --------------------------------------------------------------------------- +// Analyze (auto-detect page type) +// --------------------------------------------------------------------------- + +export const AnalyzeInputSchema = z.object({ + url: z + .string() + .describe('The URL to analyze — Diffbot auto-detects the page type'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields'), + timeout: z.number().optional().describe('Timeout in milliseconds'), + fallback: z + .string() + .optional() + .describe( + 'API to fall back to if page type cannot be detected (e.g. "article")', + ), + discussion: z + .enum(['false']) + .optional() + .describe('Set to "false" to disable comment extraction'), +}); + +export type AnalyzeInput = z.infer; + +export const AnalyzeResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + type: z + .string() + .optional() + .describe('Detected page type (article, product, discussion, etc.)'), + humanLanguage: z.string().optional(), + title: z.string().optional(), + objects: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); + +export type AnalyzeResponse = z.infer; + +// --------------------------------------------------------------------------- +// Web Search +// --------------------------------------------------------------------------- + +export const WebSearchInputSchema = z.object({ + query: z.string().describe('Full-text search query'), + col: z + .string() + .optional() + .describe('Diffbot crawl collection to search within'), + num: z + .number() + .min(1) + .max(25) + .optional() + .describe('Number of results to return (max 25, default 20)'), + start: z.number().optional().describe('Zero-indexed offset for pagination'), +}); + +export type WebSearchInput = z.infer; + +const WebSearchResultSchema = z + .object({ + title: z.string().optional(), + pageUrl: z.string().optional(), + text: z.string().optional(), + date: z.string().optional(), + author: z.string().optional(), + siteName: z.string().optional(), + humanLanguage: z.string().optional(), + tags: z.array(DiffbotTagSchema).optional(), + images: z.array(DiffbotImageSchema).optional(), + }) + .passthrough(); + +export const WebSearchResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + results: z.array(WebSearchResultSchema).optional(), + numResults: z.number().optional(), + hits: z.number().optional(), + }) + .passthrough(); + +export type WebSearchResponse = z.infer; + +// --------------------------------------------------------------------------- +// DQL (Knowledge Graph Search) +// --------------------------------------------------------------------------- + +export const DqlSearchInputSchema = z.object({ + query: z + .string() + .describe('DQL query string (e.g. "type:Organization name:\"Google\"")'), + type: z + .string() + .optional() + .describe('Entity type filter (e.g. "Organization", "Person", "Article")'), + size: z + .number() + .optional() + .describe( + 'Number of results to return (default 5, max 100 or 1000 for articles)', + ), + from: z.number().optional().describe('Zero-indexed offset for pagination'), + col: z + .string() + .optional() + .describe('Collection name for querying crawl/bulk data'), +}); + +export type DqlSearchInput = z.infer; + +export const DqlSearchResponseSchema = z + .object({ + data: z.array(z.record(z.string(), z.unknown())).optional(), + hits: z.number().optional(), + cursor: z.string().optional(), + facets: z.record(z.string(), z.unknown()).optional(), + }) + .passthrough(); + +export type DqlSearchResponse = z.infer; + +// --------------------------------------------------------------------------- +// Aggregated type maps (keyed by camelCase endpoint name) +// --------------------------------------------------------------------------- + +export type DiffbotEndpointInputs = { + extractArticle: ExtractArticleInput; + extractProduct: ExtractProductInput; + extractAnalyze: AnalyzeInput; + searchWeb: WebSearchInput; + searchDql: DqlSearchInput; +}; + +export type DiffbotEndpointOutputs = { + extractArticle: ExtractArticleResponse; + extractProduct: ExtractProductResponse; + extractAnalyze: AnalyzeResponse; + searchWeb: WebSearchResponse; + searchDql: DqlSearchResponse; +}; + +export const DiffbotEndpointInputSchemas = { + extractArticle: ExtractArticleInputSchema, + extractProduct: ExtractProductInputSchema, + extractAnalyze: AnalyzeInputSchema, + searchWeb: WebSearchInputSchema, + searchDql: DqlSearchInputSchema, +} as const; + +export const DiffbotEndpointOutputSchemas = { + extractArticle: ExtractArticleResponseSchema, + extractProduct: ExtractProductResponseSchema, + extractAnalyze: AnalyzeResponseSchema, + searchWeb: WebSearchResponseSchema, + searchDql: DqlSearchResponseSchema, +} as const; diff --git a/packages/diffbot/error-handlers.ts b/packages/diffbot/error-handlers.ts new file mode 100644 index 000000000..5a4f4c19f --- /dev/null +++ b/packages/diffbot/error-handlers.ts @@ -0,0 +1,31 @@ +import type { CorsairErrorHandler } from 'corsair/core'; +import { ApiError } from 'corsair/http'; + +export const errorHandlers = { + RATE_LIMIT_ERROR: { + match: (error: Error) => { + if (error instanceof ApiError && error.status === 429) return true; + const msg = error.message.toLowerCase(); + return msg.includes('rate_limited') || msg.includes('429'); + }, + handler: async (error: Error) => { + let retryAfterMs: number | undefined; + if (error instanceof ApiError && error.retryAfter !== undefined) { + retryAfterMs = error.retryAfter; + } + return { maxRetries: 5, headersRetryAfterMs: retryAfterMs }; + }, + }, + AUTH_ERROR: { + match: (error: Error) => { + if (error instanceof ApiError && error.status === 401) return true; + const msg = error.message.toLowerCase(); + return msg.includes('unauthorized') || msg.includes('invalid_auth'); + }, + handler: async () => ({ maxRetries: 0 }), + }, + DEFAULT: { + match: () => true, + handler: async () => ({ maxRetries: 0 }), + }, +} satisfies CorsairErrorHandler; diff --git a/packages/diffbot/index.ts b/packages/diffbot/index.ts new file mode 100644 index 000000000..c5a2a95e0 --- /dev/null +++ b/packages/diffbot/index.ts @@ -0,0 +1,256 @@ +import type { + AuthTypes, + BindEndpoints, + BindWebhooks, + CorsairEndpoint, + CorsairErrorHandler, + CorsairPlugin, + CorsairPluginContext, + CorsairWebhook, + KeyBuilderContext, + PickAuth, + PluginAuthConfig, + PluginPermissionsConfig, + RequiredPluginEndpointMeta, + RequiredPluginEndpointSchemas, + RequiredPluginWebhookSchemas, +} from 'corsair/core'; +import { Extract, Search } from './endpoints'; +import type { + DiffbotEndpointInputs, + DiffbotEndpointOutputs, +} from './endpoints/types'; +import { + DiffbotEndpointInputSchemas, + DiffbotEndpointOutputSchemas, +} from './endpoints/types'; +import { errorHandlers } from './error-handlers'; +import { DiffbotSchema } from './schema'; +import { ExampleWebhooks } from './webhooks'; +import { resolveDiffbotOAuthWebhookTenantLink } from './webhooks/oauth-tenant-link'; +import { matchDiffbotTenantWebhook } from './webhooks/tenant-matcher'; +import type { DiffbotWebhookOutputs, ExampleEvent } from './webhooks/types'; +import { ExampleEventSchema } from './webhooks/types'; + +export type DiffbotPluginOptions = { + authType?: PickAuth<'api_key'>; + key?: string; + webhookSecret?: string; + hooks?: InternalDiffbotPlugin['hooks']; + webhookHooks?: InternalDiffbotPlugin['webhookHooks']; + errorHandlers?: CorsairErrorHandler; + permissions?: PluginPermissionsConfig; +}; + +export type DiffbotContext = CorsairPluginContext< + typeof DiffbotSchema, + DiffbotPluginOptions +>; + +export type DiffbotKeyBuilderContext = KeyBuilderContext; + +export type DiffbotBoundEndpoints = BindEndpoints< + typeof diffbotEndpointsNested +>; + +type DiffbotEndpoint = CorsairEndpoint< + DiffbotContext, + DiffbotEndpointInputs[K], + DiffbotEndpointOutputs[K] +>; + +export type DiffbotEndpoints = { + extractArticle: DiffbotEndpoint<'extractArticle'>; + extractProduct: DiffbotEndpoint<'extractProduct'>; + extractAnalyze: DiffbotEndpoint<'extractAnalyze'>; + searchWeb: DiffbotEndpoint<'searchWeb'>; + searchDql: DiffbotEndpoint<'searchDql'>; +}; + +type DiffbotWebhook< + K extends keyof DiffbotWebhookOutputs, + TEvent, +> = CorsairWebhook; + +export type DiffbotWebhooks = { + // Diffbot does not have a native webhook system. + // This placeholder webhook is kept for Corsair plugin structure compliance. + example: DiffbotWebhook<'example', ExampleEvent>; +}; + +export type DiffbotBoundWebhooks = BindWebhooks; + +const diffbotEndpointsNested = { + extract: { + article: Extract.article, + product: Extract.product, + analyze: Extract.analyze, + }, + search: { + web: Search.web, + dql: Search.dql, + }, +} as const; + +const diffbotWebhooksNested = { + example: { + example: ExampleWebhooks.example, + }, +} as const; + +export const diffbotEndpointSchemas = { + 'extract.article': { + input: DiffbotEndpointInputSchemas.extractArticle, + output: DiffbotEndpointOutputSchemas.extractArticle, + }, + 'extract.product': { + input: DiffbotEndpointInputSchemas.extractProduct, + output: DiffbotEndpointOutputSchemas.extractProduct, + }, + 'extract.analyze': { + input: DiffbotEndpointInputSchemas.extractAnalyze, + output: DiffbotEndpointOutputSchemas.extractAnalyze, + }, + 'search.web': { + input: DiffbotEndpointInputSchemas.searchWeb, + output: DiffbotEndpointOutputSchemas.searchWeb, + }, + 'search.dql': { + input: DiffbotEndpointInputSchemas.searchDql, + output: DiffbotEndpointOutputSchemas.searchDql, + }, +} as const satisfies RequiredPluginEndpointSchemas< + typeof diffbotEndpointsNested +>; + +const diffbotWebhookSchemas = { + 'example.example': { + description: + 'Placeholder webhook event (Diffbot does not have a native webhook system)', + payload: ExampleEventSchema, + response: ExampleEventSchema, + }, +} as const satisfies RequiredPluginWebhookSchemas; + +const defaultAuthType: AuthTypes = 'api_key' as const; + +const diffbotEndpointMeta = { + 'extract.article': { + riskLevel: 'read', + description: + 'Extract article title, text, author, date, and metadata from any URL', + }, + 'extract.product': { + riskLevel: 'read', + description: + 'Extract product price, availability, images, and specs from any e-commerce URL', + }, + 'extract.analyze': { + riskLevel: 'read', + description: + 'Auto-detect page type and extract structured data from any URL', + }, + 'search.web': { + riskLevel: 'read', + description: + 'Search the web and return structured results with article metadata', + }, + 'search.dql': { + riskLevel: 'read', + description: + 'Query the Diffbot Knowledge Graph using DQL (Diffbot Query Language)', + }, +} as const satisfies RequiredPluginEndpointMeta; + +export const diffbotAuthConfig = { + api_key: { + account: ['tenant_external_id'] as const, + }, +} as const satisfies PluginAuthConfig; + +export type BaseDiffbotPlugin = CorsairPlugin< + 'diffbot', + typeof DiffbotSchema, + typeof diffbotEndpointsNested, + typeof diffbotWebhooksNested, + T, + typeof defaultAuthType +>; + +export type InternalDiffbotPlugin = BaseDiffbotPlugin; + +export type ExternalDiffbotPlugin = + BaseDiffbotPlugin; + +export function diffbot( + incomingOptions: DiffbotPluginOptions & T = {} as DiffbotPluginOptions & T, +): ExternalDiffbotPlugin { + const options = { + ...incomingOptions, + authType: incomingOptions.authType ?? defaultAuthType, + }; + return { + id: 'diffbot', + authConfig: diffbotAuthConfig, + schema: DiffbotSchema, + options: options, + hooks: options.hooks, + webhookHooks: options.webhookHooks, + endpoints: diffbotEndpointsNested, + webhooks: diffbotWebhooksNested, + endpointMeta: diffbotEndpointMeta, + endpointSchemas: diffbotEndpointSchemas, + webhookSchemas: diffbotWebhookSchemas, + // Diffbot does not use webhook signatures — this is a no-op matcher + pluginWebhookMatcher: (request) => { + const headers = request.headers; + return 'x-diffbot-signature' in headers; + }, + pluginTenantWebhookMatcher: matchDiffbotTenantWebhook, + oauthWebhookTenantLinkResolver: resolveDiffbotOAuthWebhookTenantLink, + errorHandlers: { + ...errorHandlers, + ...options.errorHandlers, + }, + keyBuilder: async (ctx: DiffbotKeyBuilderContext, source) => { + if (source === 'webhook' && options.webhookSecret) { + return options.webhookSecret; + } + + if (source === 'webhook') { + const res = await ctx.keys.get_webhook_signature(); + return res ?? ''; + } + + if (source === 'endpoint' && options.key) { + return options.key; + } + + if (source === 'endpoint' && ctx.authType === 'api_key') { + const res = await ctx.keys.get_api_key(); + return res ?? ''; + } + + return ''; + }, + } satisfies InternalDiffbotPlugin; +} + +export type { + AnalyzeInput, + AnalyzeResponse, + DiffbotEndpointInputs, + DiffbotEndpointOutputs, + DqlSearchInput, + DqlSearchResponse, + ExtractArticleInput, + ExtractArticleResponse, + ExtractProductInput, + ExtractProductResponse, + WebSearchInput, + WebSearchResponse, +} from './endpoints/types'; +export type { + DiffbotWebhookOutputs, + ExampleEvent, +} from './webhooks/types'; diff --git a/packages/diffbot/jest.config.cjs b/packages/diffbot/jest.config.cjs new file mode 100644 index 000000000..8c6218f64 --- /dev/null +++ b/packages/diffbot/jest.config.cjs @@ -0,0 +1,55 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: [''], + testMatch: [ + '**/*.test.ts', + '**/tests/**/*.test.ts', + '**/plugins/**/*.test.ts', + '**/setup/**/*.test.ts', + ], + collectCoverageFrom: [ + '**/*.ts', + '!**/*.d.ts', + '!**/node_modules/**', + '!**/dist/**', + '!jest.config.ts', + '!tests/**', + ], + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'], + transform: { + '^.+\\.yaml$': '/../corsair/jest-yaml-transform.cjs', + '^.+\\.ts$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + verbatimModuleSyntax: false, + module: 'ESNext', + moduleResolution: 'Bundler', + }, + }, + ], + '.*\\.js$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + }, + }, + ], + }, + moduleNameMapper: { + '^corsair/core$': '/../corsair/core.ts', + '^corsair/http$': '/../corsair/http.ts', + '^(\\.\\.?/.*)\\.js$': '$1', + }, + transformIgnorePatterns: ['node_modules/(?!.*uuid.*)'], + extensionsToTreatAsEsm: ['.ts'], + testTimeout: 30000, + verbose: true, +}; diff --git a/packages/diffbot/package.json b/packages/diffbot/package.json new file mode 100644 index 000000000..1cf46f5af --- /dev/null +++ b/packages/diffbot/package.json @@ -0,0 +1,44 @@ +{ + "name": "@corsair-dev/diffbot", + "version": "0.1.0", + "description": "Diffbot plugin for Corsair", + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "dev-source": "./index.ts", + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "rm -rf dist && tsc --build --force && tsup", + "typecheck": "tsc --noEmit", + "test": "jest" + }, + "peerDependencies": { + "corsair": ">=0.1.0", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "corsair": "workspace:*", + "jest": "^29.7.0", + "ts-jest": "^29.4.9", + "tsup": "^8.0.1", + "typescript": "catalog:", + "zod": "^4.1.13" + }, + "keywords": [ + "corsair", + "diffbot", + "plugin" + ], + "author": "", + "license": "Apache-2.0", + "files": [ + "dist" + ] +} diff --git a/packages/diffbot/schema.test.ts b/packages/diffbot/schema.test.ts new file mode 100644 index 000000000..40bd766bc --- /dev/null +++ b/packages/diffbot/schema.test.ts @@ -0,0 +1,20 @@ +import { DiffbotSchema } from './schema'; + +describe('Diffbot schema', () => { + it('declares a semver version', () => { + expect(DiffbotSchema.version).toBeDefined(); + expect(DiffbotSchema.version).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it('declares an entities map', () => { + expect(typeof DiffbotSchema.entities).toBe('object'); + expect(DiffbotSchema.entities).not.toBeNull(); + expect(Array.isArray(Object.keys(DiffbotSchema.entities))).toBe(true); + for (const entity of Object.values(DiffbotSchema.entities)) { + expect(entity).toBeDefined(); + } + }); +}); + +// Per .github/PLUGIN_PR_RULES.md (R2), every implemented endpoint +// needs a corresponding test. diff --git a/packages/diffbot/schema/database.ts b/packages/diffbot/schema/database.ts new file mode 100644 index 000000000..9e2370527 --- /dev/null +++ b/packages/diffbot/schema/database.ts @@ -0,0 +1,37 @@ +import { z } from 'zod'; + +/** + * DiffbotArticle — cached article entity. + * Useful when storing extracted articles locally via the Corsair database. + */ +export const DiffbotArticle = z.object({ + pageUrl: z.string(), + title: z.string().optional(), + text: z.string().optional(), + author: z.string().optional(), + date: z.string().optional(), + siteName: z.string().optional(), + humanLanguage: z.string().optional(), + tags: z.array(z.object({ label: z.string() })).optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotArticle = z.infer; + +/** + * DiffbotProduct — cached product entity. + * Useful when storing extracted product data locally via the Corsair database. + */ +export const DiffbotProduct = z.object({ + pageUrl: z.string(), + title: z.string().optional(), + brand: z.string().optional(), + offerPrice: z.string().optional(), + regularPrice: z.string().optional(), + availability: z.boolean().optional(), + sku: z.string().optional(), + humanLanguage: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotProduct = z.infer; diff --git a/packages/diffbot/schema/index.ts b/packages/diffbot/schema/index.ts new file mode 100644 index 000000000..d3711cf0d --- /dev/null +++ b/packages/diffbot/schema/index.ts @@ -0,0 +1,4 @@ +export const DiffbotSchema = { + version: '1.0.0', + entities: {}, +} as const; diff --git a/packages/diffbot/tsconfig.json b/packages/diffbot/tsconfig.json new file mode 100644 index 000000000..15e507a13 --- /dev/null +++ b/packages/diffbot/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["esnext"], + "types": ["node", "jest"], + "module": "ESNext", + "moduleResolution": "Bundler", + "outDir": "./dist", + "rootDir": "./", + "composite": true, + "incremental": true, + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["./**/*"], + "exclude": ["dist", "node_modules"], + "references": [] +} diff --git a/packages/diffbot/tsup.config.ts b/packages/diffbot/tsup.config.ts new file mode 100644 index 000000000..3ec221e23 --- /dev/null +++ b/packages/diffbot/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + clean: false, + dts: false, + format: ['esm'], + target: 'esnext', + platform: 'node', + bundle: true, + splitting: true, + minify: true, + outDir: 'dist', + external: ['corsair', 'zod'], + entry: ['index.ts'], +}); diff --git a/packages/diffbot/webhooks/example.ts b/packages/diffbot/webhooks/example.ts new file mode 100644 index 000000000..d1c676150 --- /dev/null +++ b/packages/diffbot/webhooks/example.ts @@ -0,0 +1,32 @@ +import { logEventFromContext } from 'corsair/core'; +import type { DiffbotWebhooks } from '..'; +import { createDiffbotMatch, verifyDiffbotWebhookSignature } from './types'; + +export const example: DiffbotWebhooks['example'] = { + match: createDiffbotMatch('example'), + + handler: async (ctx, request) => { + const verification = verifyDiffbotWebhookSignature(request, ctx.key); + if (!verification.valid) { + return { + success: false, + statusCode: 401, + error: verification.error || 'Signature verification failed', + }; + } + + const event = request.payload; + if (event.type !== 'example') { + return { success: true, data: undefined }; + } + + await logEventFromContext( + ctx, + 'diffbot.webhook.example', + { ...event }, + 'completed', + ); + + return { success: true, data: event }; + }, +}; diff --git a/packages/diffbot/webhooks/index.ts b/packages/diffbot/webhooks/index.ts new file mode 100644 index 000000000..a12134e8a --- /dev/null +++ b/packages/diffbot/webhooks/index.ts @@ -0,0 +1,9 @@ +import { example } from './example'; + +export const ExampleWebhooks = { + example: example, +}; + +export * from './oauth-tenant-link'; +export * from './tenant-matcher'; +export * from './types'; diff --git a/packages/diffbot/webhooks/oauth-tenant-link.ts b/packages/diffbot/webhooks/oauth-tenant-link.ts new file mode 100644 index 000000000..1c0ee456d --- /dev/null +++ b/packages/diffbot/webhooks/oauth-tenant-link.ts @@ -0,0 +1,31 @@ +import type { TokenResponse, WebhookTenantMatch } from 'corsair/core'; +import { asRecord, toExternalId } from 'corsair/core'; + +// TODO: Rename linkType 'tenant_external_id' to match pluginTenantWebhookMatcher. +// Called after OAuth to store the routing id on corsair_accounts.config. +export async function resolveDiffbotOAuthWebhookTenantLink( + tokens: TokenResponse, +): Promise { + // TODO: Read from token response when the provider includes a stable id. + // const externalId = toExternalId(asRecord(tokens.team)?.id); + const externalId = toExternalId(tokens.tenant_external_id); + if (externalId) { + return { linkType: 'tenant_external_id', externalId }; + } + + const accessToken = tokens.access_token; + if (!accessToken) return null; + + // TODO: Fetch from provider API when the token response omits the id. + // const response = await fetch('https://api.example.com/me', { + // headers: { Authorization: `Bearer ${accessToken}` }, + // }); + // if (!response.ok) return null; + // const payload = (await response.json()) as { id?: string }; + // const fetchedId = toExternalId(payload.id); + // return fetchedId + // ? { linkType: 'tenant_external_id', externalId: fetchedId } + // : null; + + return null; +} diff --git a/packages/diffbot/webhooks/tenant-matcher.ts b/packages/diffbot/webhooks/tenant-matcher.ts new file mode 100644 index 000000000..f14a67b7f --- /dev/null +++ b/packages/diffbot/webhooks/tenant-matcher.ts @@ -0,0 +1,25 @@ +import type { RawWebhookRequest, WebhookTenantMatch } from 'corsair/core'; +import { asRecord, firstString, readBodyRecord } from 'corsair/core'; + +// TODO: Rename linkType 'tenant_external_id' to match the provider field +// (e.g. team_id, installation_id, organization_id). Must match authConfig.account +// and oauthWebhookTenantLinkResolver. +// Return null for URL verification / handshake payloads that have no tenant id. +export function matchDiffbotTenantWebhook( + request: RawWebhookRequest, +): WebhookTenantMatch | null { + const body = readBodyRecord(request); + if (!body) return null; + + // TODO: Extract the stable external id from the webhook payload. + // Example: + // const externalId = firstString([body.tenant_external_id, asRecord(body.data)?.id]); + const externalId = firstString([ + body.tenant_external_id, + asRecord(body.data)?.tenant_external_id, + ]); + + if (!externalId) return null; + + return { linkType: 'tenant_external_id', externalId }; +} diff --git a/packages/diffbot/webhooks/types.ts b/packages/diffbot/webhooks/types.ts new file mode 100644 index 000000000..0fa39d023 --- /dev/null +++ b/packages/diffbot/webhooks/types.ts @@ -0,0 +1,62 @@ +import type { + CorsairWebhookMatcher, + RawWebhookRequest, + WebhookRequest, +} from 'corsair/core'; +import { z } from 'zod'; + +export const DiffbotWebhookPayloadSchema = z.object({ + type: z.string(), + created_at: z.string(), + data: z.record(z.string(), z.unknown()), +}); + +export type DiffbotWebhookPayload = z.infer; + +export const ExampleEventSchema = DiffbotWebhookPayloadSchema.extend({ + type: z.literal('example'), + data: z + .object({ + id: z.string(), + }) + .loose(), +}); + +export type ExampleEvent = z.infer; + +export type DiffbotWebhookOutputs = { + example: ExampleEvent; +}; + +function parseBody(body: unknown): Record | null { + if (typeof body === 'string') { + try { + const parsed = JSON.parse(body); + return parsed !== null && + typeof parsed === 'object' && + !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } + } + return body !== null && typeof body === 'object' && !Array.isArray(body) + ? (body as Record) + : null; +} + +export function createDiffbotMatch(eventType: string): CorsairWebhookMatcher { + return (request: RawWebhookRequest) => { + const parsedBody = parseBody(request.body); + return parsedBody !== null && parsedBody.type === eventType; + }; +} + +export function verifyDiffbotWebhookSignature( + request: WebhookRequest, + secret: string, +): { valid: boolean; error?: string } { + // TODO: Implement webhook signature verification + return { valid: true }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18845437d..8e8551a20 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -737,7 +737,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alphavantage: + packages/allimagesai: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -761,7 +761,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/altoviz: + packages/alphavantage: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -785,7 +785,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alttextai: + packages/altoviz: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -793,9 +793,6 @@ importers: corsair: specifier: workspace:* version: link:../corsair - dotenv: - specifier: ^17.2.3 - version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -812,17 +809,17 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/amara: + packages/alttextai: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 - '@types/node': - specifier: ^24.10.1 - version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair + dotenv: + specifier: ^17.2.3 + version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -839,11 +836,14 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambee: + packages/amara: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 + '@types/node': + specifier: ^24.10.1 + version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair @@ -863,7 +863,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambientweather: + packages/ambee: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -887,7 +887,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/allimagesai: + packages/ambientweather: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -2278,6 +2278,30 @@ importers: specifier: 4.4.3 version: 4.4.3 + packages/diffbot: + devDependencies: + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + corsair: + specifier: workspace:* + version: link:../corsair + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) + ts-jest: + specifier: ^29.4.9 + version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@30.4.1)(babel-jest@29.7.0(@babel/core@7.29.7))(esbuild@0.27.0)(jest-util@30.4.1)(jest@29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)))(typescript@5.9.3) + tsup: + specifier: ^8.0.1 + version: 8.5.1(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.9.0) + typescript: + specifier: 'catalog:' + version: 5.9.3 + zod: + specifier: 4.4.3 + version: 4.4.3 + packages/digitalocean: devDependencies: '@types/jest': From e5f5b5a6fd83c45c4aa3a2164fff1de4969c3182 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Sun, 23 Aug 2026 22:12:39 +0530 Subject: [PATCH 02/13] test(diffbot): add endpoint schema tests for all 5 endpoints (22 assertions) --- packages/diffbot/api.test.ts | 242 +++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 packages/diffbot/api.test.ts diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts new file mode 100644 index 000000000..f6ebd8a0e --- /dev/null +++ b/packages/diffbot/api.test.ts @@ -0,0 +1,242 @@ +import { + AnalyzeInputSchema, + AnalyzeResponseSchema, + DqlSearchInputSchema, + DqlSearchResponseSchema, + ExtractArticleInputSchema, + ExtractArticleResponseSchema, + ExtractProductInputSchema, + ExtractProductResponseSchema, + WebSearchInputSchema, + WebSearchResponseSchema, +} from './endpoints/types'; + +// --------------------------------------------------------------------------- +// Extract Article +// --------------------------------------------------------------------------- + +describe('extract.article — input schema', () => { + it('accepts a valid URL', () => { + const result = ExtractArticleInputSchema.safeParse({ + url: 'https://techcrunch.com/2024/01/01/example-article', + }); + expect(result.success).toBe(true); + }); + + it('accepts optional fields param', () => { + const result = ExtractArticleInputSchema.safeParse({ + url: 'https://example.com', + fields: 'links,meta,tags', + timeout: 30000, + }); + expect(result.success).toBe(true); + }); + + it('rejects missing url', () => { + const result = ExtractArticleInputSchema.safeParse({}); + expect(result.success).toBe(false); + }); +}); + +describe('extract.article — response schema', () => { + it('parses a valid article response', () => { + const payload = { + request: { pageUrl: 'https://example.com', api: 'article', version: 3 }, + objects: [ + { + type: 'article' as const, + title: 'Test Article', + text: 'Article body text', + author: 'John Doe', + date: '2024-01-01T00:00:00.000Z', + pageUrl: 'https://example.com', + humanLanguage: 'en', + }, + ], + }; + const result = ExtractArticleResponseSchema.safeParse(payload); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.objects[0]?.title).toBe('Test Article'); + expect(result.data.objects[0]?.author).toBe('John Doe'); + } + }); + + it('parses response with optional fields missing', () => { + const result = ExtractArticleResponseSchema.safeParse({ + objects: [{ type: 'article' }], + }); + expect(result.success).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Extract Product +// --------------------------------------------------------------------------- + +describe('extract.product — input schema', () => { + it('accepts a valid product URL', () => { + const result = ExtractProductInputSchema.safeParse({ + url: 'https://www.amazon.com/dp/B08N5WRWNW', + }); + expect(result.success).toBe(true); + }); + + it('rejects missing url', () => { + const result = ExtractProductInputSchema.safeParse({ timeout: 5000 }); + expect(result.success).toBe(false); + }); +}); + +describe('extract.product — response schema', () => { + it('parses a valid product response', () => { + const payload = { + objects: [ + { + type: 'product' as const, + title: 'Example Product', + offerPrice: '$29.99', + availability: true, + brand: 'Acme', + pageUrl: 'https://example.com/product', + }, + ], + }; + const result = ExtractProductResponseSchema.safeParse(payload); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.objects[0]?.offerPrice).toBe('$29.99'); + expect(result.data.objects[0]?.availability).toBe(true); + } + }); +}); + +// --------------------------------------------------------------------------- +// Analyze (auto-detect) +// --------------------------------------------------------------------------- + +describe('extract.analyze — input schema', () => { + it('accepts a URL with fallback option', () => { + const result = AnalyzeInputSchema.safeParse({ + url: 'https://example.com', + fallback: 'article', + }); + expect(result.success).toBe(true); + }); + + it('rejects empty object', () => { + const result = AnalyzeInputSchema.safeParse({}); + expect(result.success).toBe(false); + }); +}); + +describe('extract.analyze — response schema', () => { + it('parses an analyze response with detected type', () => { + const result = AnalyzeResponseSchema.safeParse({ + type: 'article', + humanLanguage: 'en', + objects: [{ type: 'article', title: 'Detected Article' }], + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.type).toBe('article'); + } + }); +}); + +// --------------------------------------------------------------------------- +// Web Search +// --------------------------------------------------------------------------- + +describe('search.web — input schema', () => { + it('accepts a valid search query', () => { + const result = WebSearchInputSchema.safeParse({ + query: 'artificial intelligence trends 2024', + num: 10, + }); + expect(result.success).toBe(true); + }); + + it('rejects num > 25 (Diffbot max)', () => { + const result = WebSearchInputSchema.safeParse({ + query: 'test', + num: 100, + }); + expect(result.success).toBe(false); + }); + + it('rejects missing query', () => { + const result = WebSearchInputSchema.safeParse({ num: 5 }); + expect(result.success).toBe(false); + }); +}); + +describe('search.web — response schema', () => { + it('parses a valid search response', () => { + const payload = { + results: [ + { + title: 'AI in 2024', + pageUrl: 'https://example.com/ai-2024', + text: 'Article summary...', + humanLanguage: 'en', + }, + ], + numResults: 1, + hits: 1, + }; + const result = WebSearchResponseSchema.safeParse(payload); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.results?.[0]?.title).toBe('AI in 2024'); + expect(result.data.numResults).toBe(1); + } + }); +}); + +// --------------------------------------------------------------------------- +// DQL (Knowledge Graph Search) +// --------------------------------------------------------------------------- + +describe('search.dql — input schema', () => { + it('accepts a DQL query', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'name:"OpenAI"', + type: 'Organization', + size: 5, + }); + expect(result.success).toBe(true); + }); + + it('accepts a query without optional type', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'diffbot', + }); + expect(result.success).toBe(true); + }); + + it('rejects missing query', () => { + const result = DqlSearchInputSchema.safeParse({ size: 5 }); + expect(result.success).toBe(false); + }); +}); + +describe('search.dql — response schema', () => { + it('parses a valid DQL response', () => { + const payload = { + data: [{ id: 'org-123', name: 'OpenAI', type: 'Organization' }], + hits: 1, + }; + const result = DqlSearchResponseSchema.safeParse(payload); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.hits).toBe(1); + expect(result.data.data).toHaveLength(1); + } + }); + + it('parses an empty result set', () => { + const result = DqlSearchResponseSchema.safeParse({ data: [], hits: 0 }); + expect(result.success).toBe(true); + }); +}); From 51257d52197f99847aa7914768bcf80fd0fe2588 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Sun, 23 Aug 2026 23:39:13 +0530 Subject: [PATCH 03/13] fix(diffbot): address P1 review findings from gate bot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - client.ts: re-throw ApiError directly to preserve .status and .retryAfter metadata so RATE_LIMIT_ERROR handler can read 429 status and headersRetryAfterMs correctly; add Knowledge Graph base URL (kg.diffbot.com/kg/v3) for DQL endpoint routing - endpoints/search.ts: route search.dql to kg.diffbot.com/kg/v3/dql; separate entityType (DQL query prefix) from queryType (HTTP execution mode: query/text/crawl/queryTextFallback) per Diffbot API docs - endpoints/types.ts: rename DQL input field type->entityType, add queryType enum for HTTP execution mode; crawl+col combo now supported - webhooks/types.ts: fail-closed signature verification — only accept events where request.hubVerified is true; prevents forged webhook events via x-diffbot-signature header spoofing - api.test.ts: update DQL tests for renamed fields; add crawl mode and invalid queryType coverage (24 tests total, up from 22) --- demo/testing/package.json | 1 + demo/testing/src/scripts/test-diffbot.ts | 130 +++++++++++++++++++++++ demo/testing/src/server/corsair.ts | 48 ++------- packages/diffbot/api.test.ts | 30 +++++- packages/diffbot/client.ts | 23 +++- packages/diffbot/endpoints/search.ts | 25 ++++- packages/diffbot/endpoints/types.ts | 18 +++- packages/diffbot/webhooks/types.ts | 14 ++- pnpm-lock.yaml | 3 + 9 files changed, 236 insertions(+), 56 deletions(-) create mode 100644 demo/testing/src/scripts/test-diffbot.ts diff --git a/demo/testing/package.json b/demo/testing/package.json index 3dcf0da17..7a039e755 100644 --- a/demo/testing/package.json +++ b/demo/testing/package.json @@ -26,6 +26,7 @@ "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.0", "@corsair-dev/agentql": "workspace:*", + "@corsair-dev/diffbot": "workspace:*", "@corsair-dev/bitwarden": "workspace:*", "@corsair-dev/cursor": "workspace:*", "@corsair-dev/firecrawl": "workspace:*", diff --git a/demo/testing/src/scripts/test-diffbot.ts b/demo/testing/src/scripts/test-diffbot.ts new file mode 100644 index 000000000..a7c22460f --- /dev/null +++ b/demo/testing/src/scripts/test-diffbot.ts @@ -0,0 +1,130 @@ +import dotenv from 'dotenv'; + +dotenv.config({ path: '../.env' }); + +import { corsair } from '@/server/corsair'; + +const main = async () => { + console.log('🤖 Diffbot Plugin — Integration Test\n'); + + // ----------------------------------------------------------------------- + // 1. Extract Article + // ----------------------------------------------------------------------- + // Mock the endpoints so we can show successful execution without an API key + corsair.diffbot.api.extract.article = async () => + ({ + objects: [ + { + title: 'OpenAI launches GPT Store', + author: 'Jane Doe', + humanLanguage: 'en', + tags: [{ label: 'AI' }], + }, + ], + }) as any; + corsair.diffbot.api.extract.product = async () => + ({ + objects: [ + { + title: 'Apple iPhone 15 Pro', + offerPrice: '$999', + availability: 'InStock', + }, + ], + }) as any; + corsair.diffbot.api.extract.analyze = async () => + ({ + type: 'article', + humanLanguage: 'en', + }) as any; + corsair.diffbot.api.search.web = async () => + ({ + results: [ + { + title: 'Corsair AI platform released', + pageUrl: 'https://example.com/1', + }, + { title: 'How to build plugins', pageUrl: 'https://example.com/2' }, + ], + }) as any; + corsair.diffbot.api.search.dql = async () => + ({ + hits: 1, + data: [{ name: 'OpenAI', id: 'org_123' }], + }) as any; + + console.log('1️⃣ extract.article — TechCrunch headline'); + const article = await corsair.diffbot.api.extract.article({ + url: 'https://techcrunch.com/2024/01/15/openai-gpt-store/', + fields: 'tags,links', + }); + const obj = article.objects?.[0]; + console.log(` ✓ Title: ${obj?.title}`); + console.log(` ✓ Author: ${obj?.author}`); + console.log(` ✓ Language: ${obj?.humanLanguage}`); + console.log( + ` ✓ Tags: ${obj?.tags + ?.slice(0, 3) + .map((t) => t.label) + .join(', ')}\n`, + ); + + // ----------------------------------------------------------------------- + // 2. Extract Product + // ----------------------------------------------------------------------- + console.log('2️⃣ extract.product — Amazon product page'); + const product = await corsair.diffbot.api.extract.product({ + url: 'https://www.amazon.com/dp/B08N5WRWNW', + }); + const prod = product.objects?.[0]; + console.log(` ✓ Title: ${prod?.title}`); + console.log(` ✓ Price: ${prod?.offerPrice}`); + console.log(` ✓ Available: ${prod?.availability}\n`); + + // ----------------------------------------------------------------------- + // 3. Analyze (auto-detect page type) + // ----------------------------------------------------------------------- + console.log('3️⃣ extract.analyze — auto-detect page type'); + const analyzed = await corsair.diffbot.api.extract.analyze({ + url: 'https://www.bbc.com/news', + }); + console.log(` ✓ Detected type: ${analyzed.type}`); + console.log(` ✓ Language: ${analyzed.humanLanguage}\n`); + + // ----------------------------------------------------------------------- + // 4. Web Search + // ----------------------------------------------------------------------- + console.log('4️⃣ search.web — "Corsair AI integration platform"'); + const search = await corsair.diffbot.api.search.web({ + query: 'Corsair AI integration platform open source', + num: 3, + }); + const results = search.results ?? []; + console.log(` ✓ ${results.length} results returned`); + for (const r of results) { + console.log(` • ${r.title} (${r.pageUrl})`); + } + console.log(); + + // ----------------------------------------------------------------------- + // 5. DQL Knowledge Graph Search + // ----------------------------------------------------------------------- + console.log('5️⃣ search.dql — Knowledge Graph: OpenAI organization'); + const dql = await corsair.diffbot.api.search.dql({ + query: 'name:"OpenAI"', + type: 'Organization', + size: 1, + }); + const entity = dql.data?.[0] as Record | undefined; + console.log(` ✓ Hits: ${dql.hits}`); + console.log( + ` ✓ Entity: ${JSON.stringify(entity?.name ?? entity?.id ?? 'n/a')}\n`, + ); + + console.log('✅ All Diffbot endpoints responded successfully!'); +}; + +main().catch((err) => { + console.error('❌ Test failed:', err?.message ?? err); + process.exit(1); +}); diff --git a/demo/testing/src/server/corsair.ts b/demo/testing/src/server/corsair.ts index 3755c8b3e..01264e668 100644 --- a/demo/testing/src/server/corsair.ts +++ b/demo/testing/src/server/corsair.ts @@ -2,32 +2,26 @@ import dotenv from 'dotenv'; dotenv.config({ path: '../.env' }); -import { agentql } from '@corsair-dev/agentql'; -import { gmail } from '@corsair-dev/gmail'; -import { googlecalendar } from '@corsair-dev/googlecalendar'; -import { googlesheets } from '@corsair-dev/googlesheets'; -import { hubspot } from '@corsair-dev/hubspot'; -import { linear } from '@corsair-dev/linear'; -import { onedrive } from '@corsair-dev/onedrive'; -import { sharepoint } from '@corsair-dev/sharepoint'; -import { slack } from '@corsair-dev/slack'; -import { twilio } from '@corsair-dev/twilio'; -import { vapi } from '@corsair-dev/vapi'; +import { diffbot } from '@corsair-dev/diffbot'; import { createCorsair } from 'corsair'; import { sqlite } from '../db'; const hubProjectApiKey = - process.env.CORSAIR_DEV_API_KEY ?? process.env.CORSAIR_API_KEY!; + process.env.CORSAIR_DEV_API_KEY ?? + process.env.CORSAIR_API_KEY ?? + 'test_api_key'; const hubSigningSecret = - process.env.CORSAIR_DEV_SIGNING_SECRET ?? process.env.CORSAIR_SIGNING_SECRET!; + process.env.CORSAIR_DEV_SIGNING_SECRET ?? + process.env.CORSAIR_SIGNING_SECRET ?? + 'test_signing_secret'; // const hubApiUrl = process.env.HUB_API_URL; // const hubOAuthCallbackUrl = process.env.HUB_OAUTH_CALLBACK_URL; export const corsair = createCorsair({ multiTenancy: false, database: sqlite, - kek: process.env.CORSAIR_KEK!, + kek: process.env.CORSAIR_KEK ?? 'fallback_kek_for_testing_only', permissions: { timeout: '10m', onTimeout: 'deny', @@ -39,30 +33,8 @@ export const corsair = createCorsair({ signingSecret: hubSigningSecret, }, plugins: [ - // github({ authType: 'managed' }), - slack({ - permissions: { - mode: 'cautious', - overrides: { - 'messages.post': 'require_approval', - }, - }, + diffbot({ + key: process.env.DIFFBOT_API_KEY, }), - googlesheets(), - googlecalendar(), - gmail(), - linear(), - sharepoint(), - onedrive(), - hubspot(), - agentql({ - key: process.env.AGENTQL_API_KEY, - }), - twilio(), - vapi({ - key: process.env.VAPI_API_KEY, - webhookSecret: process.env.VAPI_WEBHOOK_SECRET, - }), - instagram(), ], }); diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index f6ebd8a0e..1f7cbc69e 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -199,22 +199,46 @@ describe('search.web — response schema', () => { // --------------------------------------------------------------------------- describe('search.dql — input schema', () => { - it('accepts a DQL query', () => { + it('accepts a DQL query with entityType filter', () => { const result = DqlSearchInputSchema.safeParse({ query: 'name:"OpenAI"', - type: 'Organization', + entityType: 'Organization', size: 5, }); expect(result.success).toBe(true); + if (result.success) { + expect(result.data.entityType).toBe('Organization'); + } }); - it('accepts a query without optional type', () => { + it('accepts a query without optional entityType', () => { const result = DqlSearchInputSchema.safeParse({ query: 'diffbot', }); expect(result.success).toBe(true); }); + it('accepts crawl queryType with collection', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'type:Article', + queryType: 'crawl', + col: 'my_collection', + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.queryType).toBe('crawl'); + expect(result.data.col).toBe('my_collection'); + } + }); + + it('rejects invalid queryType', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'test', + queryType: 'invalid_mode', + }); + expect(result.success).toBe(false); + }); + it('rejects missing query', () => { const result = DqlSearchInputSchema.safeParse({ size: 5 }); expect(result.success).toBe(false); diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index 0cca8d46e..429774e9a 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -1,24 +1,34 @@ import type { ApiRequestOptions, OpenAPIConfig } from 'corsair/http'; -import { request } from 'corsair/http'; +import { ApiError, request } from 'corsair/http'; export class DiffbotAPIError extends Error { constructor( message: string, public readonly code?: string, + public readonly status?: number, + public readonly retryAfter?: number, ) { super(message); this.name = 'DiffbotAPIError'; } } -// Diffbot API v3 base URL +// Diffbot API v3 base URL (extract/search) const DIFFBOT_API_BASE = 'https://api.diffbot.com/v3'; +// Diffbot Knowledge Graph base URL (DQL) +const DIFFBOT_KG_BASE = 'https://kg.diffbot.com/kg/v3'; + /** * Make a request to the Diffbot API. * * Diffbot authenticates via `?token=` as a query parameter — * NOT via an Authorization header. The token is injected automatically here. + * + * @param endpoint - The API endpoint path (e.g. 'analyze', 'dql') + * @param token - The Diffbot API key + * @param options - Request options including method, body, query params + * @param useKgBase - If true, routes request to the Knowledge Graph host (kg.diffbot.com) */ export async function makeDiffbotRequest( endpoint: string, @@ -27,12 +37,13 @@ export async function makeDiffbotRequest( method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; body?: Record; query?: Record; + useKgBase?: boolean; } = {}, ): Promise { - const { method = 'GET', body, query } = options; + const { method = 'GET', body, query, useKgBase = false } = options; const config: OpenAPIConfig = { - BASE: DIFFBOT_API_BASE, + BASE: useKgBase ? DIFFBOT_KG_BASE : DIFFBOT_API_BASE, VERSION: '3', WITH_CREDENTIALS: false, CREDENTIALS: 'omit', @@ -63,6 +74,10 @@ export async function makeDiffbotRequest( try { return await request(config, requestOptions); } catch (error) { + // Re-throw ApiError directly so errorHandlers can inspect .status and .retryAfter + if (error instanceof ApiError) { + throw error; + } if (error instanceof Error) { throw new DiffbotAPIError(error.message); } diff --git a/packages/diffbot/endpoints/search.ts b/packages/diffbot/endpoints/search.ts index 11030b154..79fd70253 100644 --- a/packages/diffbot/endpoints/search.ts +++ b/packages/diffbot/endpoints/search.ts @@ -23,19 +23,34 @@ export const web: DiffbotEndpoints['searchWeb'] = async (ctx, input) => { /** * Query the Diffbot Knowledge Graph using DQL (Diffbot Query Language). - * Docs: https://docs.diffbot.com/reference/dql-get + * Docs: https://docs.diffbot.com/reference/dqlget + * + * Uses the Knowledge Graph host: https://kg.diffbot.com/kg/v3/dql + * + * - `entityType`: optional DQL entity filter prepended to the query string (e.g. "Organization") + * - `queryType`: optional HTTP `type` parameter selecting the execution mode + * ("query" | "text" | "queryTextFallback" | "crawl"). Defaults to "query". */ export const dql: DiffbotEndpoints['searchDql'] = async (ctx, input) => { - const { query, type, size, from, col } = input; + const { query, entityType, queryType, size, from, col } = input; - // Build the DQL query string with optional type prefix - const fullQuery = type ? `type:${type} ${query}` : query; + // Build the DQL query string with optional entity type prefix + const fullQuery = entityType ? `type:${entityType} ${query}` : query; const response = await makeDiffbotRequest< DiffbotEndpointOutputs['searchDql'] >('dql', ctx.key, { method: 'GET', - query: { query: fullQuery, size, from, col }, + // Route to Knowledge Graph host (kg.diffbot.com/kg/v3) + useKgBase: true, + query: { + query: fullQuery, + // HTTP `type` controls execution mode (query/text/crawl etc.) + type: queryType, + size, + from, + col, + }, }); await logEventFromContext( diff --git a/packages/diffbot/endpoints/types.ts b/packages/diffbot/endpoints/types.ts index f53a1c66b..fe6453f5a 100644 --- a/packages/diffbot/endpoints/types.ts +++ b/packages/diffbot/endpoints/types.ts @@ -271,11 +271,21 @@ export type WebSearchResponse = z.infer; export const DqlSearchInputSchema = z.object({ query: z .string() - .describe('DQL query string (e.g. "type:Organization name:\"Google\"")'), - type: z + .describe( + "DQL query string (e.g. 'name:\"OpenAI\"'). Do NOT include a 'type:' prefix here — use entityType instead.", + ), + entityType: z .string() .optional() - .describe('Entity type filter (e.g. "Organization", "Person", "Article")'), + .describe( + 'Entity type filter prepended to the DQL query (e.g. "Organization", "Person", "Article")', + ), + queryType: z + .enum(['query', 'text', 'queryTextFallback', 'crawl']) + .optional() + .describe( + 'Execution mode for the DQL request. Use "crawl" with col to search crawl collections.', + ), size: z .number() .optional() @@ -286,7 +296,7 @@ export const DqlSearchInputSchema = z.object({ col: z .string() .optional() - .describe('Collection name for querying crawl/bulk data'), + .describe('Crawl collection name — only valid when queryType is "crawl"'), }); export type DqlSearchInput = z.infer; diff --git a/packages/diffbot/webhooks/types.ts b/packages/diffbot/webhooks/types.ts index 0fa39d023..336365b2d 100644 --- a/packages/diffbot/webhooks/types.ts +++ b/packages/diffbot/webhooks/types.ts @@ -57,6 +57,16 @@ export function verifyDiffbotWebhookSignature( request: WebhookRequest, secret: string, ): { valid: boolean; error?: string } { - // TODO: Implement webhook signature verification - return { valid: true }; + // Diffbot does not provide a native webhook signature mechanism. + // Accept only events that have been verified by Corsair Hub (hubVerified flag). + // This prevents unauthenticated callers from forging webhook events by + // setting an x-diffbot-signature header on arbitrary payloads. + if (request.hubVerified) { + return { valid: true }; + } + return { + valid: false, + error: + 'Diffbot webhook authentication is not configured. Hub verification is required.', + }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e8551a20..43a8e44ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -171,6 +171,9 @@ importers: '@corsair-dev/cursor': specifier: workspace:* version: link:../../packages/cursor + '@corsair-dev/diffbot': + specifier: workspace:* + version: link:../../packages/diffbot '@corsair-dev/firecrawl': specifier: workspace:* version: link:../../packages/firecrawl From 212e0c4d5f460b6243567c5b25dc28bf9c80bf22 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Sun, 23 Aug 2026 23:52:08 +0530 Subject: [PATCH 04/13] chore: revert demo/testing and pnpm-lock out-of-scope changes Removes demo/testing/package.json, demo/testing/src/server/corsair.ts, demo/testing/src/scripts/test-diffbot.ts and pnpm-lock.yaml changes that are outside the allowed plugin PR scope (R1). Plugin PR scope is limited to packages/diffbot/** and packages/corsair/core/constants.ts only. --- demo/testing/package.json | 1 - demo/testing/src/scripts/test-diffbot.ts | 130 ----------------------- demo/testing/src/server/corsair.ts | 48 +++++++-- pnpm-lock.yaml | 53 +++------ 4 files changed, 51 insertions(+), 181 deletions(-) delete mode 100644 demo/testing/src/scripts/test-diffbot.ts diff --git a/demo/testing/package.json b/demo/testing/package.json index 7a039e755..3dcf0da17 100644 --- a/demo/testing/package.json +++ b/demo/testing/package.json @@ -26,7 +26,6 @@ "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.0", "@corsair-dev/agentql": "workspace:*", - "@corsair-dev/diffbot": "workspace:*", "@corsair-dev/bitwarden": "workspace:*", "@corsair-dev/cursor": "workspace:*", "@corsair-dev/firecrawl": "workspace:*", diff --git a/demo/testing/src/scripts/test-diffbot.ts b/demo/testing/src/scripts/test-diffbot.ts deleted file mode 100644 index a7c22460f..000000000 --- a/demo/testing/src/scripts/test-diffbot.ts +++ /dev/null @@ -1,130 +0,0 @@ -import dotenv from 'dotenv'; - -dotenv.config({ path: '../.env' }); - -import { corsair } from '@/server/corsair'; - -const main = async () => { - console.log('🤖 Diffbot Plugin — Integration Test\n'); - - // ----------------------------------------------------------------------- - // 1. Extract Article - // ----------------------------------------------------------------------- - // Mock the endpoints so we can show successful execution without an API key - corsair.diffbot.api.extract.article = async () => - ({ - objects: [ - { - title: 'OpenAI launches GPT Store', - author: 'Jane Doe', - humanLanguage: 'en', - tags: [{ label: 'AI' }], - }, - ], - }) as any; - corsair.diffbot.api.extract.product = async () => - ({ - objects: [ - { - title: 'Apple iPhone 15 Pro', - offerPrice: '$999', - availability: 'InStock', - }, - ], - }) as any; - corsair.diffbot.api.extract.analyze = async () => - ({ - type: 'article', - humanLanguage: 'en', - }) as any; - corsair.diffbot.api.search.web = async () => - ({ - results: [ - { - title: 'Corsair AI platform released', - pageUrl: 'https://example.com/1', - }, - { title: 'How to build plugins', pageUrl: 'https://example.com/2' }, - ], - }) as any; - corsair.diffbot.api.search.dql = async () => - ({ - hits: 1, - data: [{ name: 'OpenAI', id: 'org_123' }], - }) as any; - - console.log('1️⃣ extract.article — TechCrunch headline'); - const article = await corsair.diffbot.api.extract.article({ - url: 'https://techcrunch.com/2024/01/15/openai-gpt-store/', - fields: 'tags,links', - }); - const obj = article.objects?.[0]; - console.log(` ✓ Title: ${obj?.title}`); - console.log(` ✓ Author: ${obj?.author}`); - console.log(` ✓ Language: ${obj?.humanLanguage}`); - console.log( - ` ✓ Tags: ${obj?.tags - ?.slice(0, 3) - .map((t) => t.label) - .join(', ')}\n`, - ); - - // ----------------------------------------------------------------------- - // 2. Extract Product - // ----------------------------------------------------------------------- - console.log('2️⃣ extract.product — Amazon product page'); - const product = await corsair.diffbot.api.extract.product({ - url: 'https://www.amazon.com/dp/B08N5WRWNW', - }); - const prod = product.objects?.[0]; - console.log(` ✓ Title: ${prod?.title}`); - console.log(` ✓ Price: ${prod?.offerPrice}`); - console.log(` ✓ Available: ${prod?.availability}\n`); - - // ----------------------------------------------------------------------- - // 3. Analyze (auto-detect page type) - // ----------------------------------------------------------------------- - console.log('3️⃣ extract.analyze — auto-detect page type'); - const analyzed = await corsair.diffbot.api.extract.analyze({ - url: 'https://www.bbc.com/news', - }); - console.log(` ✓ Detected type: ${analyzed.type}`); - console.log(` ✓ Language: ${analyzed.humanLanguage}\n`); - - // ----------------------------------------------------------------------- - // 4. Web Search - // ----------------------------------------------------------------------- - console.log('4️⃣ search.web — "Corsair AI integration platform"'); - const search = await corsair.diffbot.api.search.web({ - query: 'Corsair AI integration platform open source', - num: 3, - }); - const results = search.results ?? []; - console.log(` ✓ ${results.length} results returned`); - for (const r of results) { - console.log(` • ${r.title} (${r.pageUrl})`); - } - console.log(); - - // ----------------------------------------------------------------------- - // 5. DQL Knowledge Graph Search - // ----------------------------------------------------------------------- - console.log('5️⃣ search.dql — Knowledge Graph: OpenAI organization'); - const dql = await corsair.diffbot.api.search.dql({ - query: 'name:"OpenAI"', - type: 'Organization', - size: 1, - }); - const entity = dql.data?.[0] as Record | undefined; - console.log(` ✓ Hits: ${dql.hits}`); - console.log( - ` ✓ Entity: ${JSON.stringify(entity?.name ?? entity?.id ?? 'n/a')}\n`, - ); - - console.log('✅ All Diffbot endpoints responded successfully!'); -}; - -main().catch((err) => { - console.error('❌ Test failed:', err?.message ?? err); - process.exit(1); -}); diff --git a/demo/testing/src/server/corsair.ts b/demo/testing/src/server/corsair.ts index 01264e668..3755c8b3e 100644 --- a/demo/testing/src/server/corsair.ts +++ b/demo/testing/src/server/corsair.ts @@ -2,26 +2,32 @@ import dotenv from 'dotenv'; dotenv.config({ path: '../.env' }); -import { diffbot } from '@corsair-dev/diffbot'; +import { agentql } from '@corsair-dev/agentql'; +import { gmail } from '@corsair-dev/gmail'; +import { googlecalendar } from '@corsair-dev/googlecalendar'; +import { googlesheets } from '@corsair-dev/googlesheets'; +import { hubspot } from '@corsair-dev/hubspot'; +import { linear } from '@corsair-dev/linear'; +import { onedrive } from '@corsair-dev/onedrive'; +import { sharepoint } from '@corsair-dev/sharepoint'; +import { slack } from '@corsair-dev/slack'; +import { twilio } from '@corsair-dev/twilio'; +import { vapi } from '@corsair-dev/vapi'; import { createCorsair } from 'corsair'; import { sqlite } from '../db'; const hubProjectApiKey = - process.env.CORSAIR_DEV_API_KEY ?? - process.env.CORSAIR_API_KEY ?? - 'test_api_key'; + process.env.CORSAIR_DEV_API_KEY ?? process.env.CORSAIR_API_KEY!; const hubSigningSecret = - process.env.CORSAIR_DEV_SIGNING_SECRET ?? - process.env.CORSAIR_SIGNING_SECRET ?? - 'test_signing_secret'; + process.env.CORSAIR_DEV_SIGNING_SECRET ?? process.env.CORSAIR_SIGNING_SECRET!; // const hubApiUrl = process.env.HUB_API_URL; // const hubOAuthCallbackUrl = process.env.HUB_OAUTH_CALLBACK_URL; export const corsair = createCorsair({ multiTenancy: false, database: sqlite, - kek: process.env.CORSAIR_KEK ?? 'fallback_kek_for_testing_only', + kek: process.env.CORSAIR_KEK!, permissions: { timeout: '10m', onTimeout: 'deny', @@ -33,8 +39,30 @@ export const corsair = createCorsair({ signingSecret: hubSigningSecret, }, plugins: [ - diffbot({ - key: process.env.DIFFBOT_API_KEY, + // github({ authType: 'managed' }), + slack({ + permissions: { + mode: 'cautious', + overrides: { + 'messages.post': 'require_approval', + }, + }, }), + googlesheets(), + googlecalendar(), + gmail(), + linear(), + sharepoint(), + onedrive(), + hubspot(), + agentql({ + key: process.env.AGENTQL_API_KEY, + }), + twilio(), + vapi({ + key: process.env.VAPI_API_KEY, + webhookSecret: process.env.VAPI_WEBHOOK_SECRET, + }), + instagram(), ], }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43a8e44ce..18845437d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -171,9 +171,6 @@ importers: '@corsair-dev/cursor': specifier: workspace:* version: link:../../packages/cursor - '@corsair-dev/diffbot': - specifier: workspace:* - version: link:../../packages/diffbot '@corsair-dev/firecrawl': specifier: workspace:* version: link:../../packages/firecrawl @@ -740,7 +737,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/allimagesai: + packages/alphavantage: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -764,7 +761,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alphavantage: + packages/altoviz: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -788,7 +785,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/altoviz: + packages/alttextai: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -796,6 +793,9 @@ importers: corsair: specifier: workspace:* version: link:../corsair + dotenv: + specifier: ^17.2.3 + version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -812,17 +812,17 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alttextai: + packages/amara: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 + '@types/node': + specifier: ^24.10.1 + version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair - dotenv: - specifier: ^17.2.3 - version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -839,14 +839,11 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/amara: + packages/ambee: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 - '@types/node': - specifier: ^24.10.1 - version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair @@ -866,7 +863,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambee: + packages/ambientweather: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -890,7 +887,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambientweather: + packages/allimagesai: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -2281,30 +2278,6 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/diffbot: - devDependencies: - '@types/jest': - specifier: ^29.5.14 - version: 29.5.14 - corsair: - specifier: workspace:* - version: link:../corsair - jest: - specifier: ^29.7.0 - version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) - ts-jest: - specifier: ^29.4.9 - version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@30.4.1)(babel-jest@29.7.0(@babel/core@7.29.7))(esbuild@0.27.0)(jest-util@30.4.1)(jest@29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)))(typescript@5.9.3) - tsup: - specifier: ^8.0.1 - version: 8.5.1(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.9.0) - typescript: - specifier: 'catalog:' - version: 5.9.3 - zod: - specifier: 4.4.3 - version: 4.4.3 - packages/digitalocean: devDependencies: '@types/jest': From e5503aec9fffc1b4db545bb3a53331dac5195902 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Sun, 23 Aug 2026 23:59:13 +0530 Subject: [PATCH 05/13] fix(diffbot): fix unused import lint error and restore updated pnpm-lock.yaml - oauth-tenant-link.ts: remove unused asRecord import to fix biome lint check - pnpm-lock.yaml: restore lockfile changes so pnpm install can run with a frozen lockfile in CI checks --- .../diffbot/webhooks/oauth-tenant-link.ts | 2 +- pnpm-lock.yaml | 50 ++++++++++++++----- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/packages/diffbot/webhooks/oauth-tenant-link.ts b/packages/diffbot/webhooks/oauth-tenant-link.ts index 1c0ee456d..0326ae138 100644 --- a/packages/diffbot/webhooks/oauth-tenant-link.ts +++ b/packages/diffbot/webhooks/oauth-tenant-link.ts @@ -1,5 +1,5 @@ import type { TokenResponse, WebhookTenantMatch } from 'corsair/core'; -import { asRecord, toExternalId } from 'corsair/core'; +import { toExternalId } from 'corsair/core'; // TODO: Rename linkType 'tenant_external_id' to match pluginTenantWebhookMatcher. // Called after OAuth to store the routing id on corsair_accounts.config. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18845437d..8e8551a20 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -737,7 +737,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alphavantage: + packages/allimagesai: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -761,7 +761,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/altoviz: + packages/alphavantage: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -785,7 +785,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/alttextai: + packages/altoviz: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -793,9 +793,6 @@ importers: corsair: specifier: workspace:* version: link:../corsair - dotenv: - specifier: ^17.2.3 - version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -812,17 +809,17 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/amara: + packages/alttextai: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 - '@types/node': - specifier: ^24.10.1 - version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair + dotenv: + specifier: ^17.2.3 + version: 17.4.2 jest: specifier: ^29.7.0 version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) @@ -839,11 +836,14 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambee: + packages/amara: devDependencies: '@types/jest': specifier: ^29.5.14 version: 29.5.14 + '@types/node': + specifier: ^24.10.1 + version: 24.10.1 corsair: specifier: workspace:* version: link:../corsair @@ -863,7 +863,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/ambientweather: + packages/ambee: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -887,7 +887,7 @@ importers: specifier: 4.4.3 version: 4.4.3 - packages/allimagesai: + packages/ambientweather: devDependencies: '@types/jest': specifier: ^29.5.14 @@ -2278,6 +2278,30 @@ importers: specifier: 4.4.3 version: 4.4.3 + packages/diffbot: + devDependencies: + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + corsair: + specifier: workspace:* + version: link:../corsair + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) + ts-jest: + specifier: ^29.4.9 + version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@30.4.1)(babel-jest@29.7.0(@babel/core@7.29.7))(esbuild@0.27.0)(jest-util@30.4.1)(jest@29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)))(typescript@5.9.3) + tsup: + specifier: ^8.0.1 + version: 8.5.1(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.9.0) + typescript: + specifier: 'catalog:' + version: 5.9.3 + zod: + specifier: 4.4.3 + version: 4.4.3 + packages/digitalocean: devDependencies: '@types/jest': From 92f45e0768cfcde3da1972876890ce2d75373f4c Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Mon, 24 Aug 2026 00:21:19 +0530 Subject: [PATCH 06/13] fix(diffbot): address review findings --- packages/diffbot/api.test.ts | 132 ++++++++++++++++++ packages/diffbot/client.ts | 8 +- packages/diffbot/error-handlers.ts | 24 ++-- packages/diffbot/index.ts | 64 +-------- packages/diffbot/webhooks/example.ts | 32 ----- packages/diffbot/webhooks/index.ts | 9 -- .../diffbot/webhooks/oauth-tenant-link.ts | 31 ---- packages/diffbot/webhooks/tenant-matcher.ts | 25 ---- packages/diffbot/webhooks/types.ts | 72 ---------- 9 files changed, 156 insertions(+), 241 deletions(-) delete mode 100644 packages/diffbot/webhooks/example.ts delete mode 100644 packages/diffbot/webhooks/index.ts delete mode 100644 packages/diffbot/webhooks/oauth-tenant-link.ts delete mode 100644 packages/diffbot/webhooks/tenant-matcher.ts delete mode 100644 packages/diffbot/webhooks/types.ts diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index 1f7cbc69e..c493b1802 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -1,3 +1,5 @@ +import { makeDiffbotRequest } from './client'; +import { Extract, Search } from './endpoints'; import { AnalyzeInputSchema, AnalyzeResponseSchema, @@ -10,6 +12,20 @@ import { WebSearchInputSchema, WebSearchResponseSchema, } from './endpoints/types'; +import { errorHandlers } from './error-handlers'; + +// Mock the client request module +jest.mock('./client', () => ({ + makeDiffbotRequest: jest.fn(), +})); + +const mockRequest = makeDiffbotRequest as jest.MockedFunction< + typeof makeDiffbotRequest +>; + +const mockCtx = { + key: 'test-token', +} as any; // --------------------------------------------------------------------------- // Extract Article @@ -264,3 +280,119 @@ describe('search.dql — response schema', () => { expect(result.success).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// Endpoint Handler Verification (Mocked requests) +// --------------------------------------------------------------------------- + +describe('Diffbot endpoint handlers (mocked request mapping)', () => { + beforeEach(() => { + mockRequest.mockReset(); + }); + + it('extractArticle invokes client correctly', async () => { + mockRequest.mockResolvedValueOnce({ objects: [] }); + await Extract.article(mockCtx, { + url: 'https://example.com/article', + fields: 'meta,links', + }); + + expect(mockRequest).toHaveBeenCalledWith('article', 'test-token', { + method: 'GET', + query: { + url: 'https://example.com/article', + fields: 'meta,links', + }, + }); + }); + + it('extractProduct invokes client correctly', async () => { + mockRequest.mockResolvedValueOnce({ objects: [] }); + await Extract.product(mockCtx, { + url: 'https://example.com/product', + }); + + expect(mockRequest).toHaveBeenCalledWith('product', 'test-token', { + method: 'GET', + query: { + url: 'https://example.com/product', + }, + }); + }); + + it('extractAnalyze invokes client correctly', async () => { + mockRequest.mockResolvedValueOnce({ type: 'article' }); + await Extract.analyze(mockCtx, { + url: 'https://example.com/page', + fallback: 'article', + }); + + expect(mockRequest).toHaveBeenCalledWith('analyze', 'test-token', { + method: 'GET', + query: { + url: 'https://example.com/page', + fallback: 'article', + }, + }); + }); + + it('searchWeb invokes client correctly', async () => { + mockRequest.mockResolvedValueOnce({ results: [] }); + await Search.web(mockCtx, { + query: 'AI news', + num: 5, + }); + + expect(mockRequest).toHaveBeenCalledWith('search', 'test-token', { + method: 'GET', + query: { + query: 'AI news', + num: 5, + }, + }); + }); + + it('searchDql invokes client correctly and routes to Knowledge Graph base URL', async () => { + mockRequest.mockResolvedValueOnce({ data: [] }); + await Search.dql(mockCtx, { + query: 'name:"OpenAI"', + entityType: 'Organization', + queryType: 'query', + size: 10, + }); + + expect(mockRequest).toHaveBeenCalledWith('dql', 'test-token', { + method: 'GET', + useKgBase: true, + query: { + query: 'type:Organization name:"OpenAI"', + type: 'query', + size: 10, + }, + }); + }); +}); + +describe('Diffbot error handlers', () => { + it('recognizes rate limits and preserves retry timing', async () => { + const error = Object.assign(new Error('Too many requests'), { + status: 429, + retryAfter: 12_000, + }); + + expect(errorHandlers.RATE_LIMIT_ERROR.match(error)).toBe(true); + expect(await errorHandlers.RATE_LIMIT_ERROR.handler(error)).toEqual({ + maxRetries: 5, + headersRetryAfterMs: 12_000, + }); + }); + + it('does not retry authentication failures', async () => { + const error = Object.assign(new Error('Unauthorized'), { status: 401 }); + + expect(errorHandlers.AUTH_ERROR.match(error)).toBe(true); + expect(await errorHandlers.AUTH_ERROR.handler()).toEqual({ + maxRetries: 0, + }); + }); +}); diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index 429774e9a..b82e44bee 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -74,9 +74,13 @@ export async function makeDiffbotRequest( try { return await request(config, requestOptions); } catch (error) { - // Re-throw ApiError directly so errorHandlers can inspect .status and .retryAfter if (error instanceof ApiError) { - throw error; + throw new DiffbotAPIError( + error.message, + undefined, + error.status, + error.retryAfter, + ); } if (error instanceof Error) { throw new DiffbotAPIError(error.message); diff --git a/packages/diffbot/error-handlers.ts b/packages/diffbot/error-handlers.ts index 5a4f4c19f..658a757c2 100644 --- a/packages/diffbot/error-handlers.ts +++ b/packages/diffbot/error-handlers.ts @@ -1,24 +1,32 @@ import type { CorsairErrorHandler } from 'corsair/core'; -import { ApiError } from 'corsair/http'; + +type DiffbotError = Error & { + status?: number; + retryAfter?: number; +}; + +function hasStatus(error: Error, status: number): boolean { + return (error as DiffbotError).status === status; +} + +function retryAfter(error: Error): number | undefined { + return (error as DiffbotError).retryAfter; +} export const errorHandlers = { RATE_LIMIT_ERROR: { match: (error: Error) => { - if (error instanceof ApiError && error.status === 429) return true; + if (hasStatus(error, 429)) return true; const msg = error.message.toLowerCase(); return msg.includes('rate_limited') || msg.includes('429'); }, handler: async (error: Error) => { - let retryAfterMs: number | undefined; - if (error instanceof ApiError && error.retryAfter !== undefined) { - retryAfterMs = error.retryAfter; - } - return { maxRetries: 5, headersRetryAfterMs: retryAfterMs }; + return { maxRetries: 5, headersRetryAfterMs: retryAfter(error) }; }, }, AUTH_ERROR: { match: (error: Error) => { - if (error instanceof ApiError && error.status === 401) return true; + if (hasStatus(error, 401)) return true; const msg = error.message.toLowerCase(); return msg.includes('unauthorized') || msg.includes('invalid_auth'); }, diff --git a/packages/diffbot/index.ts b/packages/diffbot/index.ts index c5a2a95e0..5750b4e48 100644 --- a/packages/diffbot/index.ts +++ b/packages/diffbot/index.ts @@ -1,19 +1,16 @@ import type { AuthTypes, BindEndpoints, - BindWebhooks, CorsairEndpoint, CorsairErrorHandler, CorsairPlugin, CorsairPluginContext, - CorsairWebhook, KeyBuilderContext, PickAuth, PluginAuthConfig, PluginPermissionsConfig, RequiredPluginEndpointMeta, RequiredPluginEndpointSchemas, - RequiredPluginWebhookSchemas, } from 'corsair/core'; import { Extract, Search } from './endpoints'; import type { @@ -26,18 +23,11 @@ import { } from './endpoints/types'; import { errorHandlers } from './error-handlers'; import { DiffbotSchema } from './schema'; -import { ExampleWebhooks } from './webhooks'; -import { resolveDiffbotOAuthWebhookTenantLink } from './webhooks/oauth-tenant-link'; -import { matchDiffbotTenantWebhook } from './webhooks/tenant-matcher'; -import type { DiffbotWebhookOutputs, ExampleEvent } from './webhooks/types'; -import { ExampleEventSchema } from './webhooks/types'; export type DiffbotPluginOptions = { authType?: PickAuth<'api_key'>; key?: string; - webhookSecret?: string; hooks?: InternalDiffbotPlugin['hooks']; - webhookHooks?: InternalDiffbotPlugin['webhookHooks']; errorHandlers?: CorsairErrorHandler; permissions?: PluginPermissionsConfig; }; @@ -67,19 +57,6 @@ export type DiffbotEndpoints = { searchDql: DiffbotEndpoint<'searchDql'>; }; -type DiffbotWebhook< - K extends keyof DiffbotWebhookOutputs, - TEvent, -> = CorsairWebhook; - -export type DiffbotWebhooks = { - // Diffbot does not have a native webhook system. - // This placeholder webhook is kept for Corsair plugin structure compliance. - example: DiffbotWebhook<'example', ExampleEvent>; -}; - -export type DiffbotBoundWebhooks = BindWebhooks; - const diffbotEndpointsNested = { extract: { article: Extract.article, @@ -92,12 +69,6 @@ const diffbotEndpointsNested = { }, } as const; -const diffbotWebhooksNested = { - example: { - example: ExampleWebhooks.example, - }, -} as const; - export const diffbotEndpointSchemas = { 'extract.article': { input: DiffbotEndpointInputSchemas.extractArticle, @@ -123,15 +94,6 @@ export const diffbotEndpointSchemas = { typeof diffbotEndpointsNested >; -const diffbotWebhookSchemas = { - 'example.example': { - description: - 'Placeholder webhook event (Diffbot does not have a native webhook system)', - payload: ExampleEventSchema, - response: ExampleEventSchema, - }, -} as const satisfies RequiredPluginWebhookSchemas; - const defaultAuthType: AuthTypes = 'api_key' as const; const diffbotEndpointMeta = { @@ -172,7 +134,7 @@ export type BaseDiffbotPlugin = CorsairPlugin< 'diffbot', typeof DiffbotSchema, typeof diffbotEndpointsNested, - typeof diffbotWebhooksNested, + Record, T, typeof defaultAuthType >; @@ -195,33 +157,15 @@ export function diffbot( schema: DiffbotSchema, options: options, hooks: options.hooks, - webhookHooks: options.webhookHooks, endpoints: diffbotEndpointsNested, - webhooks: diffbotWebhooksNested, + webhooks: {}, endpointMeta: diffbotEndpointMeta, endpointSchemas: diffbotEndpointSchemas, - webhookSchemas: diffbotWebhookSchemas, - // Diffbot does not use webhook signatures — this is a no-op matcher - pluginWebhookMatcher: (request) => { - const headers = request.headers; - return 'x-diffbot-signature' in headers; - }, - pluginTenantWebhookMatcher: matchDiffbotTenantWebhook, - oauthWebhookTenantLinkResolver: resolveDiffbotOAuthWebhookTenantLink, errorHandlers: { ...errorHandlers, ...options.errorHandlers, }, keyBuilder: async (ctx: DiffbotKeyBuilderContext, source) => { - if (source === 'webhook' && options.webhookSecret) { - return options.webhookSecret; - } - - if (source === 'webhook') { - const res = await ctx.keys.get_webhook_signature(); - return res ?? ''; - } - if (source === 'endpoint' && options.key) { return options.key; } @@ -250,7 +194,3 @@ export type { WebSearchInput, WebSearchResponse, } from './endpoints/types'; -export type { - DiffbotWebhookOutputs, - ExampleEvent, -} from './webhooks/types'; diff --git a/packages/diffbot/webhooks/example.ts b/packages/diffbot/webhooks/example.ts deleted file mode 100644 index d1c676150..000000000 --- a/packages/diffbot/webhooks/example.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { logEventFromContext } from 'corsair/core'; -import type { DiffbotWebhooks } from '..'; -import { createDiffbotMatch, verifyDiffbotWebhookSignature } from './types'; - -export const example: DiffbotWebhooks['example'] = { - match: createDiffbotMatch('example'), - - handler: async (ctx, request) => { - const verification = verifyDiffbotWebhookSignature(request, ctx.key); - if (!verification.valid) { - return { - success: false, - statusCode: 401, - error: verification.error || 'Signature verification failed', - }; - } - - const event = request.payload; - if (event.type !== 'example') { - return { success: true, data: undefined }; - } - - await logEventFromContext( - ctx, - 'diffbot.webhook.example', - { ...event }, - 'completed', - ); - - return { success: true, data: event }; - }, -}; diff --git a/packages/diffbot/webhooks/index.ts b/packages/diffbot/webhooks/index.ts deleted file mode 100644 index a12134e8a..000000000 --- a/packages/diffbot/webhooks/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { example } from './example'; - -export const ExampleWebhooks = { - example: example, -}; - -export * from './oauth-tenant-link'; -export * from './tenant-matcher'; -export * from './types'; diff --git a/packages/diffbot/webhooks/oauth-tenant-link.ts b/packages/diffbot/webhooks/oauth-tenant-link.ts deleted file mode 100644 index 0326ae138..000000000 --- a/packages/diffbot/webhooks/oauth-tenant-link.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { TokenResponse, WebhookTenantMatch } from 'corsair/core'; -import { toExternalId } from 'corsair/core'; - -// TODO: Rename linkType 'tenant_external_id' to match pluginTenantWebhookMatcher. -// Called after OAuth to store the routing id on corsair_accounts.config. -export async function resolveDiffbotOAuthWebhookTenantLink( - tokens: TokenResponse, -): Promise { - // TODO: Read from token response when the provider includes a stable id. - // const externalId = toExternalId(asRecord(tokens.team)?.id); - const externalId = toExternalId(tokens.tenant_external_id); - if (externalId) { - return { linkType: 'tenant_external_id', externalId }; - } - - const accessToken = tokens.access_token; - if (!accessToken) return null; - - // TODO: Fetch from provider API when the token response omits the id. - // const response = await fetch('https://api.example.com/me', { - // headers: { Authorization: `Bearer ${accessToken}` }, - // }); - // if (!response.ok) return null; - // const payload = (await response.json()) as { id?: string }; - // const fetchedId = toExternalId(payload.id); - // return fetchedId - // ? { linkType: 'tenant_external_id', externalId: fetchedId } - // : null; - - return null; -} diff --git a/packages/diffbot/webhooks/tenant-matcher.ts b/packages/diffbot/webhooks/tenant-matcher.ts deleted file mode 100644 index f14a67b7f..000000000 --- a/packages/diffbot/webhooks/tenant-matcher.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { RawWebhookRequest, WebhookTenantMatch } from 'corsair/core'; -import { asRecord, firstString, readBodyRecord } from 'corsair/core'; - -// TODO: Rename linkType 'tenant_external_id' to match the provider field -// (e.g. team_id, installation_id, organization_id). Must match authConfig.account -// and oauthWebhookTenantLinkResolver. -// Return null for URL verification / handshake payloads that have no tenant id. -export function matchDiffbotTenantWebhook( - request: RawWebhookRequest, -): WebhookTenantMatch | null { - const body = readBodyRecord(request); - if (!body) return null; - - // TODO: Extract the stable external id from the webhook payload. - // Example: - // const externalId = firstString([body.tenant_external_id, asRecord(body.data)?.id]); - const externalId = firstString([ - body.tenant_external_id, - asRecord(body.data)?.tenant_external_id, - ]); - - if (!externalId) return null; - - return { linkType: 'tenant_external_id', externalId }; -} diff --git a/packages/diffbot/webhooks/types.ts b/packages/diffbot/webhooks/types.ts deleted file mode 100644 index 336365b2d..000000000 --- a/packages/diffbot/webhooks/types.ts +++ /dev/null @@ -1,72 +0,0 @@ -import type { - CorsairWebhookMatcher, - RawWebhookRequest, - WebhookRequest, -} from 'corsair/core'; -import { z } from 'zod'; - -export const DiffbotWebhookPayloadSchema = z.object({ - type: z.string(), - created_at: z.string(), - data: z.record(z.string(), z.unknown()), -}); - -export type DiffbotWebhookPayload = z.infer; - -export const ExampleEventSchema = DiffbotWebhookPayloadSchema.extend({ - type: z.literal('example'), - data: z - .object({ - id: z.string(), - }) - .loose(), -}); - -export type ExampleEvent = z.infer; - -export type DiffbotWebhookOutputs = { - example: ExampleEvent; -}; - -function parseBody(body: unknown): Record | null { - if (typeof body === 'string') { - try { - const parsed = JSON.parse(body); - return parsed !== null && - typeof parsed === 'object' && - !Array.isArray(parsed) - ? (parsed as Record) - : null; - } catch { - return null; - } - } - return body !== null && typeof body === 'object' && !Array.isArray(body) - ? (body as Record) - : null; -} - -export function createDiffbotMatch(eventType: string): CorsairWebhookMatcher { - return (request: RawWebhookRequest) => { - const parsedBody = parseBody(request.body); - return parsedBody !== null && parsedBody.type === eventType; - }; -} - -export function verifyDiffbotWebhookSignature( - request: WebhookRequest, - secret: string, -): { valid: boolean; error?: string } { - // Diffbot does not provide a native webhook signature mechanism. - // Accept only events that have been verified by Corsair Hub (hubVerified flag). - // This prevents unauthenticated callers from forging webhook events by - // setting an x-diffbot-signature header on arbitrary payloads. - if (request.hubVerified) { - return { valid: true }; - } - return { - valid: false, - error: - 'Diffbot webhook authentication is not configured. Hub verification is required.', - }; -} From 443ec842afe18802091dae6053827f8c56e9e4f8 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Mon, 24 Aug 2026 00:24:42 +0530 Subject: [PATCH 07/13] fix(diffbot): preserve API error metadata --- packages/diffbot/client.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index b82e44bee..95e007fc0 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -75,12 +75,7 @@ export async function makeDiffbotRequest( return await request(config, requestOptions); } catch (error) { if (error instanceof ApiError) { - throw new DiffbotAPIError( - error.message, - undefined, - error.status, - error.retryAfter, - ); + throw error; } if (error instanceof Error) { throw new DiffbotAPIError(error.message); From ec8af0b8256b9462cf5f6c7c8ae859d4035a2d57 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Mon, 24 Aug 2026 00:27:00 +0530 Subject: [PATCH 08/13] test(diffbot): type endpoint test context --- packages/diffbot/api.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index c493b1802..05b95a2a0 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -13,6 +13,7 @@ import { WebSearchResponseSchema, } from './endpoints/types'; import { errorHandlers } from './error-handlers'; +import type { DiffbotContext } from './index'; // Mock the client request module jest.mock('./client', () => ({ @@ -25,7 +26,7 @@ const mockRequest = makeDiffbotRequest as jest.MockedFunction< const mockCtx = { key: 'test-token', -} as any; +} as DiffbotContext; // --------------------------------------------------------------------------- // Extract Article From 56d5a29bff3e3750249244cf5393ecaea56e16bf Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Mon, 24 Aug 2026 00:38:46 +0530 Subject: [PATCH 09/13] fix(diffbot): validate DQL collection mode --- packages/diffbot/api.test.ts | 17 +++++++ packages/diffbot/endpoints/types.ts | 70 ++++++++++++++++------------- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index 05b95a2a0..824d188ef 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -248,6 +248,23 @@ describe('search.dql — input schema', () => { } }); + it('rejects collection without crawl queryType', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'type:Article', + col: 'my_collection', + }); + expect(result.success).toBe(false); + }); + + it('rejects collection for non-crawl queryType', () => { + const result = DqlSearchInputSchema.safeParse({ + query: 'type:Article', + queryType: 'query', + col: 'my_collection', + }); + expect(result.success).toBe(false); + }); + it('rejects invalid queryType', () => { const result = DqlSearchInputSchema.safeParse({ query: 'test', diff --git a/packages/diffbot/endpoints/types.ts b/packages/diffbot/endpoints/types.ts index fe6453f5a..1de08a551 100644 --- a/packages/diffbot/endpoints/types.ts +++ b/packages/diffbot/endpoints/types.ts @@ -268,36 +268,46 @@ export type WebSearchResponse = z.infer; // DQL (Knowledge Graph Search) // --------------------------------------------------------------------------- -export const DqlSearchInputSchema = z.object({ - query: z - .string() - .describe( - "DQL query string (e.g. 'name:\"OpenAI\"'). Do NOT include a 'type:' prefix here — use entityType instead.", - ), - entityType: z - .string() - .optional() - .describe( - 'Entity type filter prepended to the DQL query (e.g. "Organization", "Person", "Article")', - ), - queryType: z - .enum(['query', 'text', 'queryTextFallback', 'crawl']) - .optional() - .describe( - 'Execution mode for the DQL request. Use "crawl" with col to search crawl collections.', - ), - size: z - .number() - .optional() - .describe( - 'Number of results to return (default 5, max 100 or 1000 for articles)', - ), - from: z.number().optional().describe('Zero-indexed offset for pagination'), - col: z - .string() - .optional() - .describe('Crawl collection name — only valid when queryType is "crawl"'), -}); +export const DqlSearchInputSchema = z + .object({ + query: z + .string() + .describe( + "DQL query string (e.g. 'name:\"OpenAI\"'). Do NOT include a 'type:' prefix here — use entityType instead.", + ), + entityType: z + .string() + .optional() + .describe( + 'Entity type filter prepended to the DQL query (e.g. "Organization", "Person", "Article")', + ), + queryType: z + .enum(['query', 'text', 'queryTextFallback', 'crawl']) + .optional() + .describe( + 'Execution mode for the DQL request. Use "crawl" with col to search crawl collections.', + ), + size: z + .number() + .optional() + .describe( + 'Number of results to return (default 5, max 100 or 1000 for articles)', + ), + from: z.number().optional().describe('Zero-indexed offset for pagination'), + col: z + .string() + .optional() + .describe('Crawl collection name — only valid when queryType is "crawl"'), + }) + .superRefine((input, ctx) => { + if (input.col !== undefined && input.queryType !== 'crawl') { + ctx.addIssue({ + code: 'custom', + path: ['col'], + message: 'col is only valid when queryType is "crawl"', + }); + } + }); export type DqlSearchInput = z.infer; From a81fa173a0c083e78b38a8b0ff13bfcbd5566a42 Mon Sep 17 00:00:00 2001 From: Shivashankar15 Date: Mon, 24 Aug 2026 00:53:04 +0530 Subject: [PATCH 10/13] fix(diffbot): protect authenticated token --- packages/diffbot/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index 95e007fc0..07b28509f 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -56,8 +56,8 @@ export async function makeDiffbotRequest( // Diffbot auth: token is a query parameter, not a header const queryWithToken: Record = { - token, ...query, + token, }; const requestOptions: ApiRequestOptions = { From dac52e112a2b79ac0fef556c1b3b28a3ea5d43cb Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Mon, 24 Aug 2026 02:29:03 +0530 Subject: [PATCH 11/13] feat(diffbot): implement all 35 Diffbot operations with database schemas and tests --- packages/diffbot/api.test.ts | 697 +++++---- packages/diffbot/client.ts | 53 +- packages/diffbot/endpoints/account.ts | 17 + packages/diffbot/endpoints/bulk.ts | 113 ++ packages/diffbot/endpoints/crawl.ts | 84 ++ packages/diffbot/endpoints/custom-api.ts | 70 + packages/diffbot/endpoints/enhance.ts | 106 ++ packages/diffbot/endpoints/extract.ts | 208 ++- packages/diffbot/endpoints/index.ts | 12 +- packages/diffbot/endpoints/kg-bulk-enhance.ts | 172 +++ packages/diffbot/endpoints/search.ts | 73 +- packages/diffbot/endpoints/types.ts | 1309 ++++++++++++++--- packages/diffbot/error-handlers.ts | 41 +- packages/diffbot/index.ts | 440 +++++- packages/diffbot/schema.test.ts | 23 +- packages/diffbot/schema/database.ts | 564 ++++++- packages/diffbot/schema/index.ts | 36 +- 17 files changed, 3339 insertions(+), 679 deletions(-) create mode 100644 packages/diffbot/endpoints/account.ts create mode 100644 packages/diffbot/endpoints/bulk.ts create mode 100644 packages/diffbot/endpoints/crawl.ts create mode 100644 packages/diffbot/endpoints/custom-api.ts create mode 100644 packages/diffbot/endpoints/enhance.ts create mode 100644 packages/diffbot/endpoints/kg-bulk-enhance.ts diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index 824d188ef..092ad3e6b 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -1,416 +1,487 @@ -import { makeDiffbotRequest } from './client'; -import { Extract, Search } from './endpoints'; +import * as clientModule from './client'; import { - AnalyzeInputSchema, - AnalyzeResponseSchema, - DqlSearchInputSchema, - DqlSearchResponseSchema, - ExtractArticleInputSchema, - ExtractArticleResponseSchema, - ExtractProductInputSchema, - ExtractProductResponseSchema, - WebSearchInputSchema, - WebSearchResponseSchema, + Account, + Bulk, + Crawl, + CustomApi, + Enhance, + Extract, + KgBulkEnhance, + Search, +} from './endpoints'; +import { + DiffbotEndpointInputSchemas, + DiffbotEndpointOutputSchemas, } from './endpoints/types'; import { errorHandlers } from './error-handlers'; -import type { DiffbotContext } from './index'; +import { diffbot } from './index'; + +describe('Diffbot Input and Output Schemas', () => { + // Account + describe('account.getAccount', () => { + it('accepts empty input', () => { + const parsed = DiffbotEndpointInputSchemas.getAccount.parse({}); + expect(parsed).toEqual({}); + }); -// Mock the client request module -jest.mock('./client', () => ({ - makeDiffbotRequest: jest.fn(), -})); + it('parses valid output', () => { + const parsed = DiffbotEndpointOutputSchemas.getAccount.parse({ + token: 'test_token', + name: 'Test User', + plan: 'kgfree', + planCalls: 10000, + status: 'active', + }); + expect(parsed.name).toBe('Test User'); + expect(parsed.planCalls).toBe(10000); + }); + }); -const mockRequest = makeDiffbotRequest as jest.MockedFunction< - typeof makeDiffbotRequest ->; + // Extract + describe('extract.getArticle', () => { + it('accepts valid url and optional fields', () => { + const input = DiffbotEndpointInputSchemas.getArticle.parse({ + url: 'https://example.com/article', + fields: 'links,meta', + }); + expect(input.url).toBe('https://example.com/article'); + }); -const mockCtx = { - key: 'test-token', -} as DiffbotContext; + it('parses article response', () => { + const output = DiffbotEndpointOutputSchemas.getArticle.parse({ + objects: [ + { + type: 'article', + title: 'Test Article', + text: 'Body text', + }, + ], + }); + expect(output.objects[0]?.title).toBe('Test Article'); + }); + }); -// --------------------------------------------------------------------------- -// Extract Article -// --------------------------------------------------------------------------- + describe('extract.getProduct', () => { + it('accepts valid product url', () => { + const input = DiffbotEndpointInputSchemas.getProduct.parse({ + url: 'https://example.com/product', + }); + expect(input.url).toBe('https://example.com/product'); + }); -describe('extract.article — input schema', () => { - it('accepts a valid URL', () => { - const result = ExtractArticleInputSchema.safeParse({ - url: 'https://techcrunch.com/2024/01/01/example-article', + it('parses product response', () => { + const output = DiffbotEndpointOutputSchemas.getProduct.parse({ + objects: [ + { + type: 'product', + title: 'Test Product', + offerPrice: '$99.00', + }, + ], + }); + expect(output.objects[0]?.offerPrice).toBe('$99.00'); }); - expect(result.success).toBe(true); }); - it('accepts optional fields param', () => { - const result = ExtractArticleInputSchema.safeParse({ - url: 'https://example.com', - fields: 'links,meta,tags', - timeout: 30000, + describe('extract.getAnalyze', () => { + it('accepts url with fallback', () => { + const input = DiffbotEndpointInputSchemas.getAnalyze.parse({ + url: 'https://example.com/page', + fallback: 'article', + }); + expect(input.fallback).toBe('article'); }); - expect(result.success).toBe(true); - }); - it('rejects missing url', () => { - const result = ExtractArticleInputSchema.safeParse({}); - expect(result.success).toBe(false); + it('parses analyze response', () => { + const output = DiffbotEndpointOutputSchemas.getAnalyze.parse({ + type: 'article', + title: 'Detected Article', + }); + expect(output.type).toBe('article'); + }); }); -}); -describe('extract.article — response schema', () => { - it('parses a valid article response', () => { - const payload = { - request: { pageUrl: 'https://example.com', api: 'article', version: 3 }, - objects: [ - { - type: 'article' as const, - title: 'Test Article', - text: 'Article body text', - author: 'John Doe', - date: '2024-01-01T00:00:00.000Z', - pageUrl: 'https://example.com', - humanLanguage: 'en', - }, - ], - }; - const result = ExtractArticleResponseSchema.safeParse(payload); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.objects[0]?.title).toBe('Test Article'); - expect(result.data.objects[0]?.author).toBe('John Doe'); - } - }); + describe('extract.getImage', () => { + it('accepts url', () => { + const input = DiffbotEndpointInputSchemas.getImage.parse({ + url: 'https://example.com/image.png', + }); + expect(input.url).toBe('https://example.com/image.png'); + }); - it('parses response with optional fields missing', () => { - const result = ExtractArticleResponseSchema.safeParse({ - objects: [{ type: 'article' }], + it('parses image response', () => { + const output = DiffbotEndpointOutputSchemas.getImage.parse({ + objects: [{ type: 'image', url: 'https://example.com/image.png' }], + }); + expect(output.objects[0]?.url).toBe('https://example.com/image.png'); }); - expect(result.success).toBe(true); }); -}); -// --------------------------------------------------------------------------- -// Extract Product -// --------------------------------------------------------------------------- + describe('extract.getVideo', () => { + it('accepts video url', () => { + const input = DiffbotEndpointInputSchemas.getVideo.parse({ + url: 'https://example.com/video', + }); + expect(input.url).toBe('https://example.com/video'); + }); -describe('extract.product — input schema', () => { - it('accepts a valid product URL', () => { - const result = ExtractProductInputSchema.safeParse({ - url: 'https://www.amazon.com/dp/B08N5WRWNW', + it('parses video response', () => { + const output = DiffbotEndpointOutputSchemas.getVideo.parse({ + objects: [{ type: 'video', duration: 120 }], + }); + expect(output.objects[0]?.duration).toBe(120); }); - expect(result.success).toBe(true); }); - it('rejects missing url', () => { - const result = ExtractProductInputSchema.safeParse({ timeout: 5000 }); - expect(result.success).toBe(false); - }); -}); + describe('extract.getDiscussion', () => { + it('accepts discussion url', () => { + const input = DiffbotEndpointInputSchemas.getDiscussion.parse({ + url: 'https://example.com/forum', + }); + expect(input.url).toBe('https://example.com/forum'); + }); -describe('extract.product — response schema', () => { - it('parses a valid product response', () => { - const payload = { - objects: [ - { - type: 'product' as const, - title: 'Example Product', - offerPrice: '$29.99', - availability: true, - brand: 'Acme', - pageUrl: 'https://example.com/product', - }, - ], - }; - const result = ExtractProductResponseSchema.safeParse(payload); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.objects[0]?.offerPrice).toBe('$29.99'); - expect(result.data.objects[0]?.availability).toBe(true); - } + it('parses discussion response', () => { + const output = DiffbotEndpointOutputSchemas.getDiscussion.parse({ + objects: [{ type: 'discussion', numPosts: 5 }], + }); + expect(output.objects[0]?.numPosts).toBe(5); + }); }); -}); -// --------------------------------------------------------------------------- -// Analyze (auto-detect) -// --------------------------------------------------------------------------- + describe('extract.getEvent', () => { + it('accepts event url', () => { + const input = DiffbotEndpointInputSchemas.getEvent.parse({ + url: 'https://example.com/event', + }); + expect(input.url).toBe('https://example.com/event'); + }); -describe('extract.analyze — input schema', () => { - it('accepts a URL with fallback option', () => { - const result = AnalyzeInputSchema.safeParse({ - url: 'https://example.com', - fallback: 'article', + it('parses event response', () => { + const output = DiffbotEndpointOutputSchemas.getEvent.parse({ + objects: [{ type: 'event', startDate: '2026-09-01' }], + }); + expect(output.objects[0]?.startDate).toBe('2026-09-01'); }); - expect(result.success).toBe(true); }); - it('rejects empty object', () => { - const result = AnalyzeInputSchema.safeParse({}); - expect(result.success).toBe(false); - }); -}); + describe('extract.extractList', () => { + it('accepts list url', () => { + const input = DiffbotEndpointInputSchemas.extractList.parse({ + url: 'https://example.com/list', + }); + expect(input.url).toBe('https://example.com/list'); + }); -describe('extract.analyze — response schema', () => { - it('parses an analyze response with detected type', () => { - const result = AnalyzeResponseSchema.safeParse({ - type: 'article', - humanLanguage: 'en', - objects: [{ type: 'article', title: 'Detected Article' }], - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.type).toBe('article'); - } + it('parses list response', () => { + const output = DiffbotEndpointOutputSchemas.extractList.parse({ + objects: [{ type: 'list', numItems: 10 }], + }); + expect(output.objects[0]?.numItems).toBe(10); + }); }); -}); -// --------------------------------------------------------------------------- -// Web Search -// --------------------------------------------------------------------------- + describe('extract.extractJob', () => { + it('accepts job url', () => { + const input = DiffbotEndpointInputSchemas.extractJob.parse({ + url: 'https://example.com/job', + }); + expect(input.url).toBe('https://example.com/job'); + }); -describe('search.web — input schema', () => { - it('accepts a valid search query', () => { - const result = WebSearchInputSchema.safeParse({ - query: 'artificial intelligence trends 2024', - num: 10, + it('parses job response', () => { + const output = DiffbotEndpointOutputSchemas.extractJob.parse({ + objects: [{ type: 'job', title: 'Software Engineer' }], + }); + expect(output.objects[0]?.title).toBe('Software Engineer'); }); - expect(result.success).toBe(true); }); - it('rejects num > 25 (Diffbot max)', () => { - const result = WebSearchInputSchema.safeParse({ - query: 'test', - num: 100, + // Search + describe('search.search & search.searchCrawlData', () => { + it('accepts dql search query', () => { + const input = DiffbotEndpointInputSchemas.search.parse({ + query: 'name:"Diffbot"', + entityType: 'Organization', + }); + expect(input.query).toBe('name:"Diffbot"'); }); - expect(result.success).toBe(false); - }); - it('rejects missing query', () => { - const result = WebSearchInputSchema.safeParse({ num: 5 }); - expect(result.success).toBe(false); - }); -}); + it('parses dql search response', () => { + const output = DiffbotEndpointOutputSchemas.search.parse({ + hits: 1, + data: [{ name: 'Diffbot' }], + }); + expect(output.hits).toBe(1); + }); -describe('search.web — response schema', () => { - it('parses a valid search response', () => { - const payload = { - results: [ - { - title: 'AI in 2024', - pageUrl: 'https://example.com/ai-2024', - text: 'Article summary...', - humanLanguage: 'en', - }, - ], - numResults: 1, - hits: 1, - }; - const result = WebSearchResponseSchema.safeParse(payload); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.results?.[0]?.title).toBe('AI in 2024'); - expect(result.data.numResults).toBe(1); - } + it('accepts crawl data search query', () => { + const input = DiffbotEndpointInputSchemas.searchCrawlData.parse({ + col: 'myCollection', + query: 'tech', + num: 10, + }); + expect(input.col).toBe('myCollection'); + }); }); -}); -// --------------------------------------------------------------------------- -// DQL (Knowledge Graph Search) -// --------------------------------------------------------------------------- + // Enhance + describe('enhance endpoints', () => { + it('accepts enhanceEntity input', () => { + const input = DiffbotEndpointInputSchemas.enhanceEntity.parse({ + name: 'Diffbot', + type: 'Organization', + }); + expect(input.name).toBe('Diffbot'); + }); -describe('search.dql — input schema', () => { - it('accepts a DQL query with entityType filter', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'name:"OpenAI"', - entityType: 'Organization', - size: 5, + it('accepts combineEntityProfiles input', () => { + const input = DiffbotEndpointInputSchemas.combineEntityProfiles.parse({ + name: 'John Doe', + employer: 'Acme', + }); + expect(input.name).toBe('John Doe'); }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.entityType).toBe('Organization'); - } - }); - it('accepts a query without optional entityType', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'diffbot', + it('accepts resolveLostId input', () => { + const input = DiffbotEndpointInputSchemas.resolveLostId.parse({ + id: 'legacy-id-123', + }); + expect(input.id).toBe('legacy-id-123'); }); - expect(result.success).toBe(true); - }); - it('accepts crawl queryType with collection', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'type:Article', - queryType: 'crawl', - col: 'my_collection', - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.queryType).toBe('crawl'); - expect(result.data.col).toBe('my_collection'); - } + it('accepts getKgCoverageReportById input', () => { + const input = DiffbotEndpointInputSchemas.getKgCoverageReportById.parse({ + reportId: 'rep_123', + }); + expect(input.reportId).toBe('rep_123'); + }); }); - it('rejects collection without crawl queryType', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'type:Article', - col: 'my_collection', + // KG Bulk Enhance + describe('kgBulkEnhance endpoints', () => { + it('accepts createKgBulkEnhance input', () => { + const input = DiffbotEndpointInputSchemas.createKgBulkEnhance.parse({ + entities: [{ name: 'Company A' }, { name: 'Company B' }], + name: 'testJob', + }); + expect(input.entities.length).toBe(2); }); - expect(result.success).toBe(false); - }); - it('rejects collection for non-crawl queryType', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'type:Article', - queryType: 'query', - col: 'my_collection', + it('accepts getBulkJobStatus input', () => { + const input = DiffbotEndpointInputSchemas.getBulkJobStatus.parse({ + bulkjobId: 'bulk_123', + }); + expect(input.bulkjobId).toBe('bulk_123'); }); - expect(result.success).toBe(false); - }); - it('rejects invalid queryType', () => { - const result = DqlSearchInputSchema.safeParse({ - query: 'test', - queryType: 'invalid_mode', + it('accepts getBulkSingleResult input', () => { + const input = DiffbotEndpointInputSchemas.getBulkSingleResult.parse({ + bulkjobId: 'bulk_123', + jobIndex: 0, + }); + expect(input.jobIndex).toBe(0); }); - expect(result.success).toBe(false); }); - it('rejects missing query', () => { - const result = DqlSearchInputSchema.safeParse({ size: 5 }); - expect(result.success).toBe(false); + // Bulk Extract + describe('bulk extract endpoints', () => { + it('accepts createBulk input', () => { + const input = DiffbotEndpointInputSchemas.createBulk.parse({ + name: 'myBulk', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: ['https://example.com/1', 'https://example.com/2'], + }); + expect(input.urls.length).toBe(2); + }); + + it('accepts startBulk input', () => { + const input = DiffbotEndpointInputSchemas.startBulk.parse({ + name: 'myBulk', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + }); + expect(input.name).toBe('myBulk'); + }); }); -}); -describe('search.dql — response schema', () => { - it('parses a valid DQL response', () => { - const payload = { - data: [{ id: 'org-123', name: 'OpenAI', type: 'Organization' }], - hits: 1, - }; - const result = DqlSearchResponseSchema.safeParse(payload); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.hits).toBe(1); - expect(result.data.data).toHaveLength(1); - } + // Crawl + describe('crawl endpoints', () => { + it('accepts startCrawl input', () => { + const input = DiffbotEndpointInputSchemas.startCrawl.parse({ + name: 'crawl1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + }); + expect(input.name).toBe('crawl1'); + }); + + it('accepts manageCrawl input', () => { + const input = DiffbotEndpointInputSchemas.manageCrawl.parse({ + name: 'crawl1', + pause: 1, + }); + expect(input.pause).toBe(1); + }); }); - it('parses an empty result set', () => { - const result = DqlSearchResponseSchema.safeParse({ data: [], hits: 0 }); - expect(result.success).toBe(true); + // Custom API + describe('customApi endpoints', () => { + it('accepts createCustomApi input', () => { + const input = DiffbotEndpointInputSchemas.createCustomApi.parse({ + api: 'custom1', + url: 'https://example.com', + }); + expect(input.api).toBe('custom1'); + }); + + it('accepts deleteCustomApi input', () => { + const input = DiffbotEndpointInputSchemas.deleteCustomApi.parse({ + api: 'custom1', + }); + expect(input.api).toBe('custom1'); + }); }); }); -// --------------------------------------------------------------------------- -// Endpoint Handler Verification (Mocked requests) -// --------------------------------------------------------------------------- +describe('Diffbot Endpoint Handlers', () => { + let makeRequestSpy: jest.SpyInstance; + const mockCtx = { + key: 'test_token', + authType: 'api_key' as const, + options: { key: 'test_token' }, + database: {}, + $getAccountId: () => 'acc_test', + } as unknown as Parameters[0]; -describe('Diffbot endpoint handlers (mocked request mapping)', () => { beforeEach(() => { - mockRequest.mockReset(); + makeRequestSpy = jest + .spyOn(clientModule, 'makeDiffbotRequest') + .mockResolvedValue({ status: 200 } as never); }); - it('extractArticle invokes client correctly', async () => { - mockRequest.mockResolvedValueOnce({ objects: [] }); - await Extract.article(mockCtx, { - url: 'https://example.com/article', - fields: 'meta,links', - }); + afterEach(() => { + jest.restoreAllMocks(); + }); - expect(mockRequest).toHaveBeenCalledWith('article', 'test-token', { + it('invokes account.getAccount correctly', async () => { + await Account.getAccount(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('account', 'test_token', { method: 'GET', - query: { - url: 'https://example.com/article', - fields: 'meta,links', - }, }); }); - it('extractProduct invokes client correctly', async () => { - mockRequest.mockResolvedValueOnce({ objects: [] }); - await Extract.product(mockCtx, { - url: 'https://example.com/product', - }); - - expect(mockRequest).toHaveBeenCalledWith('product', 'test-token', { + it('invokes extract.getArticle correctly', async () => { + await Extract.getArticle(mockCtx, { url: 'https://example.com/article' }); + expect(makeRequestSpy).toHaveBeenCalledWith('article', 'test_token', { method: 'GET', - query: { - url: 'https://example.com/product', - }, + query: expect.objectContaining({ url: 'https://example.com/article' }), }); }); - it('extractAnalyze invokes client correctly', async () => { - mockRequest.mockResolvedValueOnce({ type: 'article' }); - await Extract.analyze(mockCtx, { - url: 'https://example.com/page', - fallback: 'article', + it('invokes search.search with DQL routing to KG base', async () => { + await Search.search(mockCtx, { + query: 'name:"Diffbot"', + entityType: 'Organization', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('dql', 'test_token', { + method: 'GET', + useKgBase: true, + query: expect.objectContaining({ + query: 'type:Organization name:"Diffbot"', + }), }); + }); - expect(mockRequest).toHaveBeenCalledWith('analyze', 'test-token', { + it('invokes enhance.enhanceEntity correctly', async () => { + await Enhance.enhanceEntity(mockCtx, { name: 'Diffbot' }); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance', 'test_token', { method: 'GET', - query: { - url: 'https://example.com/page', - fallback: 'article', - }, + useKgBase: true, + query: expect.objectContaining({ name: 'Diffbot' }), }); }); - it('searchWeb invokes client correctly', async () => { - mockRequest.mockResolvedValueOnce({ results: [] }); - await Search.web(mockCtx, { - query: 'AI news', - num: 5, + it('invokes kgBulkEnhance.createKgBulkEnhance correctly', async () => { + await KgBulkEnhance.createKgBulkEnhance(mockCtx, { + entities: [{ name: 'Diffbot' }], + }); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance/bulk', 'test_token', { + method: 'POST', + useKgBase: true, + body: [{ name: 'Diffbot' }], + query: expect.anything(), }); + }); - expect(mockRequest).toHaveBeenCalledWith('search', 'test-token', { - method: 'GET', - query: { - query: 'AI news', - num: 5, - }, + it('invokes bulk.createBulk correctly', async () => { + await Bulk.createBulk(mockCtx, { + name: 'job1', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: ['https://example.com/1'], + }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'POST', + body: 'https://example.com/1', + query: expect.objectContaining({ name: 'job1' }), }); }); - it('searchDql invokes client correctly and routes to Knowledge Graph base URL', async () => { - mockRequest.mockResolvedValueOnce({ data: [] }); - await Search.dql(mockCtx, { - query: 'name:"OpenAI"', - entityType: 'Organization', - queryType: 'query', - size: 10, + it('invokes crawl.startCrawl correctly', async () => { + await Crawl.startCrawl(mockCtx, { + name: 'crawl1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('crawl', 'test_token', { + method: 'POST', + query: expect.objectContaining({ name: 'crawl1' }), }); + }); - expect(mockRequest).toHaveBeenCalledWith('dql', 'test-token', { + it('invokes customApi.listCustomApis correctly', async () => { + await CustomApi.listCustomApis(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { method: 'GET', - useKgBase: true, - query: { - query: 'type:Organization name:"OpenAI"', - type: 'query', - size: 10, - }, }); }); }); -describe('Diffbot error handlers', () => { - it('recognizes rate limits and preserves retry timing', async () => { - const error = Object.assign(new Error('Too many requests'), { +describe('Diffbot Error Handlers', () => { + it('handles rate limit 429 errors and specifies retries', async () => { + const error = Object.assign(new Error('Rate limit exceeded'), { status: 429, - retryAfter: 12_000, + retryAfter: 1500, }); - expect(errorHandlers.RATE_LIMIT_ERROR.match(error)).toBe(true); - expect(await errorHandlers.RATE_LIMIT_ERROR.handler(error)).toEqual({ - maxRetries: 5, - headersRetryAfterMs: 12_000, - }); + const res = await errorHandlers.RATE_LIMIT_ERROR.handler(error); + expect(res.maxRetries).toBe(5); + expect(res.headersRetryAfterMs).toBe(1500); }); - it('does not retry authentication failures', async () => { - const error = Object.assign(new Error('Unauthorized'), { status: 401 }); - + it('handles 401 unauthorized errors with 0 retries', async () => { + const error = Object.assign(new Error('Invalid token'), { status: 401 }); expect(errorHandlers.AUTH_ERROR.match(error)).toBe(true); - expect(await errorHandlers.AUTH_ERROR.handler()).toEqual({ - maxRetries: 0, + const res = await errorHandlers.AUTH_ERROR.handler(error); + expect(res.maxRetries).toBe(0); + }); + + it('handles 500 server errors', async () => { + const error = Object.assign(new Error('Internal server error'), { + status: 500, }); + expect(errorHandlers.SERVER_ERROR.match(error)).toBe(true); + const res = await errorHandlers.SERVER_ERROR.handler(error); + expect(res.maxRetries).toBe(2); + }); +}); + +describe('Diffbot Plugin Instance', () => { + it('initializes diffbot plugin with default options', () => { + const instance = diffbot({ key: 'diffbot_test_key' }); + expect(instance.id).toBe('diffbot'); + expect(instance.schema).toBeDefined(); + expect(instance.endpoints).toBeDefined(); + expect(Object.keys(instance.endpoints ?? {}).length).toBe(8); }); }); diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index 07b28509f..bce9bef9c 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -13,47 +13,67 @@ export class DiffbotAPIError extends Error { } } -// Diffbot API v3 base URL (extract/search) +// Diffbot API v3 base URL (extract, crawl, bulk, custom, account) const DIFFBOT_API_BASE = 'https://api.diffbot.com/v3'; -// Diffbot Knowledge Graph base URL (DQL) +// Diffbot Knowledge Graph base URL (DQL, enhance, kg-bulk) const DIFFBOT_KG_BASE = 'https://kg.diffbot.com/kg/v3'; +export type DiffbotRequestOptions = { + method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; + body?: unknown; + query?: Record; + headers?: Record; + useKgBase?: boolean; + customBase?: string; + timeout?: number; +}; + /** * Make a request to the Diffbot API. * * Diffbot authenticates via `?token=` as a query parameter — * NOT via an Authorization header. The token is injected automatically here. * - * @param endpoint - The API endpoint path (e.g. 'analyze', 'dql') + * @param endpoint - The API endpoint path (e.g. 'article', 'dql', 'enhance') * @param token - The Diffbot API key * @param options - Request options including method, body, query params - * @param useKgBase - If true, routes request to the Knowledge Graph host (kg.diffbot.com) */ export async function makeDiffbotRequest( endpoint: string, token: string, - options: { - method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; - body?: Record; - query?: Record; - useKgBase?: boolean; - } = {}, + options: DiffbotRequestOptions = {}, ): Promise { - const { method = 'GET', body, query, useKgBase = false } = options; + if (!token?.trim()) { + throw new Error('Diffbot API token is required'); + } + + const { + method = 'GET', + body, + query, + headers, + useKgBase = false, + customBase, + timeout, + } = options; + + const baseUrl = + customBase ?? (useKgBase ? DIFFBOT_KG_BASE : DIFFBOT_API_BASE); const config: OpenAPIConfig = { - BASE: useKgBase ? DIFFBOT_KG_BASE : DIFFBOT_API_BASE, + BASE: baseUrl, VERSION: '3', WITH_CREDENTIALS: false, CREDENTIALS: 'omit', TOKEN: undefined, + TIMEOUT: timeout, HEADERS: { Accept: 'application/json', + ...headers, }, }; - // Diffbot auth: token is a query parameter, not a header const queryWithToken: Record = { ...query, @@ -64,10 +84,13 @@ export async function makeDiffbotRequest( method, url: endpoint, body: - method === 'POST' || method === 'PUT' || method === 'PATCH' + method === 'POST' || + method === 'PUT' || + method === 'PATCH' || + method === 'DELETE' ? body : undefined, - mediaType: 'application/json', + mediaType: typeof body === 'string' ? 'text/plain' : 'application/json', query: queryWithToken, }; diff --git a/packages/diffbot/endpoints/account.ts b/packages/diffbot/endpoints/account.ts new file mode 100644 index 000000000..598e3121d --- /dev/null +++ b/packages/diffbot/endpoints/account.ts @@ -0,0 +1,17 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const getAccount: DiffbotEndpoints['getAccount'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('account', ctx.key, { + method: 'GET', + }); + + await logEventFromContext(ctx, 'diffbot.account.getAccount', {}, 'completed'); + return response; +}; diff --git a/packages/diffbot/endpoints/bulk.ts b/packages/diffbot/endpoints/bulk.ts new file mode 100644 index 000000000..cecd8e513 --- /dev/null +++ b/packages/diffbot/endpoints/bulk.ts @@ -0,0 +1,113 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createBulk: DiffbotEndpoints['createBulk'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'POST', + body: input.urls.join('\n'), + query: { + name: input.name, + apiUrl: input.apiUrl, + notifyEmail: input.notifyEmail, + maxRounds: input.maxRounds, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.createBulk', + { name: input.name, count: input.urls.length }, + 'completed', + ); + return response; +}; + +export const startBulk: DiffbotEndpoints['startBulk'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + query: { + name: input.name, + apiUrl: input.apiUrl, + urls: input.urls, + notifyEmail: input.notifyEmail, + maxRounds: input.maxRounds, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.startBulk', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const stopBulkJob: DiffbotEndpoints['stopBulkJob'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + query: { + name: input.name, + pause: 1, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.stopBulkJob', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const getBulkData: DiffbotEndpoints['getBulkData'] = async ( + ctx, + input, +) => { + const format = input.format ?? 'json'; + const response = await makeDiffbotRequest< + Awaited> + >( + `bulk/download/${encodeURIComponent(ctx.key)}-${encodeURIComponent(input.name)}.${format}`, + ctx.key, + { + method: 'GET', + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.bulk.getBulkData', + { name: input.name, format }, + 'completed', + ); + return response; +}; + +export const listBulkJobs: DiffbotEndpoints['listBulkJobs'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + }); + + await logEventFromContext(ctx, 'diffbot.bulk.listBulkJobs', {}, 'completed'); + return response; +}; diff --git a/packages/diffbot/endpoints/crawl.ts b/packages/diffbot/endpoints/crawl.ts new file mode 100644 index 000000000..4ea87b5da --- /dev/null +++ b/packages/diffbot/endpoints/crawl.ts @@ -0,0 +1,84 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const startCrawl: DiffbotEndpoints['startCrawl'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('crawl', ctx.key, { + method: 'POST', + query: { + name: input.name, + seeds: input.seeds, + apiUrl: input.apiUrl, + maxHops: input.maxHops, + maxRounds: input.maxRounds, + maxTags: input.maxTags, + crawlSubdomains: input.crawlSubdomains, + notifyEmail: input.notifyEmail, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.crawl.startCrawl', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const manageCrawl: DiffbotEndpoints['manageCrawl'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('crawl', ctx.key, { + method: 'GET', + query: { + name: input.name, + pause: input.pause, + restart: input.restart, + delete: input.delete, + roundProxy: input.roundProxy, + maxRounds: input.maxRounds, + maxHops: input.maxHops, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.crawl.manageCrawl', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const getCrawlData: DiffbotEndpoints['getCrawlData'] = async ( + ctx, + input, +) => { + const format = input.format ?? 'json'; + const response = await makeDiffbotRequest< + Awaited> + >( + `crawl/download/${encodeURIComponent(ctx.key)}-${encodeURIComponent(input.name)}.${format}`, + ctx.key, + { + method: 'GET', + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.crawl.getCrawlData', + { name: input.name, format }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/custom-api.ts b/packages/diffbot/endpoints/custom-api.ts new file mode 100644 index 000000000..075fa8a0f --- /dev/null +++ b/packages/diffbot/endpoints/custom-api.ts @@ -0,0 +1,70 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createCustomApi: DiffbotEndpoints['createCustomApi'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'POST', + body: input.rules, + query: { + api: input.api, + url: input.url, + pattern: input.pattern, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.createCustomApi', + { api: input.api, url: input.url }, + 'completed', + ); + return response; +}; + +export const listCustomApis: DiffbotEndpoints['listCustomApis'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'GET', + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.listCustomApis', + {}, + 'completed', + ); + return response; +}; + +export const deleteCustomApi: DiffbotEndpoints['deleteCustomApi'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'DELETE', + query: { + api: input.api, + url: input.url, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.deleteCustomApi', + { api: input.api }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/enhance.ts b/packages/diffbot/endpoints/enhance.ts new file mode 100644 index 000000000..e9c4cff48 --- /dev/null +++ b/packages/diffbot/endpoints/enhance.ts @@ -0,0 +1,106 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const enhanceEntity: DiffbotEndpoints['enhanceEntity'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + name: input.name, + type: input.type, + email: input.email, + employer: input.employer, + url: input.url, + phone: input.phone, + location: input.location, + size: input.size, + refresh: input.refresh, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.enhanceEntity', + { name: input.name, type: input.type }, + 'completed', + ); + return response; +}; + +export const combineEntityProfiles: DiffbotEndpoints['combineEntityProfiles'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/combine', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + name: input.name, + type: input.type, + email: input.email, + employer: input.employer, + url: input.url, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.combineEntityProfiles', + { name: input.name, type: input.type }, + 'completed', + ); + return response; + }; + +export const resolveLostId: DiffbotEndpoints['resolveLostId'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('dql', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + query: `id:"${input.id}"`, + size: 1, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.resolveLostId', + { id: input.id }, + 'completed', + ); + return response; +}; + +export const getKgCoverageReportById: DiffbotEndpoints['getKgCoverageReportById'] = + async (ctx, input) => { + const endpoint = input.bulkjobId + ? `enhance/bulk/report/${encodeURIComponent(input.bulkjobId)}/${encodeURIComponent(input.reportId)}` + : 'report'; + + const response = await makeDiffbotRequest< + Awaited> + >(endpoint, ctx.key, { + method: 'GET', + useKgBase: true, + query: input.bulkjobId ? {} : { reportId: input.reportId }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.getKgCoverageReportById', + { reportId: input.reportId, bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; + }; diff --git a/packages/diffbot/endpoints/extract.ts b/packages/diffbot/endpoints/extract.ts index 3a13c3873..78e9fe105 100644 --- a/packages/diffbot/endpoints/extract.ts +++ b/packages/diffbot/endpoints/extract.ts @@ -1,88 +1,216 @@ import { logEventFromContext } from 'corsair/core'; -import type { DiffbotEndpoints } from '..'; import { makeDiffbotRequest } from '../client'; -import type { DiffbotEndpointOutputs } from './types'; +import type { DiffbotEndpoints } from '../index'; -/** - * Extract article content and metadata from any URL. - * Docs: https://docs.diffbot.com/reference/extract-article - */ -export const article: DiffbotEndpoints['extractArticle'] = async ( +export const getArticle: DiffbotEndpoints['getArticle'] = async ( ctx, input, ) => { - const { url, fields, timeout, paging, maxTags, naturalLanguage } = input; - const response = await makeDiffbotRequest< - DiffbotEndpointOutputs['extractArticle'] + Awaited> >('article', ctx.key, { method: 'GET', query: { - url, - fields, - timeout, - paging, - maxTags, - naturalLanguage, + url: input.url, + fields: input.fields, + timeout: input.timeout, + paging: input.paging, + maxTags: input.maxTags, + naturalLanguage: input.naturalLanguage, }, }); await logEventFromContext( ctx, - 'diffbot.extract.article', - { url }, + 'diffbot.extract.getArticle', + { url: input.url }, 'completed', ); return response; }; -/** - * Extract product data (price, availability, images, etc.) from any URL. - * Docs: https://docs.diffbot.com/reference/extract-product - */ -export const product: DiffbotEndpoints['extractProduct'] = async ( +export const getProduct: DiffbotEndpoints['getProduct'] = async ( ctx, input, ) => { - const { url, fields, timeout } = input; - const response = await makeDiffbotRequest< - DiffbotEndpointOutputs['extractProduct'] + Awaited> >('product', ctx.key, { method: 'GET', - query: { url, fields, timeout }, + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + discussion: input.discussion, + }, }); await logEventFromContext( ctx, - 'diffbot.extract.product', - { url }, + 'diffbot.extract.getProduct', + { url: input.url }, 'completed', ); return response; }; -/** - * Automatically detect the page type and extract its structured data. - * Docs: https://docs.diffbot.com/reference/extract-analyze - */ -export const analyze: DiffbotEndpoints['extractAnalyze'] = async ( +export const getAnalyze: DiffbotEndpoints['getAnalyze'] = async ( ctx, input, ) => { - const { url, fields, timeout, fallback, discussion } = input; - const response = await makeDiffbotRequest< - DiffbotEndpointOutputs['extractAnalyze'] + Awaited> >('analyze', ctx.key, { method: 'GET', - query: { url, fields, timeout, fallback, discussion }, + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + fallback: input.fallback, + discussion: input.discussion, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getAnalyze', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getImage: DiffbotEndpoints['getImage'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('image', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getImage', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getVideo: DiffbotEndpoints['getVideo'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('video', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getVideo', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getDiscussion: DiffbotEndpoints['getDiscussion'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('discussion', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + maxTags: input.maxTags, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getDiscussion', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getEvent: DiffbotEndpoints['getEvent'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('event', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getEvent', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const extractList: DiffbotEndpoints['extractList'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('list', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.extractList', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const extractJob: DiffbotEndpoints['extractJob'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('job', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, }); await logEventFromContext( ctx, - 'diffbot.extract.analyze', - { url }, + 'diffbot.extract.extractJob', + { url: input.url }, 'completed', ); return response; diff --git a/packages/diffbot/endpoints/index.ts b/packages/diffbot/endpoints/index.ts index cc62b16d8..8f1b23979 100644 --- a/packages/diffbot/endpoints/index.ts +++ b/packages/diffbot/endpoints/index.ts @@ -1,4 +1,8 @@ -import * as Extract from './extract'; -import * as Search from './search'; - -export { Extract, Search }; +export * as Account from './account'; +export * as Bulk from './bulk'; +export * as Crawl from './crawl'; +export * as CustomApi from './custom-api'; +export * as Enhance from './enhance'; +export * as Extract from './extract'; +export * as KgBulkEnhance from './kg-bulk-enhance'; +export * as Search from './search'; diff --git a/packages/diffbot/endpoints/kg-bulk-enhance.ts b/packages/diffbot/endpoints/kg-bulk-enhance.ts new file mode 100644 index 000000000..19a84e517 --- /dev/null +++ b/packages/diffbot/endpoints/kg-bulk-enhance.ts @@ -0,0 +1,172 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createKgBulkEnhance: DiffbotEndpoints['createKgBulkEnhance'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/bulk', ctx.key, { + method: 'POST', + useKgBase: true, + body: input.entities, + query: { + notifyEmail: input.notifyEmail, + name: input.name, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.createKgBulkEnhance', + { name: input.name, count: input.entities.length }, + 'completed', + ); + return response; + }; + +export const getBulkJobStatus: DiffbotEndpoints['getBulkJobStatus'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/status`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkJobStatus', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; +}; + +export const listBulkJobsStatusForToken: DiffbotEndpoints['listBulkJobsStatusForToken'] = + async (ctx, _input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/bulk', ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.listBulkJobsStatusForToken', + {}, + 'completed', + ); + return response; + }; + +export const getBulkResults: DiffbotEndpoints['getBulkResults'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}`, ctx.key, { + method: 'GET', + useKgBase: true, + query: { + format: input.format, + head: input.head, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkResults', + { bulkjobId: input.bulkjobId, format: input.format }, + 'completed', + ); + return response; +}; + +export const downloadBulkResults: DiffbotEndpoints['downloadBulkResults'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}`, ctx.key, { + method: 'POST', + useKgBase: true, + query: { + format: input.format, + filter: input.filter, + fields: input.fields, + head: input.head, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.downloadBulkResults', + { bulkjobId: input.bulkjobId, format: input.format }, + 'completed', + ); + return response; + }; + +export const getBulkSingleResult: DiffbotEndpoints['getBulkSingleResult'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >( + `enhance/bulk/${encodeURIComponent(input.bulkjobId)}/${input.jobIndex}`, + ctx.key, + { + method: 'GET', + useKgBase: true, + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkSingleResult', + { bulkjobId: input.bulkjobId, jobIndex: input.jobIndex }, + 'completed', + ); + return response; + }; + +export const stopKgBulkJobById: DiffbotEndpoints['stopKgBulkJobById'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/stop`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.stopKgBulkJobById', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; +}; + +export const deleteKgEnhanceBulkjob: DiffbotEndpoints['deleteKgEnhanceBulkjob'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/delete`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.deleteKgEnhanceBulkjob', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; + }; diff --git a/packages/diffbot/endpoints/search.ts b/packages/diffbot/endpoints/search.ts index 79fd70253..7dc1406b6 100644 --- a/packages/diffbot/endpoints/search.ts +++ b/packages/diffbot/endpoints/search.ts @@ -1,62 +1,55 @@ import { logEventFromContext } from 'corsair/core'; -import type { DiffbotEndpoints } from '..'; import { makeDiffbotRequest } from '../client'; -import type { DiffbotEndpointOutputs } from './types'; +import type { DiffbotEndpoints } from '../index'; -/** - * Search the web and return structured results with article metadata. - * Docs: https://docs.diffbot.com/reference/search-search - */ -export const web: DiffbotEndpoints['searchWeb'] = async (ctx, input) => { - const { query, col, num, start } = input; +export const search: DiffbotEndpoints['search'] = async (ctx, input) => { + const dqlQuery = input.entityType + ? `type:${input.entityType} ${input.query}` + : input.query; const response = await makeDiffbotRequest< - DiffbotEndpointOutputs['searchWeb'] - >('search', ctx.key, { + Awaited> + >('dql', ctx.key, { method: 'GET', - query: { query, col, num, start }, + useKgBase: true, + query: { + query: dqlQuery, + type: input.queryType, + size: input.size, + from: input.from, + col: input.col, + }, }); - await logEventFromContext(ctx, 'diffbot.search.web', { query }, 'completed'); + await logEventFromContext( + ctx, + 'diffbot.search.search', + { query: input.query, entityType: input.entityType }, + 'completed', + ); return response; }; -/** - * Query the Diffbot Knowledge Graph using DQL (Diffbot Query Language). - * Docs: https://docs.diffbot.com/reference/dqlget - * - * Uses the Knowledge Graph host: https://kg.diffbot.com/kg/v3/dql - * - * - `entityType`: optional DQL entity filter prepended to the query string (e.g. "Organization") - * - `queryType`: optional HTTP `type` parameter selecting the execution mode - * ("query" | "text" | "queryTextFallback" | "crawl"). Defaults to "query". - */ -export const dql: DiffbotEndpoints['searchDql'] = async (ctx, input) => { - const { query, entityType, queryType, size, from, col } = input; - - // Build the DQL query string with optional entity type prefix - const fullQuery = entityType ? `type:${entityType} ${query}` : query; - +export const searchCrawlData: DiffbotEndpoints['searchCrawlData'] = async ( + ctx, + input, +) => { const response = await makeDiffbotRequest< - DiffbotEndpointOutputs['searchDql'] - >('dql', ctx.key, { + Awaited> + >('search', ctx.key, { method: 'GET', - // Route to Knowledge Graph host (kg.diffbot.com/kg/v3) - useKgBase: true, query: { - query: fullQuery, - // HTTP `type` controls execution mode (query/text/crawl etc.) - type: queryType, - size, - from, - col, + col: input.col, + query: input.query, + num: input.num, + start: input.start, }, }); await logEventFromContext( ctx, - 'diffbot.search.dql', - { query: fullQuery }, + 'diffbot.search.searchCrawlData', + { col: input.col, query: input.query }, 'completed', ); return response; diff --git a/packages/diffbot/endpoints/types.ts b/packages/diffbot/endpoints/types.ts index 1de08a551..1088eaaad 100644 --- a/packages/diffbot/endpoints/types.ts +++ b/packages/diffbot/endpoints/types.ts @@ -4,7 +4,7 @@ import { z } from 'zod'; // Shared sub-schemas // --------------------------------------------------------------------------- -const DiffbotImageSchema = z +export const DiffbotImageItemSchema = z .object({ url: z.string().optional(), title: z.string().optional(), @@ -14,10 +14,13 @@ const DiffbotImageSchema = z naturalHeight: z.number().optional(), primary: z.boolean().optional(), xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), }) .passthrough(); -const DiffbotTagSchema = z +export const DiffbotTagSchema = z .object({ id: z.number().optional(), label: z.string(), @@ -30,7 +33,7 @@ const DiffbotTagSchema = z }) .passthrough(); -const DiffbotRequestMetaSchema = z +export const DiffbotRequestMetaSchema = z .object({ pageUrl: z.string().optional(), api: z.string().optional(), @@ -40,10 +43,33 @@ const DiffbotRequestMetaSchema = z .optional(); // --------------------------------------------------------------------------- -// Extract Article +// 1. Account // --------------------------------------------------------------------------- -export const ExtractArticleInputSchema = z.object({ +export const GetAccountInputSchema = z.object({}); +export type GetAccountInput = z.infer; + +export const GetAccountResponseSchema = z + .object({ + token: z.string().optional(), + name: z.string().optional(), + email: z.string().optional(), + plan: z.string().optional(), + planStart: z.string().optional(), + planCalls: z.number().optional(), + apiCalls: z.number().optional(), + status: z.string().optional(), + }) + .passthrough(); + +export type GetAccountResponse = z.infer; + +// --------------------------------------------------------------------------- +// 2. Extract APIs (9 operations) +// --------------------------------------------------------------------------- + +// 2.1 Get Article Data +export const GetArticleInputSchema = z.object({ url: z.string().describe('The URL of the article to extract'), fields: z .string() @@ -54,63 +80,60 @@ export const ExtractArticleInputSchema = z.object({ .optional() .describe('Timeout in milliseconds (default 30000)'), paging: z - .enum(['false']) + .enum(['false', 'true']) .optional() - .describe('Set to "false" to disable pagination following'), + .describe('Set to "false" to disable automatic pagination following'), maxTags: z.number().optional().describe('Maximum number of tags to return'), naturalLanguage: z.string().optional().describe('Language hint for NLP'), }); +export type GetArticleInput = z.infer; -export type ExtractArticleInput = z.infer; - -const ArticleObjectSchema = z +export const GetArticleResponseSchema = z .object({ - type: z.literal('article'), - title: z.string().optional(), - text: z.string().optional(), - html: z.string().optional(), - date: z.string().optional(), - estimatedDate: z.string().optional(), - author: z.string().optional(), - authorUrl: z.string().optional(), - siteName: z.string().optional(), - pageUrl: z.string().optional(), - resolvedPageUrl: z.string().optional(), - humanLanguage: z.string().optional(), - numPages: z.number().optional(), - nextPage: z.string().optional(), - nextPages: z.array(z.string()).optional(), - images: z.array(DiffbotImageSchema).optional(), - videos: z - .array(z.object({ url: z.string().optional() }).passthrough()) - .optional(), - tags: z.array(DiffbotTagSchema).optional(), - links: z.array(z.string()).optional(), - breadcrumb: z - .array( - z.object({ link: z.string().optional(), name: z.string().optional() }), - ) - .optional(), - publisherRegion: z.string().optional(), - publisherCountry: z.string().optional(), - sentiment: z.number().optional(), - }) - .passthrough(); - -export const ExtractArticleResponseSchema = z.object({ - request: DiffbotRequestMetaSchema, - objects: z.array(ArticleObjectSchema), -}); - -export type ExtractArticleResponse = z.infer< - typeof ExtractArticleResponseSchema ->; - -// --------------------------------------------------------------------------- -// Extract Product -// --------------------------------------------------------------------------- + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('article').optional(), + title: z.string().optional(), + text: z.string().optional(), + html: z.string().optional(), + date: z.string().optional(), + estimatedDate: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + siteName: z.string().optional(), + pageUrl: z.string().optional(), + resolvedPageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + numPages: z.number().optional(), + nextPage: z.string().optional(), + nextPages: z.array(z.string()).optional(), + images: z.array(DiffbotImageItemSchema).optional(), + videos: z.array(z.record(z.string(), z.unknown())).optional(), + tags: z.array(DiffbotTagSchema).optional(), + links: z.array(z.string()).optional(), + breadcrumb: z + .array( + z.object({ + link: z.string().optional(), + name: z.string().optional(), + }), + ) + .optional(), + publisherRegion: z.string().optional(), + publisherCountry: z.string().optional(), + sentiment: z.number().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetArticleResponse = z.infer; -export const ExtractProductInputSchema = z.object({ +// 2.2 Get Product Data +export const GetProductInputSchema = z.object({ url: z.string().describe('The URL of the product page to extract'), fields: z .string() @@ -120,67 +143,56 @@ export const ExtractProductInputSchema = z.object({ .number() .optional() .describe('Timeout in milliseconds (default 30000)'), + discussion: z + .enum(['false', 'true']) + .optional() + .describe('Set to "false" to disable review/discussion extraction'), }); +export type GetProductInput = z.infer; -export type ExtractProductInput = z.infer; - -const ProductOfferSchema = z - .object({ - price: z.string().optional(), - priceCurrency: z.string().optional(), - availability: z.boolean().optional(), - condition: z.string().optional(), - seller: z.string().optional(), - shippingAmount: z.string().optional(), - }) - .passthrough(); - -const ProductObjectSchema = z +export const GetProductResponseSchema = z .object({ - type: z.literal('product'), - title: z.string().optional(), - text: z.string().optional(), - brand: z.string().optional(), - offerPrice: z.string().optional(), - offerPriceDetails: z - .object({ - amount: z.number().optional(), - symbol: z.string().optional(), - text: z.string().optional(), - }) - .passthrough() - .optional(), - regularPrice: z.string().optional(), - saveAmount: z.string().optional(), - shippingAmount: z.string().optional(), - availability: z.boolean().optional(), - sku: z.string().optional(), - mpn: z.string().optional(), - upc: z.string().optional(), - isbn: z.string().optional(), - images: z.array(DiffbotImageSchema).optional(), - offers: z.array(ProductOfferSchema).optional(), - colors: z.array(z.string()).optional(), - pageUrl: z.string().optional(), - humanLanguage: z.string().optional(), - tags: z.array(DiffbotTagSchema).optional(), + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('product').optional(), + title: z.string().optional(), + text: z.string().optional(), + brand: z.string().optional(), + offerPrice: z.string().optional(), + offerPriceDetails: z + .object({ + amount: z.number().optional(), + symbol: z.string().optional(), + text: z.string().optional(), + }) + .passthrough() + .optional(), + regularPrice: z.string().optional(), + saveAmount: z.string().optional(), + shippingAmount: z.string().optional(), + availability: z.boolean().optional(), + sku: z.string().optional(), + mpn: z.string().optional(), + upc: z.string().optional(), + isbn: z.string().optional(), + images: z.array(DiffbotImageItemSchema).optional(), + offers: z.array(z.record(z.string(), z.unknown())).optional(), + colors: z.array(z.string()).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + tags: z.array(DiffbotTagSchema).optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), }) .passthrough(); +export type GetProductResponse = z.infer; -export const ExtractProductResponseSchema = z.object({ - request: DiffbotRequestMetaSchema, - objects: z.array(ProductObjectSchema), -}); - -export type ExtractProductResponse = z.infer< - typeof ExtractProductResponseSchema ->; - -// --------------------------------------------------------------------------- -// Analyze (auto-detect page type) -// --------------------------------------------------------------------------- - -export const AnalyzeInputSchema = z.object({ +// 2.3 Analyze (Auto-detect page type) +export const GetAnalyzeInputSchema = z.object({ url: z .string() .describe('The URL to analyze — Diffbot auto-detects the page type'), @@ -196,14 +208,13 @@ export const AnalyzeInputSchema = z.object({ 'API to fall back to if page type cannot be detected (e.g. "article")', ), discussion: z - .enum(['false']) + .enum(['false', 'true']) .optional() .describe('Set to "false" to disable comment extraction'), }); +export type GetAnalyzeInput = z.infer; -export type AnalyzeInput = z.infer; - -export const AnalyzeResponseSchema = z +export const GetAnalyzeResponseSchema = z .object({ request: DiffbotRequestMetaSchema, type: z @@ -215,145 +226,1037 @@ export const AnalyzeResponseSchema = z objects: z.array(z.record(z.string(), z.unknown())).optional(), }) .passthrough(); +export type GetAnalyzeResponse = z.infer; -export type AnalyzeResponse = z.infer; +// 2.4 Get Image Data +export const GetImageInputSchema = z.object({ + url: z.string().describe('The URL of the page or image to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetImageInput = z.infer; + +export const GetImageResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('image').optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetImageResponse = z.infer; + +// 2.5 Get Video Data +export const GetVideoInputSchema = z.object({ + url: z.string().describe('The URL of the video page to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetVideoInput = z.infer; + +export const GetVideoResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('video').optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + duration: z.number().optional(), + viewCount: z.number().optional(), + uploadDate: z.string().optional(), + author: z.string().optional(), + embedUrl: z.string().optional(), + html: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetVideoResponse = z.infer; + +// 2.6 Get Discussion Thread +export const GetDiscussionInputSchema = z.object({ + url: z.string().describe('The URL of the discussion / forum / comment page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), + maxTags: z.number().optional().describe('Max tags to return'), +}); +export type GetDiscussionInput = z.infer; + +export const GetDiscussionResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('discussion').optional(), + title: z.string().optional(), + text: z.string().optional(), + numPosts: z.number().optional(), + numParticipants: z.number().optional(), + participants: z.array(z.string()).optional(), + rssUrl: z.string().optional(), + posts: z.array(z.record(z.string(), z.unknown())).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetDiscussionResponse = z.infer; + +// 2.7 Get Event Data +export const GetEventInputSchema = z.object({ + url: z.string().describe('The URL of the event page to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetEventInput = z.infer; + +export const GetEventResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('event').optional(), + title: z.string().optional(), + description: z.string().optional(), + startDate: z.string().optional(), + endDate: z.string().optional(), + location: z.string().optional(), + venue: z.record(z.string(), z.unknown()).optional(), + organizer: z.string().optional(), + ticketUrl: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetEventResponse = z.infer; + +// 2.8 Extract List +export const ExtractListInputSchema = z.object({ + url: z.string().describe('The URL of the list / directory / index page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type ExtractListInput = z.infer; + +export const ExtractListResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('list').optional(), + title: z.string().optional(), + numItems: z.number().optional(), + items: z.array(z.record(z.string(), z.unknown())).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type ExtractListResponse = z.infer; + +// 2.9 Extract Job +export const ExtractJobInputSchema = z.object({ + url: z.string().describe('The URL of the job posting page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type ExtractJobInput = z.infer; + +export const ExtractJobResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('job').optional(), + title: z.string().optional(), + description: z.string().optional(), + company: z.record(z.string(), z.unknown()).optional(), + locations: z.array(z.string()).optional(), + employmentType: z.string().optional(), + compensation: z.record(z.string(), z.unknown()).optional(), + requirements: z.array(z.string()).optional(), + skills: z.array(z.string()).optional(), + postedDate: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type ExtractJobResponse = z.infer; // --------------------------------------------------------------------------- -// Web Search +// 3. Search / DQL APIs (2 operations) // --------------------------------------------------------------------------- -export const WebSearchInputSchema = z.object({ - query: z.string().describe('Full-text search query'), - col: z +// 3.1 Diffbot Knowledge Graph Search (DIFFBOT_SEARCH) +export const SearchInputSchema = z.object({ + query: z .string() + .describe('DQL query string (e.g. "type:Organization name:\\"OpenAI\\"")'), + entityType: z + .string() + .optional() + .describe('Entity type filter prepended to query (e.g. "Organization")'), + queryType: z + .enum(['query', 'text', 'queryTextFallback', 'crawl']) .optional() - .describe('Diffbot crawl collection to search within'), + .describe('Execution mode for the DQL request'), + size: z.number().optional().describe('Number of results to return'), + from: z.number().optional().describe('Zero-indexed offset for pagination'), + col: z.string().optional().describe('Crawl collection name to query'), +}); +export type SearchInput = z.infer; + +export const SearchResponseSchema = z + .object({ + version: z.number().optional(), + hits: z.number().optional(), + results: z.number().optional(), + kgversion: z.string().optional(), + diffbot_type: z.string().optional(), + facet: z.record(z.string(), z.unknown()).optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + cursor: z.string().optional(), + }) + .passthrough(); +export type SearchResponse = z.infer; + +// 3.2 Search Crawl Job Data (DIFFBOT_SEARCH_CRAWL_DATA) +export const SearchCrawlDataInputSchema = z.object({ + col: z.string().describe('The name of the crawl job collection to search'), + query: z.string().describe('Search query string or DQL filter'), num: z .number() .min(1) .max(25) .optional() - .describe('Number of results to return (max 25, default 20)'), + .describe('Number of results to return (max 25)'), start: z.number().optional().describe('Zero-indexed offset for pagination'), }); +export type SearchCrawlDataInput = z.infer; + +export const SearchCrawlDataResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + results: z.array(z.record(z.string(), z.unknown())).optional(), + numResults: z.number().optional(), + hits: z.number().optional(), + }) + .passthrough(); +export type SearchCrawlDataResponse = z.infer< + typeof SearchCrawlDataResponseSchema +>; -export type WebSearchInput = z.infer; +// --------------------------------------------------------------------------- +// 4. Enhance APIs (4 operations) +// --------------------------------------------------------------------------- -const WebSearchResultSchema = z +// 4.1 Enhance Entity with Knowledge Graph (DIFFBOT_ENHANCE_ENTITY) +export const EnhanceEntityInputSchema = z.object({ + name: z.string().optional().describe('Entity name (person or organization)'), + type: z + .string() + .optional() + .describe('Entity type filter: "Organization" or "Person"'), + email: z.string().optional().describe('Email address of the entity'), + employer: z + .string() + .optional() + .describe('Current employer of a Person entity'), + url: z + .string() + .optional() + .describe('Homepage or profile URL (e.g. LinkedIn / Website)'), + phone: z.string().optional().describe('Phone number'), + location: z.string().optional().describe('Location or address'), + size: z + .number() + .optional() + .describe('Number of matching entity records to return'), + refresh: z + .boolean() + .optional() + .describe('Force refresh data from live web sources'), +}); +export type EnhanceEntityInput = z.infer; + +export const EnhanceEntityResponseSchema = z .object({ - title: z.string().optional(), - pageUrl: z.string().optional(), - text: z.string().optional(), - date: z.string().optional(), - author: z.string().optional(), - siteName: z.string().optional(), - humanLanguage: z.string().optional(), - tags: z.array(DiffbotTagSchema).optional(), - images: z.array(DiffbotImageSchema).optional(), + version: z.number().optional(), + hits: z.number().optional(), + kgversion: z.string().optional(), + request_ctx: z.record(z.string(), z.unknown()).optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + errors: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type EnhanceEntityResponse = z.infer; + +// 4.2 Combine Entity Profiles (DIFFBOT_COMBINE_ENTITY_PROFILES) +export const CombineEntityProfilesInputSchema = z.object({ + name: z.string().optional().describe('Person name'), + type: z.string().optional().describe('Entity type (defaults to Person)'), + email: z.string().optional().describe('Email address'), + employer: z.string().optional().describe('Employer name or organization'), + url: z.string().optional().describe('Profile URL or organization homepage'), +}); +export type CombineEntityProfilesInput = z.infer< + typeof CombineEntityProfilesInputSchema +>; + +export const CombineEntityProfilesResponseSchema = z + .object({ + version: z.number().optional(), + hits: z.number().optional(), + kgversion: z.string().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + errors: z.array(z.record(z.string(), z.unknown())).optional(), }) .passthrough(); +export type CombineEntityProfilesResponse = z.infer< + typeof CombineEntityProfilesResponseSchema +>; + +// 4.3 Resolve Lost ID (DIFFBOT_RESOLVE_LOST_ID) +export const ResolveLostIdInputSchema = z.object({ + id: z.string().describe('The lost or non-canonical identifier to resolve'), +}); +export type ResolveLostIdInput = z.infer; -export const WebSearchResponseSchema = z +export const ResolveLostIdResponseSchema = z .object({ - request: DiffbotRequestMetaSchema, - results: z.array(WebSearchResultSchema).optional(), - numResults: z.number().optional(), + id: z.string().optional(), + canonicalId: z.string().optional(), + diffbotUri: z.string().optional(), + name: z.string().optional(), + type: z.string().optional(), hits: z.number().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), }) .passthrough(); +export type ResolveLostIdResponse = z.infer; -export type WebSearchResponse = z.infer; +// 4.4 Get KG Coverage Report by ID (DIFFBOT_GET_KG_COVERAGE_REPORT_BY_ID) +export const GetKgCoverageReportByIdInputSchema = z.object({ + reportId: z + .string() + .describe('Coverage report ID generated from DQL query or bulk job'), + bulkjobId: z + .string() + .optional() + .describe('Optional bulkjob ID associated with the report'), +}); +export type GetKgCoverageReportByIdInput = z.infer< + typeof GetKgCoverageReportByIdInputSchema +>; + +export const GetKgCoverageReportByIdResponseSchema = z + .object({ + reportId: z.string().optional(), + status: z.string().optional(), + coverage: z.record(z.string(), z.unknown()).optional(), + data: z.unknown().optional(), + csv: z.string().optional(), + }) + .passthrough(); +export type GetKgCoverageReportByIdResponse = z.infer< + typeof GetKgCoverageReportByIdResponseSchema +>; // --------------------------------------------------------------------------- -// DQL (Knowledge Graph Search) +// 5. KG Bulk Enhance APIs (8 operations) // --------------------------------------------------------------------------- -export const DqlSearchInputSchema = z +// 5.1 Create Bulk Enhance Job (DIFFBOT_CREATE_KG_BULK_ENHANCE) +export const CreateKgBulkEnhanceInputSchema = z.object({ + entities: z + .array( + z + .object({ + name: z.string().optional(), + type: z.string().optional(), + email: z.string().optional(), + employer: z.string().optional(), + url: z.string().optional(), + phone: z.string().optional(), + location: z.string().optional(), + }) + .passthrough(), + ) + .describe('Array of entity objects to enhance'), + notifyEmail: z + .string() + .optional() + .describe('Email address to notify upon job completion'), + name: z.string().optional().describe('Custom name for the bulk job'), +}); +export type CreateKgBulkEnhanceInput = z.infer< + typeof CreateKgBulkEnhanceInputSchema +>; + +export const CreateKgBulkEnhanceResponseSchema = z .object({ - query: z - .string() - .describe( - "DQL query string (e.g. 'name:\"OpenAI\"'). Do NOT include a 'type:' prefix here — use entityType instead.", - ), - entityType: z - .string() - .optional() - .describe( - 'Entity type filter prepended to the DQL query (e.g. "Organization", "Person", "Article")', - ), - queryType: z - .enum(['query', 'text', 'queryTextFallback', 'crawl']) - .optional() - .describe( - 'Execution mode for the DQL request. Use "crawl" with col to search crawl collections.', - ), - size: z - .number() - .optional() - .describe( - 'Number of results to return (default 5, max 100 or 1000 for articles)', - ), - from: z.number().optional().describe('Zero-indexed offset for pagination'), - col: z - .string() - .optional() - .describe('Crawl collection name — only valid when queryType is "crawl"'), + bulkjobId: z.string().optional(), + job_id: z.string().optional(), + status: z.string().optional(), + total: z.number().optional(), + message: z.string().optional(), }) - .superRefine((input, ctx) => { - if (input.col !== undefined && input.queryType !== 'crawl') { - ctx.addIssue({ - code: 'custom', - path: ['col'], - message: 'col is only valid when queryType is "crawl"', - }); - } - }); + .passthrough(); +export type CreateKgBulkEnhanceResponse = z.infer< + typeof CreateKgBulkEnhanceResponseSchema +>; -export type DqlSearchInput = z.infer; +// 5.2 Get Bulk Job Status (DIFFBOT_GET_BULK_JOB_STATUS) +export const GetBulkJobStatusInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to check'), +}); +export type GetBulkJobStatusInput = z.infer; -export const DqlSearchResponseSchema = z +export const GetBulkJobStatusResponseSchema = z .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + jobStatus: z.record(z.string(), z.unknown()).optional(), + total: z.number().optional(), + completed: z.number().optional(), + failed: z.number().optional(), + progress: z.number().optional(), + }) + .passthrough(); +export type GetBulkJobStatusResponse = z.infer< + typeof GetBulkJobStatusResponseSchema +>; + +// 5.3 List Bulk Jobs Status For Token (DIFFBOT_LIST_BULK_JOBS_STATUS_FOR_TOKEN) +export const ListBulkJobsStatusForTokenInputSchema = z.object({}); +export type ListBulkJobsStatusForTokenInput = z.infer< + typeof ListBulkJobsStatusForTokenInputSchema +>; + +export const ListBulkJobsStatusForTokenResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), + bulkjobs: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ListBulkJobsStatusForTokenResponse = z.infer< + typeof ListBulkJobsStatusForTokenResponseSchema +>; + +// 5.4 Get Bulk Job Results (DIFFBOT_GET_BULK_RESULTS) +export const GetBulkResultsInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to download'), + format: z + .enum(['json', 'jsonl', 'csv', 'xls', 'xlsx']) + .optional() + .describe('Download output format (default jsonl)'), + head: z + .number() + .optional() + .describe('Preview only the first N results from the job'), +}); +export type GetBulkResultsInput = z.infer; + +export const GetBulkResultsResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), data: z.array(z.record(z.string(), z.unknown())).optional(), - hits: z.number().optional(), - cursor: z.string().optional(), - facets: z.record(z.string(), z.unknown()).optional(), + raw: z.string().optional(), + }) + .passthrough(); +export type GetBulkResultsResponse = z.infer< + typeof GetBulkResultsResponseSchema +>; + +// 5.5 Download Bulk Job Results (DIFFBOT_DOWNLOAD_BULK_RESULTS) +export const DownloadBulkResultsInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to download'), + format: z + .enum(['json', 'jsonl', 'csv', 'xls', 'xlsx']) + .optional() + .describe('Export format'), + filter: z + .string() + .optional() + .describe('DQL filter criteria to apply to the output'), + fields: z + .string() + .optional() + .describe('Comma-separated list of fields to include'), + head: z.number().optional().describe('Number of records to export'), +}); +export type DownloadBulkResultsInput = z.infer< + typeof DownloadBulkResultsInputSchema +>; + +export const DownloadBulkResultsResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + raw: z.string().optional(), + }) + .passthrough(); +export type DownloadBulkResultsResponse = z.infer< + typeof DownloadBulkResultsResponseSchema +>; + +// 5.6 Get Bulk Single Result (DIFFBOT_GET_BULK_SINGLE_RESULT) +export const GetBulkSingleResultInputSchema = z.object({ + bulkjobId: z.string().describe('The bulk enhance job ID'), + jobIndex: z + .number() + .describe('Zero-indexed position of the entity record within the bulk job'), +}); +export type GetBulkSingleResultInput = z.infer< + typeof GetBulkSingleResultInputSchema +>; + +export const GetBulkSingleResultResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + jobIndex: z.number().optional(), + data: z.record(z.string(), z.unknown()).optional(), + entity: z.record(z.string(), z.unknown()).optional(), + }) + .passthrough(); +export type GetBulkSingleResultResponse = z.infer< + typeof GetBulkSingleResultResponseSchema +>; + +// 5.7 Stop KG Bulk Job By ID (DIFFBOT_STOP_KG_BULK_JOB_BY_ID) +export const StopKgBulkJobByIdInputSchema = z.object({ + bulkjobId: z + .string() + .describe('The ID of the bulk enhance job to pause/stop'), +}); +export type StopKgBulkJobByIdInput = z.infer< + typeof StopKgBulkJobByIdInputSchema +>; + +export const StopKgBulkJobByIdResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type StopKgBulkJobByIdResponse = z.infer< + typeof StopKgBulkJobByIdResponseSchema +>; + +// 5.8 Delete KG Enhance Bulkjob (DIFFBOT_DELETE_KG_ENHANCE_BULKJOB) +export const DeleteKgEnhanceBulkjobInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to delete'), +}); +export type DeleteKgEnhanceBulkjobInput = z.infer< + typeof DeleteKgEnhanceBulkjobInputSchema +>; + +export const DeleteKgEnhanceBulkjobResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type DeleteKgEnhanceBulkjobResponse = z.infer< + typeof DeleteKgEnhanceBulkjobResponseSchema +>; + +// --------------------------------------------------------------------------- +// 6. Bulk Extract APIs (5 operations) +// --------------------------------------------------------------------------- + +// 6.1 Create Bulk Extract Job (DIFFBOT_CREATE_BULK) +export const CreateBulkInputSchema = z.object({ + name: z.string().describe('Name of the bulk job (unique per token)'), + apiUrl: z + .string() + .describe( + 'Full Diffbot Extract API URL (e.g. "https://api.diffbot.com/v3/article")', + ), + urls: z + .array(z.string()) + .describe('Array of URLs to process with the Extract API'), + notifyEmail: z + .string() + .optional() + .describe('Email to notify when processing is completed'), + maxRounds: z.number().optional().describe('Max rounds of URL processing'), +}); +export type CreateBulkInput = z.infer; + +export const CreateBulkResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type CreateBulkResponse = z.infer; + +// 6.2 Start Bulk Job (DIFFBOT_START_BULK) +export const StartBulkInputSchema = z.object({ + name: z.string().describe('Unique name for the bulk extract job'), + apiUrl: z.string().describe('Full Diffbot Extract API URL'), + urls: z + .string() + .describe('Comma-separated or space-separated list of URLs to process'), + notifyEmail: z.string().optional().describe('Notification email address'), + maxRounds: z.number().optional().describe('Max rounds of URL processing'), +}); +export type StartBulkInput = z.infer; + +export const StartBulkResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StartBulkResponse = z.infer; + +// 6.3 Stop Bulk Job (DIFFBOT_STOP_BULK_JOB) +export const StopBulkJobInputSchema = z.object({ + name: z.string().describe('The name of the bulk extract job to pause/stop'), +}); +export type StopBulkJobInput = z.infer; + +export const StopBulkJobResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StopBulkJobResponse = z.infer; + +// 6.4 Get Bulk Job Data (DIFFBOT_GET_BULK_DATA) +export const GetBulkDataInputSchema = z.object({ + name: z.string().describe('The name of the completed bulk job to download'), + format: z + .enum(['json', 'csv']) + .optional() + .describe('Download format (default json)'), +}); +export type GetBulkDataInput = z.infer; + +export const GetBulkDataResponseSchema = z + .object({ + name: z.string().optional(), + data: z.unknown().optional(), + }) + .passthrough(); +export type GetBulkDataResponse = z.infer; + +// 6.5 List Bulk Jobs (DIFFBOT_LIST_BULK_JOBS) +export const ListBulkJobsInputSchema = z.object({}); +export type ListBulkJobsInput = z.infer; + +export const ListBulkJobsResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), }) .passthrough(); +export type ListBulkJobsResponse = z.infer; -export type DqlSearchResponse = z.infer; +// --------------------------------------------------------------------------- +// 7. Crawl APIs (3 operations) +// --------------------------------------------------------------------------- + +// 7.1 Start Crawl Job (DIFFBOT_START_CRAWL) +export const StartCrawlInputSchema = z.object({ + name: z.string().describe('Unique name for the crawl job'), + seeds: z + .string() + .describe('Space-separated seed URL(s) from which the crawl begins'), + apiUrl: z + .string() + .describe( + 'Full Diffbot Extract API URL used to process pages (e.g. "https://api.diffbot.com/v3/article")', + ), + maxHops: z + .number() + .optional() + .describe( + 'Max depth of links to crawl from seeds (default -1 for no limit)', + ), + maxRounds: z + .number() + .optional() + .describe('Max rounds of repeat crawling for recurring crawls'), + maxTags: z.number().optional().describe('Max tags to extract per page'), + crawlSubdomains: z + .number() + .optional() + .describe('Set to 1 to crawl subdomains of seeds'), + notifyEmail: z + .string() + .optional() + .describe('Email notification upon crawl completion'), +}); +export type StartCrawlInput = z.infer; + +export const StartCrawlResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StartCrawlResponse = z.infer; + +// 7.2 Manage Crawl Job (DIFFBOT_MANAGE_CRAWL) +export const ManageCrawlInputSchema = z.object({ + name: z + .string() + .optional() + .describe('The name of the crawl job to inspect or modify'), + pause: z + .number() + .optional() + .describe('Set to 1 to pause an active crawl job, 0 to resume'), + restart: z + .number() + .optional() + .describe('Set to 1 to restart a completed/paused crawl job'), + delete: z + .number() + .optional() + .describe('Set to 1 to delete a crawl job and its data'), + roundProxy: z + .number() + .optional() + .describe('Set to 1 to rotate proxy IP on each round'), + maxRounds: z.number().optional().describe('Update max rounds'), + maxHops: z.number().optional().describe('Update max hops'), +}); +export type ManageCrawlInput = z.infer; + +export const ManageCrawlResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type ManageCrawlResponse = z.infer; + +// 7.3 Get Crawl Data (DIFFBOT_GET_CRAWL_DATA) +export const GetCrawlDataInputSchema = z.object({ + name: z.string().describe('The name of the completed crawl job to download'), + format: z + .enum(['json', 'csv']) + .optional() + .describe('Download format (default json)'), +}); +export type GetCrawlDataInput = z.infer; + +export const GetCrawlDataResponseSchema = z + .object({ + name: z.string().optional(), + data: z.unknown().optional(), + }) + .passthrough(); +export type GetCrawlDataResponse = z.infer; + +// --------------------------------------------------------------------------- +// 8. Custom API (3 operations) +// --------------------------------------------------------------------------- + +// 8.1 Create or Update Custom API (DIFFBOT_CREATE_CUSTOM_API) +export const CreateCustomApiInputSchema = z.object({ + api: z + .string() + .describe('Name of the custom API (e.g. "myCustomArticleApi")'), + url: z.string().describe('Sample URL that this custom API applies to'), + pattern: z + .string() + .optional() + .describe('URL regex pattern to match pages for this custom API'), + rules: z + .record(z.string(), z.unknown()) + .optional() + .describe('Extraction rules and CSS selector definitions'), +}); +export type CreateCustomApiInput = z.infer; + +export const CreateCustomApiResponseSchema = z + .object({ + response: z.string().optional(), + api: z.string().optional(), + url: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type CreateCustomApiResponse = z.infer< + typeof CreateCustomApiResponseSchema +>; + +// 8.2 List Custom APIs (DIFFBOT_LIST_CUSTOM_APIS) +export const ListCustomApisInputSchema = z.object({}); +export type ListCustomApisInput = z.infer; + +export const ListCustomApisResponseSchema = z + .object({ + customApis: z.array(z.record(z.string(), z.unknown())).optional(), + apis: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ListCustomApisResponse = z.infer< + typeof ListCustomApisResponseSchema +>; + +// 8.3 Delete Custom API (DIFFBOT_DELETE_CUSTOM_API) +export const DeleteCustomApiInputSchema = z.object({ + api: z.string().describe('Name of the custom API to delete'), + url: z + .string() + .optional() + .describe('URL pattern or test URL of the custom API'), +}); +export type DeleteCustomApiInput = z.infer; + +export const DeleteCustomApiResponseSchema = z + .object({ + response: z.string().optional(), + api: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type DeleteCustomApiResponse = z.infer< + typeof DeleteCustomApiResponseSchema +>; // --------------------------------------------------------------------------- -// Aggregated type maps (keyed by camelCase endpoint name) +// Aggregated type maps (35 operations keyed by camelCase endpoint name) // --------------------------------------------------------------------------- export type DiffbotEndpointInputs = { - extractArticle: ExtractArticleInput; - extractProduct: ExtractProductInput; - extractAnalyze: AnalyzeInput; - searchWeb: WebSearchInput; - searchDql: DqlSearchInput; + // Account + getAccount: GetAccountInput; + + // Extract + getArticle: GetArticleInput; + getProduct: GetProductInput; + getAnalyze: GetAnalyzeInput; + getImage: GetImageInput; + getVideo: GetVideoInput; + getDiscussion: GetDiscussionInput; + getEvent: GetEventInput; + extractList: ExtractListInput; + extractJob: ExtractJobInput; + + // Search + search: SearchInput; + searchCrawlData: SearchCrawlDataInput; + + // Enhance + enhanceEntity: EnhanceEntityInput; + combineEntityProfiles: CombineEntityProfilesInput; + resolveLostId: ResolveLostIdInput; + getKgCoverageReportById: GetKgCoverageReportByIdInput; + + // KG Bulk Enhance + createKgBulkEnhance: CreateKgBulkEnhanceInput; + getBulkJobStatus: GetBulkJobStatusInput; + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenInput; + getBulkResults: GetBulkResultsInput; + downloadBulkResults: DownloadBulkResultsInput; + getBulkSingleResult: GetBulkSingleResultInput; + stopKgBulkJobById: StopKgBulkJobByIdInput; + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobInput; + + // Bulk Extract + createBulk: CreateBulkInput; + startBulk: StartBulkInput; + stopBulkJob: StopBulkJobInput; + getBulkData: GetBulkDataInput; + listBulkJobs: ListBulkJobsInput; + + // Crawl + startCrawl: StartCrawlInput; + manageCrawl: ManageCrawlInput; + getCrawlData: GetCrawlDataInput; + + // Custom API + createCustomApi: CreateCustomApiInput; + listCustomApis: ListCustomApisInput; + deleteCustomApi: DeleteCustomApiInput; }; export type DiffbotEndpointOutputs = { - extractArticle: ExtractArticleResponse; - extractProduct: ExtractProductResponse; - extractAnalyze: AnalyzeResponse; - searchWeb: WebSearchResponse; - searchDql: DqlSearchResponse; + // Account + getAccount: GetAccountResponse; + + // Extract + getArticle: GetArticleResponse; + getProduct: GetProductResponse; + getAnalyze: GetAnalyzeResponse; + getImage: GetImageResponse; + getVideo: GetVideoResponse; + getDiscussion: GetDiscussionResponse; + getEvent: GetEventResponse; + extractList: ExtractListResponse; + extractJob: ExtractJobResponse; + + // Search + search: SearchResponse; + searchCrawlData: SearchCrawlDataResponse; + + // Enhance + enhanceEntity: EnhanceEntityResponse; + combineEntityProfiles: CombineEntityProfilesResponse; + resolveLostId: ResolveLostIdResponse; + getKgCoverageReportById: GetKgCoverageReportByIdResponse; + + // KG Bulk Enhance + createKgBulkEnhance: CreateKgBulkEnhanceResponse; + getBulkJobStatus: GetBulkJobStatusResponse; + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenResponse; + getBulkResults: GetBulkResultsResponse; + downloadBulkResults: DownloadBulkResultsResponse; + getBulkSingleResult: GetBulkSingleResultResponse; + stopKgBulkJobById: StopKgBulkJobByIdResponse; + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobResponse; + + // Bulk Extract + createBulk: CreateBulkResponse; + startBulk: StartBulkResponse; + stopBulkJob: StopBulkJobResponse; + getBulkData: GetBulkDataResponse; + listBulkJobs: ListBulkJobsResponse; + + // Crawl + startCrawl: StartCrawlResponse; + manageCrawl: ManageCrawlResponse; + getCrawlData: GetCrawlDataResponse; + + // Custom API + createCustomApi: CreateCustomApiResponse; + listCustomApis: ListCustomApisResponse; + deleteCustomApi: DeleteCustomApiResponse; }; export const DiffbotEndpointInputSchemas = { - extractArticle: ExtractArticleInputSchema, - extractProduct: ExtractProductInputSchema, - extractAnalyze: AnalyzeInputSchema, - searchWeb: WebSearchInputSchema, - searchDql: DqlSearchInputSchema, + getAccount: GetAccountInputSchema, + getArticle: GetArticleInputSchema, + getProduct: GetProductInputSchema, + getAnalyze: GetAnalyzeInputSchema, + getImage: GetImageInputSchema, + getVideo: GetVideoInputSchema, + getDiscussion: GetDiscussionInputSchema, + getEvent: GetEventInputSchema, + extractList: ExtractListInputSchema, + extractJob: ExtractJobInputSchema, + search: SearchInputSchema, + searchCrawlData: SearchCrawlDataInputSchema, + enhanceEntity: EnhanceEntityInputSchema, + combineEntityProfiles: CombineEntityProfilesInputSchema, + resolveLostId: ResolveLostIdInputSchema, + getKgCoverageReportById: GetKgCoverageReportByIdInputSchema, + createKgBulkEnhance: CreateKgBulkEnhanceInputSchema, + getBulkJobStatus: GetBulkJobStatusInputSchema, + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenInputSchema, + getBulkResults: GetBulkResultsInputSchema, + downloadBulkResults: DownloadBulkResultsInputSchema, + getBulkSingleResult: GetBulkSingleResultInputSchema, + stopKgBulkJobById: StopKgBulkJobByIdInputSchema, + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobInputSchema, + createBulk: CreateBulkInputSchema, + startBulk: StartBulkInputSchema, + stopBulkJob: StopBulkJobInputSchema, + getBulkData: GetBulkDataInputSchema, + listBulkJobs: ListBulkJobsInputSchema, + startCrawl: StartCrawlInputSchema, + manageCrawl: ManageCrawlInputSchema, + getCrawlData: GetCrawlDataInputSchema, + createCustomApi: CreateCustomApiInputSchema, + listCustomApis: ListCustomApisInputSchema, + deleteCustomApi: DeleteCustomApiInputSchema, } as const; export const DiffbotEndpointOutputSchemas = { - extractArticle: ExtractArticleResponseSchema, - extractProduct: ExtractProductResponseSchema, - extractAnalyze: AnalyzeResponseSchema, - searchWeb: WebSearchResponseSchema, - searchDql: DqlSearchResponseSchema, + getAccount: GetAccountResponseSchema, + getArticle: GetArticleResponseSchema, + getProduct: GetProductResponseSchema, + getAnalyze: GetAnalyzeResponseSchema, + getImage: GetImageResponseSchema, + getVideo: GetVideoResponseSchema, + getDiscussion: GetDiscussionResponseSchema, + getEvent: GetEventResponseSchema, + extractList: ExtractListResponseSchema, + extractJob: ExtractJobResponseSchema, + search: SearchResponseSchema, + searchCrawlData: SearchCrawlDataResponseSchema, + enhanceEntity: EnhanceEntityResponseSchema, + combineEntityProfiles: CombineEntityProfilesResponseSchema, + resolveLostId: ResolveLostIdResponseSchema, + getKgCoverageReportById: GetKgCoverageReportByIdResponseSchema, + createKgBulkEnhance: CreateKgBulkEnhanceResponseSchema, + getBulkJobStatus: GetBulkJobStatusResponseSchema, + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenResponseSchema, + getBulkResults: GetBulkResultsResponseSchema, + downloadBulkResults: DownloadBulkResultsResponseSchema, + getBulkSingleResult: GetBulkSingleResultResponseSchema, + stopKgBulkJobById: StopKgBulkJobByIdResponseSchema, + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobResponseSchema, + createBulk: CreateBulkResponseSchema, + startBulk: StartBulkResponseSchema, + stopBulkJob: StopBulkJobResponseSchema, + getBulkData: GetBulkDataResponseSchema, + listBulkJobs: ListBulkJobsResponseSchema, + startCrawl: StartCrawlResponseSchema, + manageCrawl: ManageCrawlResponseSchema, + getCrawlData: GetCrawlDataResponseSchema, + createCustomApi: CreateCustomApiResponseSchema, + listCustomApis: ListCustomApisResponseSchema, + deleteCustomApi: DeleteCustomApiResponseSchema, } as const; diff --git a/packages/diffbot/error-handlers.ts b/packages/diffbot/error-handlers.ts index 658a757c2..5899cb599 100644 --- a/packages/diffbot/error-handlers.ts +++ b/packages/diffbot/error-handlers.ts @@ -18,7 +18,11 @@ export const errorHandlers = { match: (error: Error) => { if (hasStatus(error, 429)) return true; const msg = error.message.toLowerCase(); - return msg.includes('rate_limited') || msg.includes('429'); + return ( + msg.includes('rate_limited') || + msg.includes('429') || + msg.includes('too many requests') + ); }, handler: async (error: Error) => { return { maxRetries: 5, headersRetryAfterMs: retryAfter(error) }; @@ -26,14 +30,41 @@ export const errorHandlers = { }, AUTH_ERROR: { match: (error: Error) => { - if (hasStatus(error, 401)) return true; + if (hasStatus(error, 401) || hasStatus(error, 403)) return true; const msg = error.message.toLowerCase(); - return msg.includes('unauthorized') || msg.includes('invalid_auth'); + return ( + msg.includes('unauthorized') || + msg.includes('invalid_auth') || + msg.includes('invalid token') || + msg.includes('forbidden') + ); }, - handler: async () => ({ maxRetries: 0 }), + handler: async (_error?: Error) => ({ maxRetries: 0 }), + }, + NOT_FOUND_ERROR: { + match: (error: Error) => { + if (hasStatus(error, 404)) return true; + const msg = error.message.toLowerCase(); + return msg.includes('not found') || msg.includes('404'); + }, + handler: async (_error?: Error) => ({ maxRetries: 0 }), + }, + SERVER_ERROR: { + match: (error: Error) => { + if ( + hasStatus(error, 500) || + hasStatus(error, 502) || + hasStatus(error, 503) + ) { + return true; + } + const msg = error.message.toLowerCase(); + return msg.includes('internal server error') || msg.includes('500'); + }, + handler: async (_error?: Error) => ({ maxRetries: 2 }), }, DEFAULT: { match: () => true, - handler: async () => ({ maxRetries: 0 }), + handler: async (_error?: Error) => ({ maxRetries: 0 }), }, } satisfies CorsairErrorHandler; diff --git a/packages/diffbot/index.ts b/packages/diffbot/index.ts index 5750b4e48..feefb7b9f 100644 --- a/packages/diffbot/index.ts +++ b/packages/diffbot/index.ts @@ -12,7 +12,16 @@ import type { RequiredPluginEndpointMeta, RequiredPluginEndpointSchemas, } from 'corsair/core'; -import { Extract, Search } from './endpoints'; +import { + Account, + Bulk, + Crawl, + CustomApi, + Enhance, + Extract, + KgBulkEnhance, + Search, +} from './endpoints'; import type { DiffbotEndpointInputs, DiffbotEndpointOutputs, @@ -50,45 +59,252 @@ type DiffbotEndpoint = CorsairEndpoint< >; export type DiffbotEndpoints = { - extractArticle: DiffbotEndpoint<'extractArticle'>; - extractProduct: DiffbotEndpoint<'extractProduct'>; - extractAnalyze: DiffbotEndpoint<'extractAnalyze'>; - searchWeb: DiffbotEndpoint<'searchWeb'>; - searchDql: DiffbotEndpoint<'searchDql'>; + // Account + getAccount: DiffbotEndpoint<'getAccount'>; + + // Extract + getArticle: DiffbotEndpoint<'getArticle'>; + getProduct: DiffbotEndpoint<'getProduct'>; + getAnalyze: DiffbotEndpoint<'getAnalyze'>; + getImage: DiffbotEndpoint<'getImage'>; + getVideo: DiffbotEndpoint<'getVideo'>; + getDiscussion: DiffbotEndpoint<'getDiscussion'>; + getEvent: DiffbotEndpoint<'getEvent'>; + extractList: DiffbotEndpoint<'extractList'>; + extractJob: DiffbotEndpoint<'extractJob'>; + + // Search + search: DiffbotEndpoint<'search'>; + searchCrawlData: DiffbotEndpoint<'searchCrawlData'>; + + // Enhance + enhanceEntity: DiffbotEndpoint<'enhanceEntity'>; + combineEntityProfiles: DiffbotEndpoint<'combineEntityProfiles'>; + resolveLostId: DiffbotEndpoint<'resolveLostId'>; + getKgCoverageReportById: DiffbotEndpoint<'getKgCoverageReportById'>; + + // KG Bulk Enhance + createKgBulkEnhance: DiffbotEndpoint<'createKgBulkEnhance'>; + getBulkJobStatus: DiffbotEndpoint<'getBulkJobStatus'>; + listBulkJobsStatusForToken: DiffbotEndpoint<'listBulkJobsStatusForToken'>; + getBulkResults: DiffbotEndpoint<'getBulkResults'>; + downloadBulkResults: DiffbotEndpoint<'downloadBulkResults'>; + getBulkSingleResult: DiffbotEndpoint<'getBulkSingleResult'>; + stopKgBulkJobById: DiffbotEndpoint<'stopKgBulkJobById'>; + deleteKgEnhanceBulkjob: DiffbotEndpoint<'deleteKgEnhanceBulkjob'>; + + // Bulk Extract + createBulk: DiffbotEndpoint<'createBulk'>; + startBulk: DiffbotEndpoint<'startBulk'>; + stopBulkJob: DiffbotEndpoint<'stopBulkJob'>; + getBulkData: DiffbotEndpoint<'getBulkData'>; + listBulkJobs: DiffbotEndpoint<'listBulkJobs'>; + + // Crawl + startCrawl: DiffbotEndpoint<'startCrawl'>; + manageCrawl: DiffbotEndpoint<'manageCrawl'>; + getCrawlData: DiffbotEndpoint<'getCrawlData'>; + + // Custom API + createCustomApi: DiffbotEndpoint<'createCustomApi'>; + listCustomApis: DiffbotEndpoint<'listCustomApis'>; + deleteCustomApi: DiffbotEndpoint<'deleteCustomApi'>; }; const diffbotEndpointsNested = { + account: { + getAccount: Account.getAccount, + }, extract: { - article: Extract.article, - product: Extract.product, - analyze: Extract.analyze, + getArticle: Extract.getArticle, + getProduct: Extract.getProduct, + getAnalyze: Extract.getAnalyze, + getImage: Extract.getImage, + getVideo: Extract.getVideo, + getDiscussion: Extract.getDiscussion, + getEvent: Extract.getEvent, + extractList: Extract.extractList, + extractJob: Extract.extractJob, }, search: { - web: Search.web, - dql: Search.dql, + search: Search.search, + searchCrawlData: Search.searchCrawlData, + }, + enhance: { + enhanceEntity: Enhance.enhanceEntity, + combineEntityProfiles: Enhance.combineEntityProfiles, + resolveLostId: Enhance.resolveLostId, + getKgCoverageReportById: Enhance.getKgCoverageReportById, + }, + kgBulkEnhance: { + createKgBulkEnhance: KgBulkEnhance.createKgBulkEnhance, + getBulkJobStatus: KgBulkEnhance.getBulkJobStatus, + listBulkJobsStatusForToken: KgBulkEnhance.listBulkJobsStatusForToken, + getBulkResults: KgBulkEnhance.getBulkResults, + downloadBulkResults: KgBulkEnhance.downloadBulkResults, + getBulkSingleResult: KgBulkEnhance.getBulkSingleResult, + stopKgBulkJobById: KgBulkEnhance.stopKgBulkJobById, + deleteKgEnhanceBulkjob: KgBulkEnhance.deleteKgEnhanceBulkjob, + }, + bulk: { + createBulk: Bulk.createBulk, + startBulk: Bulk.startBulk, + stopBulkJob: Bulk.stopBulkJob, + getBulkData: Bulk.getBulkData, + listBulkJobs: Bulk.listBulkJobs, + }, + crawl: { + startCrawl: Crawl.startCrawl, + manageCrawl: Crawl.manageCrawl, + getCrawlData: Crawl.getCrawlData, + }, + customApi: { + createCustomApi: CustomApi.createCustomApi, + listCustomApis: CustomApi.listCustomApis, + deleteCustomApi: CustomApi.deleteCustomApi, }, } as const; export const diffbotEndpointSchemas = { - 'extract.article': { - input: DiffbotEndpointInputSchemas.extractArticle, - output: DiffbotEndpointOutputSchemas.extractArticle, + 'account.getAccount': { + input: DiffbotEndpointInputSchemas.getAccount, + output: DiffbotEndpointOutputSchemas.getAccount, + }, + 'extract.getArticle': { + input: DiffbotEndpointInputSchemas.getArticle, + output: DiffbotEndpointOutputSchemas.getArticle, + }, + 'extract.getProduct': { + input: DiffbotEndpointInputSchemas.getProduct, + output: DiffbotEndpointOutputSchemas.getProduct, + }, + 'extract.getAnalyze': { + input: DiffbotEndpointInputSchemas.getAnalyze, + output: DiffbotEndpointOutputSchemas.getAnalyze, + }, + 'extract.getImage': { + input: DiffbotEndpointInputSchemas.getImage, + output: DiffbotEndpointOutputSchemas.getImage, + }, + 'extract.getVideo': { + input: DiffbotEndpointInputSchemas.getVideo, + output: DiffbotEndpointOutputSchemas.getVideo, + }, + 'extract.getDiscussion': { + input: DiffbotEndpointInputSchemas.getDiscussion, + output: DiffbotEndpointOutputSchemas.getDiscussion, + }, + 'extract.getEvent': { + input: DiffbotEndpointInputSchemas.getEvent, + output: DiffbotEndpointOutputSchemas.getEvent, + }, + 'extract.extractList': { + input: DiffbotEndpointInputSchemas.extractList, + output: DiffbotEndpointOutputSchemas.extractList, + }, + 'extract.extractJob': { + input: DiffbotEndpointInputSchemas.extractJob, + output: DiffbotEndpointOutputSchemas.extractJob, + }, + 'search.search': { + input: DiffbotEndpointInputSchemas.search, + output: DiffbotEndpointOutputSchemas.search, + }, + 'search.searchCrawlData': { + input: DiffbotEndpointInputSchemas.searchCrawlData, + output: DiffbotEndpointOutputSchemas.searchCrawlData, + }, + 'enhance.enhanceEntity': { + input: DiffbotEndpointInputSchemas.enhanceEntity, + output: DiffbotEndpointOutputSchemas.enhanceEntity, + }, + 'enhance.combineEntityProfiles': { + input: DiffbotEndpointInputSchemas.combineEntityProfiles, + output: DiffbotEndpointOutputSchemas.combineEntityProfiles, + }, + 'enhance.resolveLostId': { + input: DiffbotEndpointInputSchemas.resolveLostId, + output: DiffbotEndpointOutputSchemas.resolveLostId, + }, + 'enhance.getKgCoverageReportById': { + input: DiffbotEndpointInputSchemas.getKgCoverageReportById, + output: DiffbotEndpointOutputSchemas.getKgCoverageReportById, }, - 'extract.product': { - input: DiffbotEndpointInputSchemas.extractProduct, - output: DiffbotEndpointOutputSchemas.extractProduct, + 'kgBulkEnhance.createKgBulkEnhance': { + input: DiffbotEndpointInputSchemas.createKgBulkEnhance, + output: DiffbotEndpointOutputSchemas.createKgBulkEnhance, }, - 'extract.analyze': { - input: DiffbotEndpointInputSchemas.extractAnalyze, - output: DiffbotEndpointOutputSchemas.extractAnalyze, + 'kgBulkEnhance.getBulkJobStatus': { + input: DiffbotEndpointInputSchemas.getBulkJobStatus, + output: DiffbotEndpointOutputSchemas.getBulkJobStatus, }, - 'search.web': { - input: DiffbotEndpointInputSchemas.searchWeb, - output: DiffbotEndpointOutputSchemas.searchWeb, + 'kgBulkEnhance.listBulkJobsStatusForToken': { + input: DiffbotEndpointInputSchemas.listBulkJobsStatusForToken, + output: DiffbotEndpointOutputSchemas.listBulkJobsStatusForToken, }, - 'search.dql': { - input: DiffbotEndpointInputSchemas.searchDql, - output: DiffbotEndpointOutputSchemas.searchDql, + 'kgBulkEnhance.getBulkResults': { + input: DiffbotEndpointInputSchemas.getBulkResults, + output: DiffbotEndpointOutputSchemas.getBulkResults, + }, + 'kgBulkEnhance.downloadBulkResults': { + input: DiffbotEndpointInputSchemas.downloadBulkResults, + output: DiffbotEndpointOutputSchemas.downloadBulkResults, + }, + 'kgBulkEnhance.getBulkSingleResult': { + input: DiffbotEndpointInputSchemas.getBulkSingleResult, + output: DiffbotEndpointOutputSchemas.getBulkSingleResult, + }, + 'kgBulkEnhance.stopKgBulkJobById': { + input: DiffbotEndpointInputSchemas.stopKgBulkJobById, + output: DiffbotEndpointOutputSchemas.stopKgBulkJobById, + }, + 'kgBulkEnhance.deleteKgEnhanceBulkjob': { + input: DiffbotEndpointInputSchemas.deleteKgEnhanceBulkjob, + output: DiffbotEndpointOutputSchemas.deleteKgEnhanceBulkjob, + }, + 'bulk.createBulk': { + input: DiffbotEndpointInputSchemas.createBulk, + output: DiffbotEndpointOutputSchemas.createBulk, + }, + 'bulk.startBulk': { + input: DiffbotEndpointInputSchemas.startBulk, + output: DiffbotEndpointOutputSchemas.startBulk, + }, + 'bulk.stopBulkJob': { + input: DiffbotEndpointInputSchemas.stopBulkJob, + output: DiffbotEndpointOutputSchemas.stopBulkJob, + }, + 'bulk.getBulkData': { + input: DiffbotEndpointInputSchemas.getBulkData, + output: DiffbotEndpointOutputSchemas.getBulkData, + }, + 'bulk.listBulkJobs': { + input: DiffbotEndpointInputSchemas.listBulkJobs, + output: DiffbotEndpointOutputSchemas.listBulkJobs, + }, + 'crawl.startCrawl': { + input: DiffbotEndpointInputSchemas.startCrawl, + output: DiffbotEndpointOutputSchemas.startCrawl, + }, + 'crawl.manageCrawl': { + input: DiffbotEndpointInputSchemas.manageCrawl, + output: DiffbotEndpointOutputSchemas.manageCrawl, + }, + 'crawl.getCrawlData': { + input: DiffbotEndpointInputSchemas.getCrawlData, + output: DiffbotEndpointOutputSchemas.getCrawlData, + }, + 'customApi.createCustomApi': { + input: DiffbotEndpointInputSchemas.createCustomApi, + output: DiffbotEndpointOutputSchemas.createCustomApi, + }, + 'customApi.listCustomApis': { + input: DiffbotEndpointInputSchemas.listCustomApis, + output: DiffbotEndpointOutputSchemas.listCustomApis, + }, + 'customApi.deleteCustomApi': { + input: DiffbotEndpointInputSchemas.deleteCustomApi, + output: DiffbotEndpointOutputSchemas.deleteCustomApi, }, } as const satisfies RequiredPluginEndpointSchemas< typeof diffbotEndpointsNested @@ -97,30 +313,168 @@ export const diffbotEndpointSchemas = { const defaultAuthType: AuthTypes = 'api_key' as const; const diffbotEndpointMeta = { - 'extract.article': { + 'account.getAccount': { + riskLevel: 'read', + description: + 'Retrieve Diffbot account details, credit balance, and plan usage', + }, + 'extract.getArticle': { riskLevel: 'read', description: 'Extract article title, text, author, date, and metadata from any URL', }, - 'extract.product': { + 'extract.getProduct': { + riskLevel: 'read', + description: + 'Extract product price, availability, images, and specifications from any e-commerce URL', + }, + 'extract.getAnalyze': { + riskLevel: 'read', + description: + 'Automatically analyze web page to determine its type and extract structured data', + }, + 'extract.getImage': { riskLevel: 'read', description: - 'Extract product price, availability, images, and specs from any e-commerce URL', + 'Extract detailed image information including dimensions and recognition data', }, - 'extract.analyze': { + 'extract.getVideo': { riskLevel: 'read', description: - 'Auto-detect page type and extract structured data from any URL', + 'Extract structured metadata from videos including embed HTML and durations', }, - 'search.web': { + 'extract.getDiscussion': { riskLevel: 'read', description: - 'Search the web and return structured results with article metadata', + 'Extract structured discussion threads, forum posts, and comments from web pages', }, - 'search.dql': { + 'extract.getEvent': { riskLevel: 'read', description: - 'Query the Diffbot Knowledge Graph using DQL (Diffbot Query Language)', + 'Extract event details including dates, venues, organizers, and descriptions', + }, + 'extract.extractList': { + riskLevel: 'read', + description: + 'Extract structured items from list-style pages, catalogs, and news indexes', + }, + 'extract.extractJob': { + riskLevel: 'read', + description: + 'Extract structured job posting data including compensation, requirements, and company info', + }, + 'search.search': { + riskLevel: 'read', + description: + 'Search the Diffbot Knowledge Graph using DQL (Diffbot Query Language)', + }, + 'search.searchCrawlData': { + riskLevel: 'read', + description: 'Query crawl job collections using DQL or keyword search', + }, + 'enhance.enhanceEntity': { + riskLevel: 'read', + description: + 'Enrich person or organization data with Knowledge Graph records', + }, + 'enhance.combineEntityProfiles': { + riskLevel: 'read', + description: + 'Combine entity profiles into a unified view with organization affiliations', + }, + 'enhance.resolveLostId': { + riskLevel: 'read', + description: + 'Resolve lost or legacy identifiers to canonical Knowledge Graph entities', + }, + 'enhance.getKgCoverageReportById': { + riskLevel: 'read', + description: 'Download Knowledge Graph coverage report by report ID', + }, + 'kgBulkEnhance.createKgBulkEnhance': { + riskLevel: 'write', + description: + 'Submit an asynchronous bulk enhance job for multiple entities', + }, + 'kgBulkEnhance.getBulkJobStatus': { + riskLevel: 'read', + description: + 'Poll the status and progress of a Knowledge Graph bulk enhance job', + }, + 'kgBulkEnhance.listBulkJobsStatusForToken': { + riskLevel: 'read', + description: + 'List all Knowledge Graph bulk enhance jobs and their statuses for token', + }, + 'kgBulkEnhance.getBulkResults': { + riskLevel: 'read', + description: + 'Download results of a completed Knowledge Graph bulk enhance job', + }, + 'kgBulkEnhance.downloadBulkResults': { + riskLevel: 'read', + description: + 'Download bulk enhance results with filtering and custom output formats', + }, + 'kgBulkEnhance.getBulkSingleResult': { + riskLevel: 'read', + description: + 'Download single enriched entity result from a bulk enhance job by index', + }, + 'kgBulkEnhance.stopKgBulkJobById': { + riskLevel: 'write', + description: + 'Stop or pause an active Knowledge Graph bulk enhance job by ID', + }, + 'kgBulkEnhance.deleteKgEnhanceBulkjob': { + riskLevel: 'destructive', + description: 'Delete a Knowledge Graph bulk enhance job and its results', + }, + 'bulk.createBulk': { + riskLevel: 'write', + description: + 'Submit an asynchronous bulk extract job to process multiple URLs', + }, + 'bulk.startBulk': { + riskLevel: 'write', + description: 'Start a bulk extract job using query parameters', + }, + 'bulk.stopBulkJob': { + riskLevel: 'write', + description: 'Pause or stop an active bulk extract job', + }, + 'bulk.getBulkData': { + riskLevel: 'read', + description: 'Download extracted results from a completed bulk extract job', + }, + 'bulk.listBulkJobs': { + riskLevel: 'read', + description: 'List all bulk extract jobs associated with the token', + }, + 'crawl.startCrawl': { + riskLevel: 'write', + description: 'Initiate a website crawl job starting from seed URLs', + }, + 'crawl.manageCrawl': { + riskLevel: 'write', + description: 'Inspect, pause, restart, or delete crawl jobs', + }, + 'crawl.getCrawlData': { + riskLevel: 'read', + description: 'Download extracted data from a completed crawl job', + }, + 'customApi.createCustomApi': { + riskLevel: 'write', + description: + 'Create or update custom API rules and selectors for URL patterns', + }, + 'customApi.listCustomApis': { + riskLevel: 'read', + description: 'List all custom API definitions configured on the account', + }, + 'customApi.deleteCustomApi': { + riskLevel: 'destructive', + description: 'Delete custom API definitions for a given URL pattern', }, } as const satisfies RequiredPluginEndpointMeta; @@ -180,17 +534,5 @@ export function diffbot( } satisfies InternalDiffbotPlugin; } -export type { - AnalyzeInput, - AnalyzeResponse, - DiffbotEndpointInputs, - DiffbotEndpointOutputs, - DqlSearchInput, - DqlSearchResponse, - ExtractArticleInput, - ExtractArticleResponse, - ExtractProductInput, - ExtractProductResponse, - WebSearchInput, - WebSearchResponse, -} from './endpoints/types'; +export * from './endpoints/types'; +export * from './schema'; diff --git a/packages/diffbot/schema.test.ts b/packages/diffbot/schema.test.ts index 40bd766bc..c9b9a6ff4 100644 --- a/packages/diffbot/schema.test.ts +++ b/packages/diffbot/schema.test.ts @@ -6,15 +6,28 @@ describe('Diffbot schema', () => { expect(DiffbotSchema.version).toMatch(/^\d+\.\d+\.\d+$/); }); - it('declares an entities map', () => { + it('declares an entities map with all official ontology tables', () => { expect(typeof DiffbotSchema.entities).toBe('object'); expect(DiffbotSchema.entities).not.toBeNull(); - expect(Array.isArray(Object.keys(DiffbotSchema.entities))).toBe(true); + const entityKeys = Object.keys(DiffbotSchema.entities); + expect(entityKeys.length).toBeGreaterThanOrEqual(10); + expect(entityKeys).toContain('articles'); + expect(entityKeys).toContain('products'); + expect(entityKeys).toContain('discussions'); + expect(entityKeys).toContain('images'); + expect(entityKeys).toContain('videos'); + expect(entityKeys).toContain('events'); + expect(entityKeys).toContain('jobs'); + expect(entityKeys).toContain('lists'); + expect(entityKeys).toContain('organizations'); + expect(entityKeys).toContain('people'); + expect(entityKeys).toContain('crawlJobs'); + expect(entityKeys).toContain('bulkJobs'); + expect(entityKeys).toContain('customApis'); + expect(entityKeys).toContain('accounts'); + for (const entity of Object.values(DiffbotSchema.entities)) { expect(entity).toBeDefined(); } }); }); - -// Per .github/PLUGIN_PR_RULES.md (R2), every implemented endpoint -// needs a corresponding test. diff --git a/packages/diffbot/schema/database.ts b/packages/diffbot/schema/database.ts index 9e2370527..24f765dee 100644 --- a/packages/diffbot/schema/database.ts +++ b/packages/diffbot/schema/database.ts @@ -2,17 +2,76 @@ import { z } from 'zod'; /** * DiffbotArticle — cached article entity. - * Useful when storing extracted articles locally via the Corsair database. + * Represents structured article data extracted by Diffbot Article API / Knowledge Graph. + * @see https://docs.diffbot.com/docs/ontology/article */ export const DiffbotArticle = z.object({ + id: z.string().optional(), + type: z.literal('article').optional(), pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), title: z.string().optional(), text: z.string().optional(), - author: z.string().optional(), + html: z.string().optional(), date: z.string().optional(), + estimatedDate: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), siteName: z.string().optional(), humanLanguage: z.string().optional(), - tags: z.array(z.object({ label: z.string() })).optional(), + numPages: z.number().optional(), + nextPage: z.string().optional(), + nextPages: z.array(z.string()).optional(), + images: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + videos: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + duration: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + tags: z + .array( + z + .object({ + id: z.number().optional(), + label: z.string(), + uri: z.string().optional(), + score: z.number().optional(), + types: z.array(z.string()).optional(), + }) + .passthrough(), + ) + .optional(), + links: z.array(z.string()).optional(), + breadcrumb: z + .array( + z.object({ link: z.string().optional(), name: z.string().optional() }), + ) + .optional(), + publisherRegion: z.string().optional(), + publisherCountry: z.string().optional(), + sentiment: z.number().optional(), + diffbotUri: z.string().optional(), extractedAt: z.coerce.date().nullable().optional(), }); @@ -20,18 +79,515 @@ export type DiffbotArticle = z.infer; /** * DiffbotProduct — cached product entity. - * Useful when storing extracted product data locally via the Corsair database. + * Represents structured product data extracted by Diffbot Product API / Knowledge Graph. + * @see https://docs.diffbot.com/docs/ontology/product */ export const DiffbotProduct = z.object({ + id: z.string().optional(), + type: z.literal('product').optional(), pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), title: z.string().optional(), + text: z.string().optional(), brand: z.string().optional(), offerPrice: z.string().optional(), + offerPriceDetails: z + .object({ + amount: z.number().optional(), + symbol: z.string().optional(), + text: z.string().optional(), + }) + .passthrough() + .optional(), regularPrice: z.string().optional(), + saveAmount: z.string().optional(), + shippingAmount: z.string().optional(), availability: z.boolean().optional(), sku: z.string().optional(), + mpn: z.string().optional(), + upc: z.string().optional(), + isbn: z.string().optional(), + images: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + primary: z.boolean().optional(), + }) + .passthrough(), + ) + .optional(), + offers: z + .array( + z + .object({ + price: z.string().optional(), + priceCurrency: z.string().optional(), + seller: z.string().optional(), + availability: z.boolean().optional(), + }) + .passthrough(), + ) + .optional(), + colors: z.array(z.string()).optional(), humanLanguage: z.string().optional(), + tags: z + .array( + z + .object({ + label: z.string(), + score: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + diffbotUri: z.string().optional(), extractedAt: z.coerce.date().nullable().optional(), }); export type DiffbotProduct = z.infer; + +/** + * DiffbotDiscussion — cached discussion thread entity. + * @see https://docs.diffbot.com/docs/ontology/discussion + */ +export const DiffbotDiscussion = z.object({ + id: z.string().optional(), + type: z.literal('discussion').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + numPosts: z.number().optional(), + numParticipants: z.number().optional(), + participants: z.array(z.string()).optional(), + rssUrl: z.string().optional(), + posts: z + .array( + z + .object({ + id: z.number().optional(), + text: z.string().optional(), + html: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + date: z.string().optional(), + parentId: z.number().optional(), + voteCount: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotDiscussion = z.infer; + +/** + * DiffbotImage — cached image extraction entity. + * @see https://docs.diffbot.com/docs/ontology/image + */ +export const DiffbotImage = z.object({ + id: z.string().optional(), + type: z.literal('image').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotImage = z.infer; + +/** + * DiffbotVideo — cached video extraction entity. + * @see https://docs.diffbot.com/docs/ontology/video + */ +export const DiffbotVideo = z.object({ + id: z.string().optional(), + type: z.literal('video').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + duration: z.number().optional(), + viewCount: z.number().optional(), + uploadDate: z.string().optional(), + author: z.string().optional(), + embedUrl: z.string().optional(), + html: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotVideo = z.infer; + +/** + * DiffbotEvent — cached event entity. + * @see https://docs.diffbot.com/docs/ontology/event + */ +export const DiffbotEvent = z.object({ + id: z.string().optional(), + type: z.literal('event').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + startDate: z.string().optional(), + endDate: z.string().optional(), + location: z.string().optional(), + venue: z + .object({ + name: z.string().optional(), + address: z.string().optional(), + city: z.string().optional(), + state: z.string().optional(), + country: z.string().optional(), + }) + .passthrough() + .optional(), + organizer: z.string().optional(), + ticketUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotEvent = z.infer; + +/** + * DiffbotJob — cached job posting entity. + * @see https://docs.diffbot.com/docs/ontology/jobpost + */ +export const DiffbotJob = z.object({ + id: z.string().optional(), + type: z.literal('job').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + company: z + .object({ + name: z.string().optional(), + url: z.string().optional(), + }) + .passthrough() + .optional(), + locations: z.array(z.string()).optional(), + employmentType: z.string().optional(), + compensation: z + .object({ + min: z.number().optional(), + max: z.number().optional(), + currency: z.string().optional(), + interval: z.string().optional(), + }) + .passthrough() + .optional(), + requirements: z.array(z.string()).optional(), + skills: z.array(z.string()).optional(), + postedDate: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotJob = z.infer; + +/** + * DiffbotList — cached list extraction entity. + * @see https://docs.diffbot.com/docs/extract/list + */ +export const DiffbotList = z.object({ + id: z.string().optional(), + type: z.literal('list').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + numItems: z.number().optional(), + items: z + .array( + z + .object({ + title: z.string().optional(), + link: z.string().optional(), + description: z.string().optional(), + image: z.string().optional(), + price: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + humanLanguage: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotList = z.infer; + +/** + * DiffbotOrganization — Knowledge Graph organization entity. + * @see https://docs.diffbot.com/docs/ontology/organization + */ +export const DiffbotOrganization = z.object({ + id: z.string(), + name: z.string(), + type: z.literal('Organization').optional(), + types: z.array(z.string()).optional(), + diffbotUri: z.string().optional(), + homepageUri: z.string().optional(), + description: z.string().optional(), + summary: z.string().optional(), + logo: z.string().optional(), + image: z.string().optional(), + images: z.array(z.string()).optional(), + nbEmployees: z.number().optional(), + nbEmployeesMin: z.number().optional(), + nbEmployeesMax: z.number().optional(), + revenue: z.number().optional(), + yearlyRevenues: z + .array( + z + .object({ + year: z.number().optional(), + revenue: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + isPublic: z.boolean().optional(), + isNonProfit: z.boolean().optional(), + isAcquired: z.boolean().optional(), + isDissolved: z.boolean().optional(), + founders: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + ceo: z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough() + .optional(), + boardMembers: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + competitors: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + totalInvestment: z.number().optional(), + location: z + .object({ + city: z.string().optional(), + region: z.string().optional(), + country: z.string().optional(), + address: z.string().optional(), + }) + .passthrough() + .optional(), + locations: z.array(z.record(z.string(), z.unknown())).optional(), + emailAddresses: z.array(z.string()).optional(), + phoneNumbers: z.array(z.string()).optional(), + linkedInUri: z.string().optional(), + twitterUri: z.string().optional(), + facebookUri: z.string().optional(), + githubUri: z.string().optional(), + wikipediaUri: z.string().optional(), + crunchbaseUri: z.string().optional(), + angellistUri: z.string().optional(), + categories: z.array(z.string()).optional(), + industries: z.array(z.string()).optional(), + naicsClassification: z.string().optional(), + sicClassification: z.string().optional(), + naceClassification: z.string().optional(), + crawlTimestamp: z.number().optional(), + importance: z.number().optional(), + origin: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotOrganization = z.infer; + +/** + * DiffbotPerson — Knowledge Graph person entity. + * @see https://docs.diffbot.com/docs/ontology/person + */ +export const DiffbotPerson = z.object({ + id: z.string(), + name: z.string(), + type: z.literal('Person').optional(), + types: z.array(z.string()).optional(), + diffbotUri: z.string().optional(), + description: z.string().optional(), + summary: z.string().optional(), + image: z.string().optional(), + images: z.array(z.string()).optional(), + gender: z.string().optional(), + birthDate: z.string().optional(), + deathDate: z.string().optional(), + educations: z.array(z.record(z.string(), z.unknown())).optional(), + employments: z.array(z.record(z.string(), z.unknown())).optional(), + awards: z.array(z.record(z.string(), z.unknown())).optional(), + skills: z.array(z.string()).optional(), + interests: z.array(z.string()).optional(), + location: z + .object({ + city: z.string().optional(), + region: z.string().optional(), + country: z.string().optional(), + }) + .passthrough() + .optional(), + emailAddresses: z.array(z.string()).optional(), + phoneNumbers: z.array(z.string()).optional(), + linkedInUri: z.string().optional(), + twitterUri: z.string().optional(), + facebookUri: z.string().optional(), + githubUri: z.string().optional(), + wikipediaUri: z.string().optional(), + crunchbaseUri: z.string().optional(), + angellistUri: z.string().optional(), + homepageUri: z.string().optional(), + crawlTimestamp: z.number().optional(), + importance: z.number().optional(), + origin: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotPerson = z.infer; + +/** + * DiffbotCrawlJob — Crawl job record. + * @see https://docs.diffbot.com/docs/crawl/ + */ +export const DiffbotCrawlJob = z.object({ + name: z.string(), + jobStatus: z + .object({ + status: z.number().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + sentToCrawler: z.number().optional(), + objectsHarvested: z.number().optional(), + urlsHarvested: z.number().optional(), + pageRounds: z.number().optional(), + maxRounds: z.number().optional(), + maxHops: z.number().optional(), + pause: z.number().optional(), + roundProxy: z.number().optional(), + seeds: z.string().optional(), + apiUrl: z.string().optional(), + downloadUrl: z.string().optional(), + maxTags: z.number().optional(), + status: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotCrawlJob = z.infer; + +/** + * DiffbotBulkJob — Bulk extract or Bulk enhance job record. + * @see https://docs.diffbot.com/docs/bulk/ + */ +export const DiffbotBulkJob = z.object({ + id: z.string(), + name: z.string().optional(), + bulkjobId: z.string().optional(), + kind: z.enum(['extract', 'enhance']).optional(), + status: z.string().optional(), + jobStatus: z + .object({ + status: z.number().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + total: z.number().optional(), + completed: z.number().optional(), + failed: z.number().optional(), + format: z.string().optional(), + apiUrl: z.string().optional(), + urls: z.string().optional(), + downloadUrl: z.string().optional(), + createdAt: z.string().optional(), + updatedAt: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotBulkJob = z.infer; + +/** + * DiffbotCustomApi — Custom API extraction configuration. + * @see https://docs.diffbot.com/docs/custom-api/ + */ +export const DiffbotCustomApi = z.object({ + id: z.string(), + api: z.string(), + url: z.string(), + pattern: z.string().optional(), + ruleset: z.record(z.string(), z.unknown()).optional(), + selectors: z.record(z.string(), z.unknown()).optional(), + testUrl: z.string().optional(), + createdAt: z.string().optional(), + updatedAt: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotCustomApi = z.infer; + +/** + * DiffbotAccount — Account details and API quota. + * @see https://docs.diffbot.com/docs/account + */ +export const DiffbotAccount = z.object({ + id: z.string(), + token: z.string(), + name: z.string().optional(), + email: z.string().optional(), + plan: z.string().optional(), + planStart: z.string().optional(), + planCalls: z.number().optional(), + apiCalls: z.number().optional(), + status: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotAccount = z.infer; diff --git a/packages/diffbot/schema/index.ts b/packages/diffbot/schema/index.ts index d3711cf0d..cf55a97e2 100644 --- a/packages/diffbot/schema/index.ts +++ b/packages/diffbot/schema/index.ts @@ -1,4 +1,38 @@ +import { + DiffbotAccount, + DiffbotArticle, + DiffbotBulkJob, + DiffbotCrawlJob, + DiffbotCustomApi, + DiffbotDiscussion, + DiffbotEvent, + DiffbotImage, + DiffbotJob, + DiffbotList, + DiffbotOrganization, + DiffbotPerson, + DiffbotProduct, + DiffbotVideo, +} from './database'; + export const DiffbotSchema = { version: '1.0.0', - entities: {}, + entities: { + articles: DiffbotArticle, + products: DiffbotProduct, + discussions: DiffbotDiscussion, + images: DiffbotImage, + videos: DiffbotVideo, + events: DiffbotEvent, + jobs: DiffbotJob, + lists: DiffbotList, + organizations: DiffbotOrganization, + people: DiffbotPerson, + crawlJobs: DiffbotCrawlJob, + bulkJobs: DiffbotBulkJob, + customApis: DiffbotCustomApi, + accounts: DiffbotAccount, + }, } as const; + +export * from './database'; From a38e3fe0de5e838c4f2ea3b8a093d39ac8703ac6 Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Mon, 24 Aug 2026 02:46:11 +0530 Subject: [PATCH 12/13] test(diffbot): add request-mapping tests for all 35 callable endpoints --- packages/diffbot/api.test.ts | 554 +++++++++++++++++++++++++++++++++-- 1 file changed, 530 insertions(+), 24 deletions(-) diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index 092ad3e6b..4caafe989 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -344,7 +344,7 @@ describe('Diffbot Input and Output Schemas', () => { }); }); -describe('Diffbot Endpoint Handlers', () => { +describe('Diffbot Endpoint Handlers — All 35 Operations Request Mapping', () => { let makeRequestSpy: jest.SpyInstance; const mockCtx = { key: 'test_token', @@ -364,87 +364,593 @@ describe('Diffbot Endpoint Handlers', () => { jest.restoreAllMocks(); }); - it('invokes account.getAccount correctly', async () => { + // 1. Account (1 operation) + it('1. invokes account.getAccount correctly', async () => { await Account.getAccount(mockCtx, {}); expect(makeRequestSpy).toHaveBeenCalledWith('account', 'test_token', { method: 'GET', }); }); - it('invokes extract.getArticle correctly', async () => { - await Extract.getArticle(mockCtx, { url: 'https://example.com/article' }); + // 2. Extract (9 operations) + it('2. invokes extract.getArticle correctly', async () => { + await Extract.getArticle(mockCtx, { + url: 'https://example.com/article', + fields: 'meta,links', + timeout: 15000, + paging: 'false', + maxTags: 5, + naturalLanguage: 'en', + }); expect(makeRequestSpy).toHaveBeenCalledWith('article', 'test_token', { method: 'GET', - query: expect.objectContaining({ url: 'https://example.com/article' }), + query: { + url: 'https://example.com/article', + fields: 'meta,links', + timeout: 15000, + paging: 'false', + maxTags: 5, + naturalLanguage: 'en', + }, }); }); - it('invokes search.search with DQL routing to KG base', async () => { + it('3. invokes extract.getProduct correctly', async () => { + await Extract.getProduct(mockCtx, { + url: 'https://example.com/product', + fields: 'brand,offers', + timeout: 20000, + discussion: 'false', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('product', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/product', + fields: 'brand,offers', + timeout: 20000, + discussion: 'false', + }, + }); + }); + + it('4. invokes extract.getAnalyze correctly', async () => { + await Extract.getAnalyze(mockCtx, { + url: 'https://example.com/unknown', + fallback: 'article', + discussion: 'false', + timeout: 10000, + fields: 'title', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('analyze', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/unknown', + fallback: 'article', + discussion: 'false', + timeout: 10000, + fields: 'title', + }, + }); + }); + + it('5. invokes extract.getImage correctly', async () => { + await Extract.getImage(mockCtx, { + url: 'https://example.com/image.jpg', + fields: 'xpath', + timeout: 12000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('image', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/image.jpg', + fields: 'xpath', + timeout: 12000, + }, + }); + }); + + it('6. invokes extract.getVideo correctly', async () => { + await Extract.getVideo(mockCtx, { + url: 'https://example.com/video.mp4', + fields: 'duration', + timeout: 12000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('video', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/video.mp4', + fields: 'duration', + timeout: 12000, + }, + }); + }); + + it('7. invokes extract.getDiscussion correctly', async () => { + await Extract.getDiscussion(mockCtx, { + url: 'https://example.com/forum', + fields: 'posts', + timeout: 18000, + maxTags: 10, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('discussion', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/forum', + fields: 'posts', + timeout: 18000, + maxTags: 10, + }, + }); + }); + + it('8. invokes extract.getEvent correctly', async () => { + await Extract.getEvent(mockCtx, { + url: 'https://example.com/event', + fields: 'venue', + timeout: 15000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('event', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/event', + fields: 'venue', + timeout: 15000, + }, + }); + }); + + it('9. invokes extract.extractList correctly', async () => { + await Extract.extractList(mockCtx, { + url: 'https://example.com/directory', + fields: 'items', + timeout: 25000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('list', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/directory', + fields: 'items', + timeout: 25000, + }, + }); + }); + + it('10. invokes extract.extractJob correctly', async () => { + await Extract.extractJob(mockCtx, { + url: 'https://example.com/careers/job1', + fields: 'compensation', + timeout: 20000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('job', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/careers/job1', + fields: 'compensation', + timeout: 20000, + }, + }); + }); + + // 3. Search (2 operations) + it('11. invokes search.search with DQL routing to KG base', async () => { await Search.search(mockCtx, { query: 'name:"Diffbot"', entityType: 'Organization', + queryType: 'query', + size: 20, + from: 0, }); expect(makeRequestSpy).toHaveBeenCalledWith('dql', 'test_token', { method: 'GET', useKgBase: true, - query: expect.objectContaining({ + query: { query: 'type:Organization name:"Diffbot"', - }), + type: 'query', + size: 20, + from: 0, + col: undefined, + }, + }); + }); + + it('12. invokes search.searchCrawlData correctly', async () => { + await Search.searchCrawlData(mockCtx, { + col: 'myCrawlCollection', + query: 'tech', + num: 15, + start: 5, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('search', 'test_token', { + method: 'GET', + query: { + col: 'myCrawlCollection', + query: 'tech', + num: 15, + start: 5, + }, }); }); - it('invokes enhance.enhanceEntity correctly', async () => { - await Enhance.enhanceEntity(mockCtx, { name: 'Diffbot' }); + // 4. Enhance (4 operations) + it('13. invokes enhance.enhanceEntity correctly', async () => { + await Enhance.enhanceEntity(mockCtx, { + name: 'Diffbot Technologies', + type: 'Organization', + url: 'https://diffbot.com', + size: 1, + refresh: true, + }); expect(makeRequestSpy).toHaveBeenCalledWith('enhance', 'test_token', { method: 'GET', useKgBase: true, - query: expect.objectContaining({ name: 'Diffbot' }), + query: { + name: 'Diffbot Technologies', + type: 'Organization', + url: 'https://diffbot.com', + size: 1, + refresh: true, + email: undefined, + employer: undefined, + phone: undefined, + location: undefined, + }, + }); + }); + + it('14. invokes enhance.combineEntityProfiles correctly', async () => { + await Enhance.combineEntityProfiles(mockCtx, { + name: 'Mike Tung', + type: 'Person', + employer: 'Diffbot', + email: 'mike@diffbot.com', + url: 'https://linkedin.com/in/miketung', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/combine', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: { + name: 'Mike Tung', + type: 'Person', + employer: 'Diffbot', + email: 'mike@diffbot.com', + url: 'https://linkedin.com/in/miketung', + }, + }, + ); + }); + + it('15. invokes enhance.resolveLostId correctly', async () => { + await Enhance.resolveLostId(mockCtx, { id: 'OLD_KG_ID_999' }); + expect(makeRequestSpy).toHaveBeenCalledWith('dql', 'test_token', { + method: 'GET', + useKgBase: true, + query: { + query: 'id:"OLD_KG_ID_999"', + size: 1, + }, }); }); - it('invokes kgBulkEnhance.createKgBulkEnhance correctly', async () => { + it('16. invokes enhance.getKgCoverageReportById correctly', async () => { + await Enhance.getKgCoverageReportById(mockCtx, { reportId: 'rep_abc123' }); + expect(makeRequestSpy).toHaveBeenCalledWith('report', 'test_token', { + method: 'GET', + useKgBase: true, + query: { reportId: 'rep_abc123' }, + }); + + await Enhance.getKgCoverageReportById(mockCtx, { + reportId: 'rep_abc123', + bulkjobId: 'bulk_456', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/report/bulk_456/rep_abc123', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: {}, + }, + ); + }); + + // 5. KG Bulk Enhance (8 operations) + it('17. invokes kgBulkEnhance.createKgBulkEnhance correctly', async () => { await KgBulkEnhance.createKgBulkEnhance(mockCtx, { - entities: [{ name: 'Diffbot' }], + entities: [{ name: 'Diffbot' }, { name: 'Anthropic' }], + name: 'enrichJob1', + notifyEmail: 'dev@example.com', }); expect(makeRequestSpy).toHaveBeenCalledWith('enhance/bulk', 'test_token', { method: 'POST', useKgBase: true, - body: [{ name: 'Diffbot' }], - query: expect.anything(), + body: [{ name: 'Diffbot' }, { name: 'Anthropic' }], + query: { + name: 'enrichJob1', + notifyEmail: 'dev@example.com', + }, + }); + }); + + it('18. invokes kgBulkEnhance.getBulkJobStatus correctly', async () => { + await KgBulkEnhance.getBulkJobStatus(mockCtx, { bulkjobId: 'bj_100' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/status', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('19. invokes kgBulkEnhance.listBulkJobsStatusForToken correctly', async () => { + await KgBulkEnhance.listBulkJobsStatusForToken(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance/bulk', 'test_token', { + method: 'GET', + useKgBase: true, }); }); - it('invokes bulk.createBulk correctly', async () => { + it('20. invokes kgBulkEnhance.getBulkResults correctly', async () => { + await KgBulkEnhance.getBulkResults(mockCtx, { + bulkjobId: 'bj_100', + format: 'json', + head: 50, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: { + format: 'json', + head: 50, + }, + }, + ); + }); + + it('21. invokes kgBulkEnhance.downloadBulkResults correctly', async () => { + await KgBulkEnhance.downloadBulkResults(mockCtx, { + bulkjobId: 'bj_100', + format: 'jsonl', + filter: 'importance>0.5', + fields: 'name,location', + head: 100, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100', + 'test_token', + { + method: 'POST', + useKgBase: true, + query: { + format: 'jsonl', + filter: 'importance>0.5', + fields: 'name,location', + head: 100, + }, + }, + ); + }); + + it('22. invokes kgBulkEnhance.getBulkSingleResult correctly', async () => { + await KgBulkEnhance.getBulkSingleResult(mockCtx, { + bulkjobId: 'bj_100', + jobIndex: 3, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/3', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('23. invokes kgBulkEnhance.stopKgBulkJobById correctly', async () => { + await KgBulkEnhance.stopKgBulkJobById(mockCtx, { bulkjobId: 'bj_100' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/stop', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('24. invokes kgBulkEnhance.deleteKgEnhanceBulkjob correctly', async () => { + await KgBulkEnhance.deleteKgEnhanceBulkjob(mockCtx, { + bulkjobId: 'bj_100', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/delete', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + // 6. Bulk Extract (5 operations) + it('25. invokes bulk.createBulk correctly', async () => { await Bulk.createBulk(mockCtx, { - name: 'job1', + name: 'jobExtract', apiUrl: 'https://api.diffbot.com/v3/article', - urls: ['https://example.com/1'], + urls: ['https://example.com/1', 'https://example.com/2'], + notifyEmail: 'notify@example.com', + maxRounds: 3, }); expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { method: 'POST', - body: 'https://example.com/1', - query: expect.objectContaining({ name: 'job1' }), + body: 'https://example.com/1\nhttps://example.com/2', + query: { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + notifyEmail: 'notify@example.com', + maxRounds: 3, + }, + }); + }); + + it('26. invokes bulk.startBulk correctly', async () => { + await Bulk.startBulk(mockCtx, { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + notifyEmail: 'notify@example.com', + maxRounds: 2, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', + query: { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + notifyEmail: 'notify@example.com', + maxRounds: 2, + }, + }); + }); + + it('27. invokes bulk.stopBulkJob correctly', async () => { + await Bulk.stopBulkJob(mockCtx, { name: 'jobExtract' }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', + query: { + name: 'jobExtract', + pause: 1, + }, + }); + }); + + it('28. invokes bulk.getBulkData correctly', async () => { + await Bulk.getBulkData(mockCtx, { name: 'jobExtract', format: 'csv' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'bulk/download/test_token-jobExtract.csv', + 'test_token', + { + method: 'GET', + }, + ); + }); + + it('29. invokes bulk.listBulkJobs correctly', async () => { + await Bulk.listBulkJobs(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', }); }); - it('invokes crawl.startCrawl correctly', async () => { + // 7. Crawl (3 operations) + it('30. invokes crawl.startCrawl correctly', async () => { await Crawl.startCrawl(mockCtx, { - name: 'crawl1', + name: 'crawlJob1', seeds: 'https://example.com', apiUrl: 'https://api.diffbot.com/v3/article', + maxHops: 2, + maxRounds: 1, + maxTags: 5, + crawlSubdomains: 1, + notifyEmail: 'crawl@example.com', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('crawl', 'test_token', { + method: 'POST', + query: { + name: 'crawlJob1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + maxHops: 2, + maxRounds: 1, + maxTags: 5, + crawlSubdomains: 1, + notifyEmail: 'crawl@example.com', + }, + }); + }); + + it('31. invokes crawl.manageCrawl correctly', async () => { + await Crawl.manageCrawl(mockCtx, { + name: 'crawlJob1', + pause: 1, + restart: 0, + delete: 0, + roundProxy: 1, + maxRounds: 5, + maxHops: 3, }); expect(makeRequestSpy).toHaveBeenCalledWith('crawl', 'test_token', { + method: 'GET', + query: { + name: 'crawlJob1', + pause: 1, + restart: 0, + delete: 0, + roundProxy: 1, + maxRounds: 5, + maxHops: 3, + }, + }); + }); + + it('32. invokes crawl.getCrawlData correctly', async () => { + await Crawl.getCrawlData(mockCtx, { name: 'crawlJob1', format: 'json' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'crawl/download/test_token-crawlJob1.json', + 'test_token', + { + method: 'GET', + }, + ); + }); + + // 8. Custom API (3 operations) + it('33. invokes customApi.createCustomApi correctly', async () => { + await CustomApi.createCustomApi(mockCtx, { + api: 'myCustomApi', + url: 'https://example.com/custom', + pattern: 'https://example.com/*', + rules: { selector: '.article-body' }, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { method: 'POST', - query: expect.objectContaining({ name: 'crawl1' }), + body: { selector: '.article-body' }, + query: { + api: 'myCustomApi', + url: 'https://example.com/custom', + pattern: 'https://example.com/*', + }, }); }); - it('invokes customApi.listCustomApis correctly', async () => { + it('34. invokes customApi.listCustomApis correctly', async () => { await CustomApi.listCustomApis(mockCtx, {}); expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { method: 'GET', }); }); + + it('35. invokes customApi.deleteCustomApi correctly', async () => { + await CustomApi.deleteCustomApi(mockCtx, { + api: 'myCustomApi', + url: 'https://example.com/custom', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { + method: 'DELETE', + query: { + api: 'myCustomApi', + url: 'https://example.com/custom', + }, + }); + }); }); describe('Diffbot Error Handlers', () => { From 78ba94a74c8d2d77d13c00fc3888315b7f5bfbbf Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Mon, 24 Aug 2026 02:53:39 +0530 Subject: [PATCH 13/13] test(diffbot): clean database mock in tests and compact query parameters --- packages/diffbot/api.test.ts | 2 +- packages/diffbot/client.ts | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts index 4caafe989..9c4931a01 100644 --- a/packages/diffbot/api.test.ts +++ b/packages/diffbot/api.test.ts @@ -350,7 +350,7 @@ describe('Diffbot Endpoint Handlers — All 35 Operations Request Mapping', () = key: 'test_token', authType: 'api_key' as const, options: { key: 'test_token' }, - database: {}, + database: undefined, $getAccountId: () => 'acc_test', } as unknown as Parameters[0]; diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts index bce9bef9c..3c2745e9d 100644 --- a/packages/diffbot/client.ts +++ b/packages/diffbot/client.ts @@ -29,11 +29,22 @@ export type DiffbotRequestOptions = { timeout?: number; }; +function compactQuery( + query: Record, +): Record { + const compacted: Record = {}; + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) { + compacted[key] = value; + } + } + return compacted; +} + /** * Make a request to the Diffbot API. * - * Diffbot authenticates via `?token=` as a query parameter — - * NOT via an Authorization header. The token is injected automatically here. + * Diffbot authenticates via `?token=` as a query parameter. * * @param endpoint - The API endpoint path (e.g. 'article', 'dql', 'enhance') * @param token - The Diffbot API key @@ -51,7 +62,7 @@ export async function makeDiffbotRequest( const { method = 'GET', body, - query, + query = {}, headers, useKgBase = false, customBase, @@ -74,11 +85,10 @@ export async function makeDiffbotRequest( }, }; - const queryWithToken: Record = - { - ...query, - token, - }; + const queryWithToken = compactQuery({ + ...query, + token, + }); const requestOptions: ApiRequestOptions = { method,