diff --git a/packages/corsair/core/constants.ts b/packages/corsair/core/constants.ts index cec7a83df..b5dca6fd6 100644 --- a/packages/corsair/core/constants.ts +++ b/packages/corsair/core/constants.ts @@ -92,6 +92,7 @@ export const BaseProviders = [ 'datadog', 'deepseek', 'devinmcp', + 'diffbot', 'digitalocean', 'discord', 'dockerhub', @@ -279,6 +280,7 @@ export const ProviderDisplayNames = { datadog: 'Datadog', deepseek: 'DeepSeek', devinmcp: 'Devin MCP', + diffbot: 'Diffbot', digitalocean: 'DigitalOcean', discord: 'Discord', dockerhub: 'Docker Hub', @@ -473,6 +475,7 @@ export type AllProviders = | 'datadog' | 'deepseek' | 'devinmcp' + | 'diffbot' | 'digitalocean' | 'discord' | 'dockerhub' diff --git a/packages/diffbot/api.test.ts b/packages/diffbot/api.test.ts new file mode 100644 index 000000000..9c4931a01 --- /dev/null +++ b/packages/diffbot/api.test.ts @@ -0,0 +1,993 @@ +import * as clientModule from './client'; +import { + Account, + Bulk, + Crawl, + CustomApi, + Enhance, + Extract, + KgBulkEnhance, + Search, +} from './endpoints'; +import { + DiffbotEndpointInputSchemas, + DiffbotEndpointOutputSchemas, +} from './endpoints/types'; +import { errorHandlers } from './error-handlers'; +import { diffbot } from './index'; + +describe('Diffbot Input and Output Schemas', () => { + // Account + describe('account.getAccount', () => { + it('accepts empty input', () => { + const parsed = DiffbotEndpointInputSchemas.getAccount.parse({}); + expect(parsed).toEqual({}); + }); + + it('parses valid output', () => { + const parsed = DiffbotEndpointOutputSchemas.getAccount.parse({ + token: 'test_token', + name: 'Test User', + plan: 'kgfree', + planCalls: 10000, + status: 'active', + }); + expect(parsed.name).toBe('Test User'); + expect(parsed.planCalls).toBe(10000); + }); + }); + + // Extract + describe('extract.getArticle', () => { + it('accepts valid url and optional fields', () => { + const input = DiffbotEndpointInputSchemas.getArticle.parse({ + url: 'https://example.com/article', + fields: 'links,meta', + }); + expect(input.url).toBe('https://example.com/article'); + }); + + it('parses article response', () => { + const output = DiffbotEndpointOutputSchemas.getArticle.parse({ + objects: [ + { + type: 'article', + title: 'Test Article', + text: 'Body text', + }, + ], + }); + expect(output.objects[0]?.title).toBe('Test Article'); + }); + }); + + describe('extract.getProduct', () => { + it('accepts valid product url', () => { + const input = DiffbotEndpointInputSchemas.getProduct.parse({ + url: 'https://example.com/product', + }); + expect(input.url).toBe('https://example.com/product'); + }); + + it('parses product response', () => { + const output = DiffbotEndpointOutputSchemas.getProduct.parse({ + objects: [ + { + type: 'product', + title: 'Test Product', + offerPrice: '$99.00', + }, + ], + }); + expect(output.objects[0]?.offerPrice).toBe('$99.00'); + }); + }); + + describe('extract.getAnalyze', () => { + it('accepts url with fallback', () => { + const input = DiffbotEndpointInputSchemas.getAnalyze.parse({ + url: 'https://example.com/page', + fallback: 'article', + }); + expect(input.fallback).toBe('article'); + }); + + it('parses analyze response', () => { + const output = DiffbotEndpointOutputSchemas.getAnalyze.parse({ + type: 'article', + title: 'Detected Article', + }); + expect(output.type).toBe('article'); + }); + }); + + describe('extract.getImage', () => { + it('accepts url', () => { + const input = DiffbotEndpointInputSchemas.getImage.parse({ + url: 'https://example.com/image.png', + }); + expect(input.url).toBe('https://example.com/image.png'); + }); + + it('parses image response', () => { + const output = DiffbotEndpointOutputSchemas.getImage.parse({ + objects: [{ type: 'image', url: 'https://example.com/image.png' }], + }); + expect(output.objects[0]?.url).toBe('https://example.com/image.png'); + }); + }); + + describe('extract.getVideo', () => { + it('accepts video url', () => { + const input = DiffbotEndpointInputSchemas.getVideo.parse({ + url: 'https://example.com/video', + }); + expect(input.url).toBe('https://example.com/video'); + }); + + it('parses video response', () => { + const output = DiffbotEndpointOutputSchemas.getVideo.parse({ + objects: [{ type: 'video', duration: 120 }], + }); + expect(output.objects[0]?.duration).toBe(120); + }); + }); + + describe('extract.getDiscussion', () => { + it('accepts discussion url', () => { + const input = DiffbotEndpointInputSchemas.getDiscussion.parse({ + url: 'https://example.com/forum', + }); + expect(input.url).toBe('https://example.com/forum'); + }); + + it('parses discussion response', () => { + const output = DiffbotEndpointOutputSchemas.getDiscussion.parse({ + objects: [{ type: 'discussion', numPosts: 5 }], + }); + expect(output.objects[0]?.numPosts).toBe(5); + }); + }); + + describe('extract.getEvent', () => { + it('accepts event url', () => { + const input = DiffbotEndpointInputSchemas.getEvent.parse({ + url: 'https://example.com/event', + }); + expect(input.url).toBe('https://example.com/event'); + }); + + it('parses event response', () => { + const output = DiffbotEndpointOutputSchemas.getEvent.parse({ + objects: [{ type: 'event', startDate: '2026-09-01' }], + }); + expect(output.objects[0]?.startDate).toBe('2026-09-01'); + }); + }); + + describe('extract.extractList', () => { + it('accepts list url', () => { + const input = DiffbotEndpointInputSchemas.extractList.parse({ + url: 'https://example.com/list', + }); + expect(input.url).toBe('https://example.com/list'); + }); + + it('parses list response', () => { + const output = DiffbotEndpointOutputSchemas.extractList.parse({ + objects: [{ type: 'list', numItems: 10 }], + }); + expect(output.objects[0]?.numItems).toBe(10); + }); + }); + + describe('extract.extractJob', () => { + it('accepts job url', () => { + const input = DiffbotEndpointInputSchemas.extractJob.parse({ + url: 'https://example.com/job', + }); + expect(input.url).toBe('https://example.com/job'); + }); + + it('parses job response', () => { + const output = DiffbotEndpointOutputSchemas.extractJob.parse({ + objects: [{ type: 'job', title: 'Software Engineer' }], + }); + expect(output.objects[0]?.title).toBe('Software Engineer'); + }); + }); + + // Search + describe('search.search & search.searchCrawlData', () => { + it('accepts dql search query', () => { + const input = DiffbotEndpointInputSchemas.search.parse({ + query: 'name:"Diffbot"', + entityType: 'Organization', + }); + expect(input.query).toBe('name:"Diffbot"'); + }); + + it('parses dql search response', () => { + const output = DiffbotEndpointOutputSchemas.search.parse({ + hits: 1, + data: [{ name: 'Diffbot' }], + }); + expect(output.hits).toBe(1); + }); + + it('accepts crawl data search query', () => { + const input = DiffbotEndpointInputSchemas.searchCrawlData.parse({ + col: 'myCollection', + query: 'tech', + num: 10, + }); + expect(input.col).toBe('myCollection'); + }); + }); + + // Enhance + describe('enhance endpoints', () => { + it('accepts enhanceEntity input', () => { + const input = DiffbotEndpointInputSchemas.enhanceEntity.parse({ + name: 'Diffbot', + type: 'Organization', + }); + expect(input.name).toBe('Diffbot'); + }); + + it('accepts combineEntityProfiles input', () => { + const input = DiffbotEndpointInputSchemas.combineEntityProfiles.parse({ + name: 'John Doe', + employer: 'Acme', + }); + expect(input.name).toBe('John Doe'); + }); + + it('accepts resolveLostId input', () => { + const input = DiffbotEndpointInputSchemas.resolveLostId.parse({ + id: 'legacy-id-123', + }); + expect(input.id).toBe('legacy-id-123'); + }); + + it('accepts getKgCoverageReportById input', () => { + const input = DiffbotEndpointInputSchemas.getKgCoverageReportById.parse({ + reportId: 'rep_123', + }); + expect(input.reportId).toBe('rep_123'); + }); + }); + + // KG Bulk Enhance + describe('kgBulkEnhance endpoints', () => { + it('accepts createKgBulkEnhance input', () => { + const input = DiffbotEndpointInputSchemas.createKgBulkEnhance.parse({ + entities: [{ name: 'Company A' }, { name: 'Company B' }], + name: 'testJob', + }); + expect(input.entities.length).toBe(2); + }); + + it('accepts getBulkJobStatus input', () => { + const input = DiffbotEndpointInputSchemas.getBulkJobStatus.parse({ + bulkjobId: 'bulk_123', + }); + expect(input.bulkjobId).toBe('bulk_123'); + }); + + it('accepts getBulkSingleResult input', () => { + const input = DiffbotEndpointInputSchemas.getBulkSingleResult.parse({ + bulkjobId: 'bulk_123', + jobIndex: 0, + }); + expect(input.jobIndex).toBe(0); + }); + }); + + // Bulk Extract + describe('bulk extract endpoints', () => { + it('accepts createBulk input', () => { + const input = DiffbotEndpointInputSchemas.createBulk.parse({ + name: 'myBulk', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: ['https://example.com/1', 'https://example.com/2'], + }); + expect(input.urls.length).toBe(2); + }); + + it('accepts startBulk input', () => { + const input = DiffbotEndpointInputSchemas.startBulk.parse({ + name: 'myBulk', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + }); + expect(input.name).toBe('myBulk'); + }); + }); + + // Crawl + describe('crawl endpoints', () => { + it('accepts startCrawl input', () => { + const input = DiffbotEndpointInputSchemas.startCrawl.parse({ + name: 'crawl1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + }); + expect(input.name).toBe('crawl1'); + }); + + it('accepts manageCrawl input', () => { + const input = DiffbotEndpointInputSchemas.manageCrawl.parse({ + name: 'crawl1', + pause: 1, + }); + expect(input.pause).toBe(1); + }); + }); + + // Custom API + describe('customApi endpoints', () => { + it('accepts createCustomApi input', () => { + const input = DiffbotEndpointInputSchemas.createCustomApi.parse({ + api: 'custom1', + url: 'https://example.com', + }); + expect(input.api).toBe('custom1'); + }); + + it('accepts deleteCustomApi input', () => { + const input = DiffbotEndpointInputSchemas.deleteCustomApi.parse({ + api: 'custom1', + }); + expect(input.api).toBe('custom1'); + }); + }); +}); + +describe('Diffbot Endpoint Handlers — All 35 Operations Request Mapping', () => { + let makeRequestSpy: jest.SpyInstance; + const mockCtx = { + key: 'test_token', + authType: 'api_key' as const, + options: { key: 'test_token' }, + database: undefined, + $getAccountId: () => 'acc_test', + } as unknown as Parameters[0]; + + beforeEach(() => { + makeRequestSpy = jest + .spyOn(clientModule, 'makeDiffbotRequest') + .mockResolvedValue({ status: 200 } as never); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + // 1. Account (1 operation) + it('1. invokes account.getAccount correctly', async () => { + await Account.getAccount(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('account', 'test_token', { + method: 'GET', + }); + }); + + // 2. Extract (9 operations) + it('2. invokes extract.getArticle correctly', async () => { + await Extract.getArticle(mockCtx, { + url: 'https://example.com/article', + fields: 'meta,links', + timeout: 15000, + paging: 'false', + maxTags: 5, + naturalLanguage: 'en', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('article', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/article', + fields: 'meta,links', + timeout: 15000, + paging: 'false', + maxTags: 5, + naturalLanguage: 'en', + }, + }); + }); + + it('3. invokes extract.getProduct correctly', async () => { + await Extract.getProduct(mockCtx, { + url: 'https://example.com/product', + fields: 'brand,offers', + timeout: 20000, + discussion: 'false', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('product', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/product', + fields: 'brand,offers', + timeout: 20000, + discussion: 'false', + }, + }); + }); + + it('4. invokes extract.getAnalyze correctly', async () => { + await Extract.getAnalyze(mockCtx, { + url: 'https://example.com/unknown', + fallback: 'article', + discussion: 'false', + timeout: 10000, + fields: 'title', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('analyze', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/unknown', + fallback: 'article', + discussion: 'false', + timeout: 10000, + fields: 'title', + }, + }); + }); + + it('5. invokes extract.getImage correctly', async () => { + await Extract.getImage(mockCtx, { + url: 'https://example.com/image.jpg', + fields: 'xpath', + timeout: 12000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('image', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/image.jpg', + fields: 'xpath', + timeout: 12000, + }, + }); + }); + + it('6. invokes extract.getVideo correctly', async () => { + await Extract.getVideo(mockCtx, { + url: 'https://example.com/video.mp4', + fields: 'duration', + timeout: 12000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('video', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/video.mp4', + fields: 'duration', + timeout: 12000, + }, + }); + }); + + it('7. invokes extract.getDiscussion correctly', async () => { + await Extract.getDiscussion(mockCtx, { + url: 'https://example.com/forum', + fields: 'posts', + timeout: 18000, + maxTags: 10, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('discussion', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/forum', + fields: 'posts', + timeout: 18000, + maxTags: 10, + }, + }); + }); + + it('8. invokes extract.getEvent correctly', async () => { + await Extract.getEvent(mockCtx, { + url: 'https://example.com/event', + fields: 'venue', + timeout: 15000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('event', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/event', + fields: 'venue', + timeout: 15000, + }, + }); + }); + + it('9. invokes extract.extractList correctly', async () => { + await Extract.extractList(mockCtx, { + url: 'https://example.com/directory', + fields: 'items', + timeout: 25000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('list', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/directory', + fields: 'items', + timeout: 25000, + }, + }); + }); + + it('10. invokes extract.extractJob correctly', async () => { + await Extract.extractJob(mockCtx, { + url: 'https://example.com/careers/job1', + fields: 'compensation', + timeout: 20000, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('job', 'test_token', { + method: 'GET', + query: { + url: 'https://example.com/careers/job1', + fields: 'compensation', + timeout: 20000, + }, + }); + }); + + // 3. Search (2 operations) + it('11. invokes search.search with DQL routing to KG base', async () => { + await Search.search(mockCtx, { + query: 'name:"Diffbot"', + entityType: 'Organization', + queryType: 'query', + size: 20, + from: 0, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('dql', 'test_token', { + method: 'GET', + useKgBase: true, + query: { + query: 'type:Organization name:"Diffbot"', + type: 'query', + size: 20, + from: 0, + col: undefined, + }, + }); + }); + + it('12. invokes search.searchCrawlData correctly', async () => { + await Search.searchCrawlData(mockCtx, { + col: 'myCrawlCollection', + query: 'tech', + num: 15, + start: 5, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('search', 'test_token', { + method: 'GET', + query: { + col: 'myCrawlCollection', + query: 'tech', + num: 15, + start: 5, + }, + }); + }); + + // 4. Enhance (4 operations) + it('13. invokes enhance.enhanceEntity correctly', async () => { + await Enhance.enhanceEntity(mockCtx, { + name: 'Diffbot Technologies', + type: 'Organization', + url: 'https://diffbot.com', + size: 1, + refresh: true, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance', 'test_token', { + method: 'GET', + useKgBase: true, + query: { + name: 'Diffbot Technologies', + type: 'Organization', + url: 'https://diffbot.com', + size: 1, + refresh: true, + email: undefined, + employer: undefined, + phone: undefined, + location: undefined, + }, + }); + }); + + it('14. invokes enhance.combineEntityProfiles correctly', async () => { + await Enhance.combineEntityProfiles(mockCtx, { + name: 'Mike Tung', + type: 'Person', + employer: 'Diffbot', + email: 'mike@diffbot.com', + url: 'https://linkedin.com/in/miketung', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/combine', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: { + name: 'Mike Tung', + type: 'Person', + employer: 'Diffbot', + email: 'mike@diffbot.com', + url: 'https://linkedin.com/in/miketung', + }, + }, + ); + }); + + it('15. invokes enhance.resolveLostId correctly', async () => { + await Enhance.resolveLostId(mockCtx, { id: 'OLD_KG_ID_999' }); + expect(makeRequestSpy).toHaveBeenCalledWith('dql', 'test_token', { + method: 'GET', + useKgBase: true, + query: { + query: 'id:"OLD_KG_ID_999"', + size: 1, + }, + }); + }); + + it('16. invokes enhance.getKgCoverageReportById correctly', async () => { + await Enhance.getKgCoverageReportById(mockCtx, { reportId: 'rep_abc123' }); + expect(makeRequestSpy).toHaveBeenCalledWith('report', 'test_token', { + method: 'GET', + useKgBase: true, + query: { reportId: 'rep_abc123' }, + }); + + await Enhance.getKgCoverageReportById(mockCtx, { + reportId: 'rep_abc123', + bulkjobId: 'bulk_456', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/report/bulk_456/rep_abc123', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: {}, + }, + ); + }); + + // 5. KG Bulk Enhance (8 operations) + it('17. invokes kgBulkEnhance.createKgBulkEnhance correctly', async () => { + await KgBulkEnhance.createKgBulkEnhance(mockCtx, { + entities: [{ name: 'Diffbot' }, { name: 'Anthropic' }], + name: 'enrichJob1', + notifyEmail: 'dev@example.com', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance/bulk', 'test_token', { + method: 'POST', + useKgBase: true, + body: [{ name: 'Diffbot' }, { name: 'Anthropic' }], + query: { + name: 'enrichJob1', + notifyEmail: 'dev@example.com', + }, + }); + }); + + it('18. invokes kgBulkEnhance.getBulkJobStatus correctly', async () => { + await KgBulkEnhance.getBulkJobStatus(mockCtx, { bulkjobId: 'bj_100' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/status', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('19. invokes kgBulkEnhance.listBulkJobsStatusForToken correctly', async () => { + await KgBulkEnhance.listBulkJobsStatusForToken(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('enhance/bulk', 'test_token', { + method: 'GET', + useKgBase: true, + }); + }); + + it('20. invokes kgBulkEnhance.getBulkResults correctly', async () => { + await KgBulkEnhance.getBulkResults(mockCtx, { + bulkjobId: 'bj_100', + format: 'json', + head: 50, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100', + 'test_token', + { + method: 'GET', + useKgBase: true, + query: { + format: 'json', + head: 50, + }, + }, + ); + }); + + it('21. invokes kgBulkEnhance.downloadBulkResults correctly', async () => { + await KgBulkEnhance.downloadBulkResults(mockCtx, { + bulkjobId: 'bj_100', + format: 'jsonl', + filter: 'importance>0.5', + fields: 'name,location', + head: 100, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100', + 'test_token', + { + method: 'POST', + useKgBase: true, + query: { + format: 'jsonl', + filter: 'importance>0.5', + fields: 'name,location', + head: 100, + }, + }, + ); + }); + + it('22. invokes kgBulkEnhance.getBulkSingleResult correctly', async () => { + await KgBulkEnhance.getBulkSingleResult(mockCtx, { + bulkjobId: 'bj_100', + jobIndex: 3, + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/3', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('23. invokes kgBulkEnhance.stopKgBulkJobById correctly', async () => { + await KgBulkEnhance.stopKgBulkJobById(mockCtx, { bulkjobId: 'bj_100' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/stop', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + it('24. invokes kgBulkEnhance.deleteKgEnhanceBulkjob correctly', async () => { + await KgBulkEnhance.deleteKgEnhanceBulkjob(mockCtx, { + bulkjobId: 'bj_100', + }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'enhance/bulk/bj_100/delete', + 'test_token', + { + method: 'GET', + useKgBase: true, + }, + ); + }); + + // 6. Bulk Extract (5 operations) + it('25. invokes bulk.createBulk correctly', async () => { + await Bulk.createBulk(mockCtx, { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: ['https://example.com/1', 'https://example.com/2'], + notifyEmail: 'notify@example.com', + maxRounds: 3, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'POST', + body: 'https://example.com/1\nhttps://example.com/2', + query: { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + notifyEmail: 'notify@example.com', + maxRounds: 3, + }, + }); + }); + + it('26. invokes bulk.startBulk correctly', async () => { + await Bulk.startBulk(mockCtx, { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + notifyEmail: 'notify@example.com', + maxRounds: 2, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', + query: { + name: 'jobExtract', + apiUrl: 'https://api.diffbot.com/v3/article', + urls: 'https://example.com/1 https://example.com/2', + notifyEmail: 'notify@example.com', + maxRounds: 2, + }, + }); + }); + + it('27. invokes bulk.stopBulkJob correctly', async () => { + await Bulk.stopBulkJob(mockCtx, { name: 'jobExtract' }); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', + query: { + name: 'jobExtract', + pause: 1, + }, + }); + }); + + it('28. invokes bulk.getBulkData correctly', async () => { + await Bulk.getBulkData(mockCtx, { name: 'jobExtract', format: 'csv' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'bulk/download/test_token-jobExtract.csv', + 'test_token', + { + method: 'GET', + }, + ); + }); + + it('29. invokes bulk.listBulkJobs correctly', async () => { + await Bulk.listBulkJobs(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('bulk', 'test_token', { + method: 'GET', + }); + }); + + // 7. Crawl (3 operations) + it('30. invokes crawl.startCrawl correctly', async () => { + await Crawl.startCrawl(mockCtx, { + name: 'crawlJob1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + maxHops: 2, + maxRounds: 1, + maxTags: 5, + crawlSubdomains: 1, + notifyEmail: 'crawl@example.com', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('crawl', 'test_token', { + method: 'POST', + query: { + name: 'crawlJob1', + seeds: 'https://example.com', + apiUrl: 'https://api.diffbot.com/v3/article', + maxHops: 2, + maxRounds: 1, + maxTags: 5, + crawlSubdomains: 1, + notifyEmail: 'crawl@example.com', + }, + }); + }); + + it('31. invokes crawl.manageCrawl correctly', async () => { + await Crawl.manageCrawl(mockCtx, { + name: 'crawlJob1', + pause: 1, + restart: 0, + delete: 0, + roundProxy: 1, + maxRounds: 5, + maxHops: 3, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('crawl', 'test_token', { + method: 'GET', + query: { + name: 'crawlJob1', + pause: 1, + restart: 0, + delete: 0, + roundProxy: 1, + maxRounds: 5, + maxHops: 3, + }, + }); + }); + + it('32. invokes crawl.getCrawlData correctly', async () => { + await Crawl.getCrawlData(mockCtx, { name: 'crawlJob1', format: 'json' }); + expect(makeRequestSpy).toHaveBeenCalledWith( + 'crawl/download/test_token-crawlJob1.json', + 'test_token', + { + method: 'GET', + }, + ); + }); + + // 8. Custom API (3 operations) + it('33. invokes customApi.createCustomApi correctly', async () => { + await CustomApi.createCustomApi(mockCtx, { + api: 'myCustomApi', + url: 'https://example.com/custom', + pattern: 'https://example.com/*', + rules: { selector: '.article-body' }, + }); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { + method: 'POST', + body: { selector: '.article-body' }, + query: { + api: 'myCustomApi', + url: 'https://example.com/custom', + pattern: 'https://example.com/*', + }, + }); + }); + + it('34. invokes customApi.listCustomApis correctly', async () => { + await CustomApi.listCustomApis(mockCtx, {}); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { + method: 'GET', + }); + }); + + it('35. invokes customApi.deleteCustomApi correctly', async () => { + await CustomApi.deleteCustomApi(mockCtx, { + api: 'myCustomApi', + url: 'https://example.com/custom', + }); + expect(makeRequestSpy).toHaveBeenCalledWith('custom', 'test_token', { + method: 'DELETE', + query: { + api: 'myCustomApi', + url: 'https://example.com/custom', + }, + }); + }); +}); + +describe('Diffbot Error Handlers', () => { + it('handles rate limit 429 errors and specifies retries', async () => { + const error = Object.assign(new Error('Rate limit exceeded'), { + status: 429, + retryAfter: 1500, + }); + expect(errorHandlers.RATE_LIMIT_ERROR.match(error)).toBe(true); + const res = await errorHandlers.RATE_LIMIT_ERROR.handler(error); + expect(res.maxRetries).toBe(5); + expect(res.headersRetryAfterMs).toBe(1500); + }); + + it('handles 401 unauthorized errors with 0 retries', async () => { + const error = Object.assign(new Error('Invalid token'), { status: 401 }); + expect(errorHandlers.AUTH_ERROR.match(error)).toBe(true); + const res = await errorHandlers.AUTH_ERROR.handler(error); + expect(res.maxRetries).toBe(0); + }); + + it('handles 500 server errors', async () => { + const error = Object.assign(new Error('Internal server error'), { + status: 500, + }); + expect(errorHandlers.SERVER_ERROR.match(error)).toBe(true); + const res = await errorHandlers.SERVER_ERROR.handler(error); + expect(res.maxRetries).toBe(2); + }); +}); + +describe('Diffbot Plugin Instance', () => { + it('initializes diffbot plugin with default options', () => { + const instance = diffbot({ key: 'diffbot_test_key' }); + expect(instance.id).toBe('diffbot'); + expect(instance.schema).toBeDefined(); + expect(instance.endpoints).toBeDefined(); + expect(Object.keys(instance.endpoints ?? {}).length).toBe(8); + }); +}); diff --git a/packages/diffbot/client.ts b/packages/diffbot/client.ts new file mode 100644 index 000000000..3c2745e9d --- /dev/null +++ b/packages/diffbot/client.ts @@ -0,0 +1,118 @@ +import type { ApiRequestOptions, OpenAPIConfig } from 'corsair/http'; +import { ApiError, request } from 'corsair/http'; + +export class DiffbotAPIError extends Error { + constructor( + message: string, + public readonly code?: string, + public readonly status?: number, + public readonly retryAfter?: number, + ) { + super(message); + this.name = 'DiffbotAPIError'; + } +} + +// Diffbot API v3 base URL (extract, crawl, bulk, custom, account) +const DIFFBOT_API_BASE = 'https://api.diffbot.com/v3'; + +// Diffbot Knowledge Graph base URL (DQL, enhance, kg-bulk) +const DIFFBOT_KG_BASE = 'https://kg.diffbot.com/kg/v3'; + +export type DiffbotRequestOptions = { + method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; + body?: unknown; + query?: Record; + headers?: Record; + useKgBase?: boolean; + customBase?: string; + timeout?: number; +}; + +function compactQuery( + query: Record, +): Record { + const compacted: Record = {}; + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) { + compacted[key] = value; + } + } + return compacted; +} + +/** + * Make a request to the Diffbot API. + * + * Diffbot authenticates via `?token=` as a query parameter. + * + * @param endpoint - The API endpoint path (e.g. 'article', 'dql', 'enhance') + * @param token - The Diffbot API key + * @param options - Request options including method, body, query params + */ +export async function makeDiffbotRequest( + endpoint: string, + token: string, + options: DiffbotRequestOptions = {}, +): Promise { + if (!token?.trim()) { + throw new Error('Diffbot API token is required'); + } + + const { + method = 'GET', + body, + query = {}, + headers, + useKgBase = false, + customBase, + timeout, + } = options; + + const baseUrl = + customBase ?? (useKgBase ? DIFFBOT_KG_BASE : DIFFBOT_API_BASE); + + const config: OpenAPIConfig = { + BASE: baseUrl, + VERSION: '3', + WITH_CREDENTIALS: false, + CREDENTIALS: 'omit', + TOKEN: undefined, + TIMEOUT: timeout, + HEADERS: { + Accept: 'application/json', + ...headers, + }, + }; + + const queryWithToken = compactQuery({ + ...query, + token, + }); + + const requestOptions: ApiRequestOptions = { + method, + url: endpoint, + body: + method === 'POST' || + method === 'PUT' || + method === 'PATCH' || + method === 'DELETE' + ? body + : undefined, + mediaType: typeof body === 'string' ? 'text/plain' : 'application/json', + query: queryWithToken, + }; + + try { + return await request(config, requestOptions); + } catch (error) { + if (error instanceof ApiError) { + throw error; + } + if (error instanceof Error) { + throw new DiffbotAPIError(error.message); + } + throw new DiffbotAPIError('Unknown error occurred'); + } +} diff --git a/packages/diffbot/endpoints/account.ts b/packages/diffbot/endpoints/account.ts new file mode 100644 index 000000000..598e3121d --- /dev/null +++ b/packages/diffbot/endpoints/account.ts @@ -0,0 +1,17 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const getAccount: DiffbotEndpoints['getAccount'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('account', ctx.key, { + method: 'GET', + }); + + await logEventFromContext(ctx, 'diffbot.account.getAccount', {}, 'completed'); + return response; +}; diff --git a/packages/diffbot/endpoints/bulk.ts b/packages/diffbot/endpoints/bulk.ts new file mode 100644 index 000000000..cecd8e513 --- /dev/null +++ b/packages/diffbot/endpoints/bulk.ts @@ -0,0 +1,113 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createBulk: DiffbotEndpoints['createBulk'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'POST', + body: input.urls.join('\n'), + query: { + name: input.name, + apiUrl: input.apiUrl, + notifyEmail: input.notifyEmail, + maxRounds: input.maxRounds, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.createBulk', + { name: input.name, count: input.urls.length }, + 'completed', + ); + return response; +}; + +export const startBulk: DiffbotEndpoints['startBulk'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + query: { + name: input.name, + apiUrl: input.apiUrl, + urls: input.urls, + notifyEmail: input.notifyEmail, + maxRounds: input.maxRounds, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.startBulk', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const stopBulkJob: DiffbotEndpoints['stopBulkJob'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + query: { + name: input.name, + pause: 1, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.bulk.stopBulkJob', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const getBulkData: DiffbotEndpoints['getBulkData'] = async ( + ctx, + input, +) => { + const format = input.format ?? 'json'; + const response = await makeDiffbotRequest< + Awaited> + >( + `bulk/download/${encodeURIComponent(ctx.key)}-${encodeURIComponent(input.name)}.${format}`, + ctx.key, + { + method: 'GET', + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.bulk.getBulkData', + { name: input.name, format }, + 'completed', + ); + return response; +}; + +export const listBulkJobs: DiffbotEndpoints['listBulkJobs'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('bulk', ctx.key, { + method: 'GET', + }); + + await logEventFromContext(ctx, 'diffbot.bulk.listBulkJobs', {}, 'completed'); + return response; +}; diff --git a/packages/diffbot/endpoints/crawl.ts b/packages/diffbot/endpoints/crawl.ts new file mode 100644 index 000000000..4ea87b5da --- /dev/null +++ b/packages/diffbot/endpoints/crawl.ts @@ -0,0 +1,84 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const startCrawl: DiffbotEndpoints['startCrawl'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('crawl', ctx.key, { + method: 'POST', + query: { + name: input.name, + seeds: input.seeds, + apiUrl: input.apiUrl, + maxHops: input.maxHops, + maxRounds: input.maxRounds, + maxTags: input.maxTags, + crawlSubdomains: input.crawlSubdomains, + notifyEmail: input.notifyEmail, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.crawl.startCrawl', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const manageCrawl: DiffbotEndpoints['manageCrawl'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('crawl', ctx.key, { + method: 'GET', + query: { + name: input.name, + pause: input.pause, + restart: input.restart, + delete: input.delete, + roundProxy: input.roundProxy, + maxRounds: input.maxRounds, + maxHops: input.maxHops, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.crawl.manageCrawl', + { name: input.name }, + 'completed', + ); + return response; +}; + +export const getCrawlData: DiffbotEndpoints['getCrawlData'] = async ( + ctx, + input, +) => { + const format = input.format ?? 'json'; + const response = await makeDiffbotRequest< + Awaited> + >( + `crawl/download/${encodeURIComponent(ctx.key)}-${encodeURIComponent(input.name)}.${format}`, + ctx.key, + { + method: 'GET', + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.crawl.getCrawlData', + { name: input.name, format }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/custom-api.ts b/packages/diffbot/endpoints/custom-api.ts new file mode 100644 index 000000000..075fa8a0f --- /dev/null +++ b/packages/diffbot/endpoints/custom-api.ts @@ -0,0 +1,70 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createCustomApi: DiffbotEndpoints['createCustomApi'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'POST', + body: input.rules, + query: { + api: input.api, + url: input.url, + pattern: input.pattern, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.createCustomApi', + { api: input.api, url: input.url }, + 'completed', + ); + return response; +}; + +export const listCustomApis: DiffbotEndpoints['listCustomApis'] = async ( + ctx, + _input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'GET', + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.listCustomApis', + {}, + 'completed', + ); + return response; +}; + +export const deleteCustomApi: DiffbotEndpoints['deleteCustomApi'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('custom', ctx.key, { + method: 'DELETE', + query: { + api: input.api, + url: input.url, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.customApi.deleteCustomApi', + { api: input.api }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/enhance.ts b/packages/diffbot/endpoints/enhance.ts new file mode 100644 index 000000000..e9c4cff48 --- /dev/null +++ b/packages/diffbot/endpoints/enhance.ts @@ -0,0 +1,106 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const enhanceEntity: DiffbotEndpoints['enhanceEntity'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + name: input.name, + type: input.type, + email: input.email, + employer: input.employer, + url: input.url, + phone: input.phone, + location: input.location, + size: input.size, + refresh: input.refresh, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.enhanceEntity', + { name: input.name, type: input.type }, + 'completed', + ); + return response; +}; + +export const combineEntityProfiles: DiffbotEndpoints['combineEntityProfiles'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/combine', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + name: input.name, + type: input.type, + email: input.email, + employer: input.employer, + url: input.url, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.combineEntityProfiles', + { name: input.name, type: input.type }, + 'completed', + ); + return response; + }; + +export const resolveLostId: DiffbotEndpoints['resolveLostId'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('dql', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + query: `id:"${input.id}"`, + size: 1, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.resolveLostId', + { id: input.id }, + 'completed', + ); + return response; +}; + +export const getKgCoverageReportById: DiffbotEndpoints['getKgCoverageReportById'] = + async (ctx, input) => { + const endpoint = input.bulkjobId + ? `enhance/bulk/report/${encodeURIComponent(input.bulkjobId)}/${encodeURIComponent(input.reportId)}` + : 'report'; + + const response = await makeDiffbotRequest< + Awaited> + >(endpoint, ctx.key, { + method: 'GET', + useKgBase: true, + query: input.bulkjobId ? {} : { reportId: input.reportId }, + }); + + await logEventFromContext( + ctx, + 'diffbot.enhance.getKgCoverageReportById', + { reportId: input.reportId, bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; + }; diff --git a/packages/diffbot/endpoints/extract.ts b/packages/diffbot/endpoints/extract.ts new file mode 100644 index 000000000..78e9fe105 --- /dev/null +++ b/packages/diffbot/endpoints/extract.ts @@ -0,0 +1,217 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const getArticle: DiffbotEndpoints['getArticle'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('article', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + paging: input.paging, + maxTags: input.maxTags, + naturalLanguage: input.naturalLanguage, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getArticle', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getProduct: DiffbotEndpoints['getProduct'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('product', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + discussion: input.discussion, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getProduct', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getAnalyze: DiffbotEndpoints['getAnalyze'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('analyze', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + fallback: input.fallback, + discussion: input.discussion, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getAnalyze', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getImage: DiffbotEndpoints['getImage'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('image', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getImage', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getVideo: DiffbotEndpoints['getVideo'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('video', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getVideo', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getDiscussion: DiffbotEndpoints['getDiscussion'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('discussion', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + maxTags: input.maxTags, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getDiscussion', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const getEvent: DiffbotEndpoints['getEvent'] = async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('event', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.getEvent', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const extractList: DiffbotEndpoints['extractList'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('list', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.extractList', + { url: input.url }, + 'completed', + ); + return response; +}; + +export const extractJob: DiffbotEndpoints['extractJob'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('job', ctx.key, { + method: 'GET', + query: { + url: input.url, + fields: input.fields, + timeout: input.timeout, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.extract.extractJob', + { url: input.url }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/index.ts b/packages/diffbot/endpoints/index.ts new file mode 100644 index 000000000..8f1b23979 --- /dev/null +++ b/packages/diffbot/endpoints/index.ts @@ -0,0 +1,8 @@ +export * as Account from './account'; +export * as Bulk from './bulk'; +export * as Crawl from './crawl'; +export * as CustomApi from './custom-api'; +export * as Enhance from './enhance'; +export * as Extract from './extract'; +export * as KgBulkEnhance from './kg-bulk-enhance'; +export * as Search from './search'; diff --git a/packages/diffbot/endpoints/kg-bulk-enhance.ts b/packages/diffbot/endpoints/kg-bulk-enhance.ts new file mode 100644 index 000000000..19a84e517 --- /dev/null +++ b/packages/diffbot/endpoints/kg-bulk-enhance.ts @@ -0,0 +1,172 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const createKgBulkEnhance: DiffbotEndpoints['createKgBulkEnhance'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/bulk', ctx.key, { + method: 'POST', + useKgBase: true, + body: input.entities, + query: { + notifyEmail: input.notifyEmail, + name: input.name, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.createKgBulkEnhance', + { name: input.name, count: input.entities.length }, + 'completed', + ); + return response; + }; + +export const getBulkJobStatus: DiffbotEndpoints['getBulkJobStatus'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/status`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkJobStatus', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; +}; + +export const listBulkJobsStatusForToken: DiffbotEndpoints['listBulkJobsStatusForToken'] = + async (ctx, _input) => { + const response = await makeDiffbotRequest< + Awaited> + >('enhance/bulk', ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.listBulkJobsStatusForToken', + {}, + 'completed', + ); + return response; + }; + +export const getBulkResults: DiffbotEndpoints['getBulkResults'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}`, ctx.key, { + method: 'GET', + useKgBase: true, + query: { + format: input.format, + head: input.head, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkResults', + { bulkjobId: input.bulkjobId, format: input.format }, + 'completed', + ); + return response; +}; + +export const downloadBulkResults: DiffbotEndpoints['downloadBulkResults'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}`, ctx.key, { + method: 'POST', + useKgBase: true, + query: { + format: input.format, + filter: input.filter, + fields: input.fields, + head: input.head, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.downloadBulkResults', + { bulkjobId: input.bulkjobId, format: input.format }, + 'completed', + ); + return response; + }; + +export const getBulkSingleResult: DiffbotEndpoints['getBulkSingleResult'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >( + `enhance/bulk/${encodeURIComponent(input.bulkjobId)}/${input.jobIndex}`, + ctx.key, + { + method: 'GET', + useKgBase: true, + }, + ); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.getBulkSingleResult', + { bulkjobId: input.bulkjobId, jobIndex: input.jobIndex }, + 'completed', + ); + return response; + }; + +export const stopKgBulkJobById: DiffbotEndpoints['stopKgBulkJobById'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/stop`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.stopKgBulkJobById', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; +}; + +export const deleteKgEnhanceBulkjob: DiffbotEndpoints['deleteKgEnhanceBulkjob'] = + async (ctx, input) => { + const response = await makeDiffbotRequest< + Awaited> + >(`enhance/bulk/${encodeURIComponent(input.bulkjobId)}/delete`, ctx.key, { + method: 'GET', + useKgBase: true, + }); + + await logEventFromContext( + ctx, + 'diffbot.kgBulkEnhance.deleteKgEnhanceBulkjob', + { bulkjobId: input.bulkjobId }, + 'completed', + ); + return response; + }; diff --git a/packages/diffbot/endpoints/search.ts b/packages/diffbot/endpoints/search.ts new file mode 100644 index 000000000..7dc1406b6 --- /dev/null +++ b/packages/diffbot/endpoints/search.ts @@ -0,0 +1,56 @@ +import { logEventFromContext } from 'corsair/core'; +import { makeDiffbotRequest } from '../client'; +import type { DiffbotEndpoints } from '../index'; + +export const search: DiffbotEndpoints['search'] = async (ctx, input) => { + const dqlQuery = input.entityType + ? `type:${input.entityType} ${input.query}` + : input.query; + + const response = await makeDiffbotRequest< + Awaited> + >('dql', ctx.key, { + method: 'GET', + useKgBase: true, + query: { + query: dqlQuery, + type: input.queryType, + size: input.size, + from: input.from, + col: input.col, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.search.search', + { query: input.query, entityType: input.entityType }, + 'completed', + ); + return response; +}; + +export const searchCrawlData: DiffbotEndpoints['searchCrawlData'] = async ( + ctx, + input, +) => { + const response = await makeDiffbotRequest< + Awaited> + >('search', ctx.key, { + method: 'GET', + query: { + col: input.col, + query: input.query, + num: input.num, + start: input.start, + }, + }); + + await logEventFromContext( + ctx, + 'diffbot.search.searchCrawlData', + { col: input.col, query: input.query }, + 'completed', + ); + return response; +}; diff --git a/packages/diffbot/endpoints/types.ts b/packages/diffbot/endpoints/types.ts new file mode 100644 index 000000000..1088eaaad --- /dev/null +++ b/packages/diffbot/endpoints/types.ts @@ -0,0 +1,1262 @@ +import { z } from 'zod'; + +// --------------------------------------------------------------------------- +// Shared sub-schemas +// --------------------------------------------------------------------------- + +export const DiffbotImageItemSchema = z + .object({ + url: z.string().optional(), + title: z.string().optional(), + width: z.number().optional(), + height: z.number().optional(), + naturalWidth: z.number().optional(), + naturalHeight: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), + }) + .passthrough(); + +export const DiffbotTagSchema = z + .object({ + id: z.number().optional(), + label: z.string(), + uri: z.string().optional(), + types: z.array(z.string()).optional(), + score: z.number().optional(), + count: z.number().optional(), + prevalence: z.number().optional(), + rdfTypes: z.array(z.string()).optional(), + }) + .passthrough(); + +export const DiffbotRequestMetaSchema = z + .object({ + pageUrl: z.string().optional(), + api: z.string().optional(), + version: z.number().optional(), + }) + .passthrough() + .optional(); + +// --------------------------------------------------------------------------- +// 1. Account +// --------------------------------------------------------------------------- + +export const GetAccountInputSchema = z.object({}); +export type GetAccountInput = z.infer; + +export const GetAccountResponseSchema = z + .object({ + token: z.string().optional(), + name: z.string().optional(), + email: z.string().optional(), + plan: z.string().optional(), + planStart: z.string().optional(), + planCalls: z.number().optional(), + apiCalls: z.number().optional(), + status: z.string().optional(), + }) + .passthrough(); + +export type GetAccountResponse = z.infer; + +// --------------------------------------------------------------------------- +// 2. Extract APIs (9 operations) +// --------------------------------------------------------------------------- + +// 2.1 Get Article Data +export const GetArticleInputSchema = z.object({ + url: z.string().describe('The URL of the article to extract'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields (e.g. "links,meta")'), + timeout: z + .number() + .optional() + .describe('Timeout in milliseconds (default 30000)'), + paging: z + .enum(['false', 'true']) + .optional() + .describe('Set to "false" to disable automatic pagination following'), + maxTags: z.number().optional().describe('Maximum number of tags to return'), + naturalLanguage: z.string().optional().describe('Language hint for NLP'), +}); +export type GetArticleInput = z.infer; + +export const GetArticleResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('article').optional(), + title: z.string().optional(), + text: z.string().optional(), + html: z.string().optional(), + date: z.string().optional(), + estimatedDate: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + siteName: z.string().optional(), + pageUrl: z.string().optional(), + resolvedPageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + numPages: z.number().optional(), + nextPage: z.string().optional(), + nextPages: z.array(z.string()).optional(), + images: z.array(DiffbotImageItemSchema).optional(), + videos: z.array(z.record(z.string(), z.unknown())).optional(), + tags: z.array(DiffbotTagSchema).optional(), + links: z.array(z.string()).optional(), + breadcrumb: z + .array( + z.object({ + link: z.string().optional(), + name: z.string().optional(), + }), + ) + .optional(), + publisherRegion: z.string().optional(), + publisherCountry: z.string().optional(), + sentiment: z.number().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetArticleResponse = z.infer; + +// 2.2 Get Product Data +export const GetProductInputSchema = z.object({ + url: z.string().describe('The URL of the product page to extract'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields'), + timeout: z + .number() + .optional() + .describe('Timeout in milliseconds (default 30000)'), + discussion: z + .enum(['false', 'true']) + .optional() + .describe('Set to "false" to disable review/discussion extraction'), +}); +export type GetProductInput = z.infer; + +export const GetProductResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('product').optional(), + title: z.string().optional(), + text: z.string().optional(), + brand: z.string().optional(), + offerPrice: z.string().optional(), + offerPriceDetails: z + .object({ + amount: z.number().optional(), + symbol: z.string().optional(), + text: z.string().optional(), + }) + .passthrough() + .optional(), + regularPrice: z.string().optional(), + saveAmount: z.string().optional(), + shippingAmount: z.string().optional(), + availability: z.boolean().optional(), + sku: z.string().optional(), + mpn: z.string().optional(), + upc: z.string().optional(), + isbn: z.string().optional(), + images: z.array(DiffbotImageItemSchema).optional(), + offers: z.array(z.record(z.string(), z.unknown())).optional(), + colors: z.array(z.string()).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + tags: z.array(DiffbotTagSchema).optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetProductResponse = z.infer; + +// 2.3 Analyze (Auto-detect page type) +export const GetAnalyzeInputSchema = z.object({ + url: z + .string() + .describe('The URL to analyze — Diffbot auto-detects the page type'), + fields: z + .string() + .optional() + .describe('Comma-separated list of optional fields'), + timeout: z.number().optional().describe('Timeout in milliseconds'), + fallback: z + .string() + .optional() + .describe( + 'API to fall back to if page type cannot be detected (e.g. "article")', + ), + discussion: z + .enum(['false', 'true']) + .optional() + .describe('Set to "false" to disable comment extraction'), +}); +export type GetAnalyzeInput = z.infer; + +export const GetAnalyzeResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + type: z + .string() + .optional() + .describe('Detected page type (article, product, discussion, etc.)'), + humanLanguage: z.string().optional(), + title: z.string().optional(), + objects: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type GetAnalyzeResponse = z.infer; + +// 2.4 Get Image Data +export const GetImageInputSchema = z.object({ + url: z.string().describe('The URL of the page or image to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetImageInput = z.infer; + +export const GetImageResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('image').optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetImageResponse = z.infer; + +// 2.5 Get Video Data +export const GetVideoInputSchema = z.object({ + url: z.string().describe('The URL of the video page to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetVideoInput = z.infer; + +export const GetVideoResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('video').optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + duration: z.number().optional(), + viewCount: z.number().optional(), + uploadDate: z.string().optional(), + author: z.string().optional(), + embedUrl: z.string().optional(), + html: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetVideoResponse = z.infer; + +// 2.6 Get Discussion Thread +export const GetDiscussionInputSchema = z.object({ + url: z.string().describe('The URL of the discussion / forum / comment page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), + maxTags: z.number().optional().describe('Max tags to return'), +}); +export type GetDiscussionInput = z.infer; + +export const GetDiscussionResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('discussion').optional(), + title: z.string().optional(), + text: z.string().optional(), + numPosts: z.number().optional(), + numParticipants: z.number().optional(), + participants: z.array(z.string()).optional(), + rssUrl: z.string().optional(), + posts: z.array(z.record(z.string(), z.unknown())).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetDiscussionResponse = z.infer; + +// 2.7 Get Event Data +export const GetEventInputSchema = z.object({ + url: z.string().describe('The URL of the event page to extract'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type GetEventInput = z.infer; + +export const GetEventResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('event').optional(), + title: z.string().optional(), + description: z.string().optional(), + startDate: z.string().optional(), + endDate: z.string().optional(), + location: z.string().optional(), + venue: z.record(z.string(), z.unknown()).optional(), + organizer: z.string().optional(), + ticketUrl: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type GetEventResponse = z.infer; + +// 2.8 Extract List +export const ExtractListInputSchema = z.object({ + url: z.string().describe('The URL of the list / directory / index page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type ExtractListInput = z.infer; + +export const ExtractListResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('list').optional(), + title: z.string().optional(), + numItems: z.number().optional(), + items: z.array(z.record(z.string(), z.unknown())).optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type ExtractListResponse = z.infer; + +// 2.9 Extract Job +export const ExtractJobInputSchema = z.object({ + url: z.string().describe('The URL of the job posting page'), + fields: z.string().optional().describe('Optional fields to return'), + timeout: z.number().optional().describe('Timeout in milliseconds'), +}); +export type ExtractJobInput = z.infer; + +export const ExtractJobResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + objects: z.array( + z + .object({ + type: z.literal('job').optional(), + title: z.string().optional(), + description: z.string().optional(), + company: z.record(z.string(), z.unknown()).optional(), + locations: z.array(z.string()).optional(), + employmentType: z.string().optional(), + compensation: z.record(z.string(), z.unknown()).optional(), + requirements: z.array(z.string()).optional(), + skills: z.array(z.string()).optional(), + postedDate: z.string().optional(), + pageUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + }) + .passthrough(), + ), + }) + .passthrough(); +export type ExtractJobResponse = z.infer; + +// --------------------------------------------------------------------------- +// 3. Search / DQL APIs (2 operations) +// --------------------------------------------------------------------------- + +// 3.1 Diffbot Knowledge Graph Search (DIFFBOT_SEARCH) +export const SearchInputSchema = z.object({ + query: z + .string() + .describe('DQL query string (e.g. "type:Organization name:\\"OpenAI\\"")'), + entityType: z + .string() + .optional() + .describe('Entity type filter prepended to query (e.g. "Organization")'), + queryType: z + .enum(['query', 'text', 'queryTextFallback', 'crawl']) + .optional() + .describe('Execution mode for the DQL request'), + size: z.number().optional().describe('Number of results to return'), + from: z.number().optional().describe('Zero-indexed offset for pagination'), + col: z.string().optional().describe('Crawl collection name to query'), +}); +export type SearchInput = z.infer; + +export const SearchResponseSchema = z + .object({ + version: z.number().optional(), + hits: z.number().optional(), + results: z.number().optional(), + kgversion: z.string().optional(), + diffbot_type: z.string().optional(), + facet: z.record(z.string(), z.unknown()).optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + cursor: z.string().optional(), + }) + .passthrough(); +export type SearchResponse = z.infer; + +// 3.2 Search Crawl Job Data (DIFFBOT_SEARCH_CRAWL_DATA) +export const SearchCrawlDataInputSchema = z.object({ + col: z.string().describe('The name of the crawl job collection to search'), + query: z.string().describe('Search query string or DQL filter'), + num: z + .number() + .min(1) + .max(25) + .optional() + .describe('Number of results to return (max 25)'), + start: z.number().optional().describe('Zero-indexed offset for pagination'), +}); +export type SearchCrawlDataInput = z.infer; + +export const SearchCrawlDataResponseSchema = z + .object({ + request: DiffbotRequestMetaSchema, + results: z.array(z.record(z.string(), z.unknown())).optional(), + numResults: z.number().optional(), + hits: z.number().optional(), + }) + .passthrough(); +export type SearchCrawlDataResponse = z.infer< + typeof SearchCrawlDataResponseSchema +>; + +// --------------------------------------------------------------------------- +// 4. Enhance APIs (4 operations) +// --------------------------------------------------------------------------- + +// 4.1 Enhance Entity with Knowledge Graph (DIFFBOT_ENHANCE_ENTITY) +export const EnhanceEntityInputSchema = z.object({ + name: z.string().optional().describe('Entity name (person or organization)'), + type: z + .string() + .optional() + .describe('Entity type filter: "Organization" or "Person"'), + email: z.string().optional().describe('Email address of the entity'), + employer: z + .string() + .optional() + .describe('Current employer of a Person entity'), + url: z + .string() + .optional() + .describe('Homepage or profile URL (e.g. LinkedIn / Website)'), + phone: z.string().optional().describe('Phone number'), + location: z.string().optional().describe('Location or address'), + size: z + .number() + .optional() + .describe('Number of matching entity records to return'), + refresh: z + .boolean() + .optional() + .describe('Force refresh data from live web sources'), +}); +export type EnhanceEntityInput = z.infer; + +export const EnhanceEntityResponseSchema = z + .object({ + version: z.number().optional(), + hits: z.number().optional(), + kgversion: z.string().optional(), + request_ctx: z.record(z.string(), z.unknown()).optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + errors: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type EnhanceEntityResponse = z.infer; + +// 4.2 Combine Entity Profiles (DIFFBOT_COMBINE_ENTITY_PROFILES) +export const CombineEntityProfilesInputSchema = z.object({ + name: z.string().optional().describe('Person name'), + type: z.string().optional().describe('Entity type (defaults to Person)'), + email: z.string().optional().describe('Email address'), + employer: z.string().optional().describe('Employer name or organization'), + url: z.string().optional().describe('Profile URL or organization homepage'), +}); +export type CombineEntityProfilesInput = z.infer< + typeof CombineEntityProfilesInputSchema +>; + +export const CombineEntityProfilesResponseSchema = z + .object({ + version: z.number().optional(), + hits: z.number().optional(), + kgversion: z.string().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + errors: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type CombineEntityProfilesResponse = z.infer< + typeof CombineEntityProfilesResponseSchema +>; + +// 4.3 Resolve Lost ID (DIFFBOT_RESOLVE_LOST_ID) +export const ResolveLostIdInputSchema = z.object({ + id: z.string().describe('The lost or non-canonical identifier to resolve'), +}); +export type ResolveLostIdInput = z.infer; + +export const ResolveLostIdResponseSchema = z + .object({ + id: z.string().optional(), + canonicalId: z.string().optional(), + diffbotUri: z.string().optional(), + name: z.string().optional(), + type: z.string().optional(), + hits: z.number().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ResolveLostIdResponse = z.infer; + +// 4.4 Get KG Coverage Report by ID (DIFFBOT_GET_KG_COVERAGE_REPORT_BY_ID) +export const GetKgCoverageReportByIdInputSchema = z.object({ + reportId: z + .string() + .describe('Coverage report ID generated from DQL query or bulk job'), + bulkjobId: z + .string() + .optional() + .describe('Optional bulkjob ID associated with the report'), +}); +export type GetKgCoverageReportByIdInput = z.infer< + typeof GetKgCoverageReportByIdInputSchema +>; + +export const GetKgCoverageReportByIdResponseSchema = z + .object({ + reportId: z.string().optional(), + status: z.string().optional(), + coverage: z.record(z.string(), z.unknown()).optional(), + data: z.unknown().optional(), + csv: z.string().optional(), + }) + .passthrough(); +export type GetKgCoverageReportByIdResponse = z.infer< + typeof GetKgCoverageReportByIdResponseSchema +>; + +// --------------------------------------------------------------------------- +// 5. KG Bulk Enhance APIs (8 operations) +// --------------------------------------------------------------------------- + +// 5.1 Create Bulk Enhance Job (DIFFBOT_CREATE_KG_BULK_ENHANCE) +export const CreateKgBulkEnhanceInputSchema = z.object({ + entities: z + .array( + z + .object({ + name: z.string().optional(), + type: z.string().optional(), + email: z.string().optional(), + employer: z.string().optional(), + url: z.string().optional(), + phone: z.string().optional(), + location: z.string().optional(), + }) + .passthrough(), + ) + .describe('Array of entity objects to enhance'), + notifyEmail: z + .string() + .optional() + .describe('Email address to notify upon job completion'), + name: z.string().optional().describe('Custom name for the bulk job'), +}); +export type CreateKgBulkEnhanceInput = z.infer< + typeof CreateKgBulkEnhanceInputSchema +>; + +export const CreateKgBulkEnhanceResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + job_id: z.string().optional(), + status: z.string().optional(), + total: z.number().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type CreateKgBulkEnhanceResponse = z.infer< + typeof CreateKgBulkEnhanceResponseSchema +>; + +// 5.2 Get Bulk Job Status (DIFFBOT_GET_BULK_JOB_STATUS) +export const GetBulkJobStatusInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to check'), +}); +export type GetBulkJobStatusInput = z.infer; + +export const GetBulkJobStatusResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + jobStatus: z.record(z.string(), z.unknown()).optional(), + total: z.number().optional(), + completed: z.number().optional(), + failed: z.number().optional(), + progress: z.number().optional(), + }) + .passthrough(); +export type GetBulkJobStatusResponse = z.infer< + typeof GetBulkJobStatusResponseSchema +>; + +// 5.3 List Bulk Jobs Status For Token (DIFFBOT_LIST_BULK_JOBS_STATUS_FOR_TOKEN) +export const ListBulkJobsStatusForTokenInputSchema = z.object({}); +export type ListBulkJobsStatusForTokenInput = z.infer< + typeof ListBulkJobsStatusForTokenInputSchema +>; + +export const ListBulkJobsStatusForTokenResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), + bulkjobs: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ListBulkJobsStatusForTokenResponse = z.infer< + typeof ListBulkJobsStatusForTokenResponseSchema +>; + +// 5.4 Get Bulk Job Results (DIFFBOT_GET_BULK_RESULTS) +export const GetBulkResultsInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to download'), + format: z + .enum(['json', 'jsonl', 'csv', 'xls', 'xlsx']) + .optional() + .describe('Download output format (default jsonl)'), + head: z + .number() + .optional() + .describe('Preview only the first N results from the job'), +}); +export type GetBulkResultsInput = z.infer; + +export const GetBulkResultsResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + raw: z.string().optional(), + }) + .passthrough(); +export type GetBulkResultsResponse = z.infer< + typeof GetBulkResultsResponseSchema +>; + +// 5.5 Download Bulk Job Results (DIFFBOT_DOWNLOAD_BULK_RESULTS) +export const DownloadBulkResultsInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to download'), + format: z + .enum(['json', 'jsonl', 'csv', 'xls', 'xlsx']) + .optional() + .describe('Export format'), + filter: z + .string() + .optional() + .describe('DQL filter criteria to apply to the output'), + fields: z + .string() + .optional() + .describe('Comma-separated list of fields to include'), + head: z.number().optional().describe('Number of records to export'), +}); +export type DownloadBulkResultsInput = z.infer< + typeof DownloadBulkResultsInputSchema +>; + +export const DownloadBulkResultsResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + data: z.array(z.record(z.string(), z.unknown())).optional(), + raw: z.string().optional(), + }) + .passthrough(); +export type DownloadBulkResultsResponse = z.infer< + typeof DownloadBulkResultsResponseSchema +>; + +// 5.6 Get Bulk Single Result (DIFFBOT_GET_BULK_SINGLE_RESULT) +export const GetBulkSingleResultInputSchema = z.object({ + bulkjobId: z.string().describe('The bulk enhance job ID'), + jobIndex: z + .number() + .describe('Zero-indexed position of the entity record within the bulk job'), +}); +export type GetBulkSingleResultInput = z.infer< + typeof GetBulkSingleResultInputSchema +>; + +export const GetBulkSingleResultResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + jobIndex: z.number().optional(), + data: z.record(z.string(), z.unknown()).optional(), + entity: z.record(z.string(), z.unknown()).optional(), + }) + .passthrough(); +export type GetBulkSingleResultResponse = z.infer< + typeof GetBulkSingleResultResponseSchema +>; + +// 5.7 Stop KG Bulk Job By ID (DIFFBOT_STOP_KG_BULK_JOB_BY_ID) +export const StopKgBulkJobByIdInputSchema = z.object({ + bulkjobId: z + .string() + .describe('The ID of the bulk enhance job to pause/stop'), +}); +export type StopKgBulkJobByIdInput = z.infer< + typeof StopKgBulkJobByIdInputSchema +>; + +export const StopKgBulkJobByIdResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type StopKgBulkJobByIdResponse = z.infer< + typeof StopKgBulkJobByIdResponseSchema +>; + +// 5.8 Delete KG Enhance Bulkjob (DIFFBOT_DELETE_KG_ENHANCE_BULKJOB) +export const DeleteKgEnhanceBulkjobInputSchema = z.object({ + bulkjobId: z.string().describe('The ID of the bulk enhance job to delete'), +}); +export type DeleteKgEnhanceBulkjobInput = z.infer< + typeof DeleteKgEnhanceBulkjobInputSchema +>; + +export const DeleteKgEnhanceBulkjobResponseSchema = z + .object({ + bulkjobId: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type DeleteKgEnhanceBulkjobResponse = z.infer< + typeof DeleteKgEnhanceBulkjobResponseSchema +>; + +// --------------------------------------------------------------------------- +// 6. Bulk Extract APIs (5 operations) +// --------------------------------------------------------------------------- + +// 6.1 Create Bulk Extract Job (DIFFBOT_CREATE_BULK) +export const CreateBulkInputSchema = z.object({ + name: z.string().describe('Name of the bulk job (unique per token)'), + apiUrl: z + .string() + .describe( + 'Full Diffbot Extract API URL (e.g. "https://api.diffbot.com/v3/article")', + ), + urls: z + .array(z.string()) + .describe('Array of URLs to process with the Extract API'), + notifyEmail: z + .string() + .optional() + .describe('Email to notify when processing is completed'), + maxRounds: z.number().optional().describe('Max rounds of URL processing'), +}); +export type CreateBulkInput = z.infer; + +export const CreateBulkResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + message: z.string().optional(), + }) + .passthrough(); +export type CreateBulkResponse = z.infer; + +// 6.2 Start Bulk Job (DIFFBOT_START_BULK) +export const StartBulkInputSchema = z.object({ + name: z.string().describe('Unique name for the bulk extract job'), + apiUrl: z.string().describe('Full Diffbot Extract API URL'), + urls: z + .string() + .describe('Comma-separated or space-separated list of URLs to process'), + notifyEmail: z.string().optional().describe('Notification email address'), + maxRounds: z.number().optional().describe('Max rounds of URL processing'), +}); +export type StartBulkInput = z.infer; + +export const StartBulkResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StartBulkResponse = z.infer; + +// 6.3 Stop Bulk Job (DIFFBOT_STOP_BULK_JOB) +export const StopBulkJobInputSchema = z.object({ + name: z.string().describe('The name of the bulk extract job to pause/stop'), +}); +export type StopBulkJobInput = z.infer; + +export const StopBulkJobResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StopBulkJobResponse = z.infer; + +// 6.4 Get Bulk Job Data (DIFFBOT_GET_BULK_DATA) +export const GetBulkDataInputSchema = z.object({ + name: z.string().describe('The name of the completed bulk job to download'), + format: z + .enum(['json', 'csv']) + .optional() + .describe('Download format (default json)'), +}); +export type GetBulkDataInput = z.infer; + +export const GetBulkDataResponseSchema = z + .object({ + name: z.string().optional(), + data: z.unknown().optional(), + }) + .passthrough(); +export type GetBulkDataResponse = z.infer; + +// 6.5 List Bulk Jobs (DIFFBOT_LIST_BULK_JOBS) +export const ListBulkJobsInputSchema = z.object({}); +export type ListBulkJobsInput = z.infer; + +export const ListBulkJobsResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ListBulkJobsResponse = z.infer; + +// --------------------------------------------------------------------------- +// 7. Crawl APIs (3 operations) +// --------------------------------------------------------------------------- + +// 7.1 Start Crawl Job (DIFFBOT_START_CRAWL) +export const StartCrawlInputSchema = z.object({ + name: z.string().describe('Unique name for the crawl job'), + seeds: z + .string() + .describe('Space-separated seed URL(s) from which the crawl begins'), + apiUrl: z + .string() + .describe( + 'Full Diffbot Extract API URL used to process pages (e.g. "https://api.diffbot.com/v3/article")', + ), + maxHops: z + .number() + .optional() + .describe( + 'Max depth of links to crawl from seeds (default -1 for no limit)', + ), + maxRounds: z + .number() + .optional() + .describe('Max rounds of repeat crawling for recurring crawls'), + maxTags: z.number().optional().describe('Max tags to extract per page'), + crawlSubdomains: z + .number() + .optional() + .describe('Set to 1 to crawl subdomains of seeds'), + notifyEmail: z + .string() + .optional() + .describe('Email notification upon crawl completion'), +}); +export type StartCrawlInput = z.infer; + +export const StartCrawlResponseSchema = z + .object({ + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type StartCrawlResponse = z.infer; + +// 7.2 Manage Crawl Job (DIFFBOT_MANAGE_CRAWL) +export const ManageCrawlInputSchema = z.object({ + name: z + .string() + .optional() + .describe('The name of the crawl job to inspect or modify'), + pause: z + .number() + .optional() + .describe('Set to 1 to pause an active crawl job, 0 to resume'), + restart: z + .number() + .optional() + .describe('Set to 1 to restart a completed/paused crawl job'), + delete: z + .number() + .optional() + .describe('Set to 1 to delete a crawl job and its data'), + roundProxy: z + .number() + .optional() + .describe('Set to 1 to rotate proxy IP on each round'), + maxRounds: z.number().optional().describe('Update max rounds'), + maxHops: z.number().optional().describe('Update max hops'), +}); +export type ManageCrawlInput = z.infer; + +export const ManageCrawlResponseSchema = z + .object({ + jobs: z.array(z.record(z.string(), z.unknown())).optional(), + response: z.string().optional(), + name: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type ManageCrawlResponse = z.infer; + +// 7.3 Get Crawl Data (DIFFBOT_GET_CRAWL_DATA) +export const GetCrawlDataInputSchema = z.object({ + name: z.string().describe('The name of the completed crawl job to download'), + format: z + .enum(['json', 'csv']) + .optional() + .describe('Download format (default json)'), +}); +export type GetCrawlDataInput = z.infer; + +export const GetCrawlDataResponseSchema = z + .object({ + name: z.string().optional(), + data: z.unknown().optional(), + }) + .passthrough(); +export type GetCrawlDataResponse = z.infer; + +// --------------------------------------------------------------------------- +// 8. Custom API (3 operations) +// --------------------------------------------------------------------------- + +// 8.1 Create or Update Custom API (DIFFBOT_CREATE_CUSTOM_API) +export const CreateCustomApiInputSchema = z.object({ + api: z + .string() + .describe('Name of the custom API (e.g. "myCustomArticleApi")'), + url: z.string().describe('Sample URL that this custom API applies to'), + pattern: z + .string() + .optional() + .describe('URL regex pattern to match pages for this custom API'), + rules: z + .record(z.string(), z.unknown()) + .optional() + .describe('Extraction rules and CSS selector definitions'), +}); +export type CreateCustomApiInput = z.infer; + +export const CreateCustomApiResponseSchema = z + .object({ + response: z.string().optional(), + api: z.string().optional(), + url: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type CreateCustomApiResponse = z.infer< + typeof CreateCustomApiResponseSchema +>; + +// 8.2 List Custom APIs (DIFFBOT_LIST_CUSTOM_APIS) +export const ListCustomApisInputSchema = z.object({}); +export type ListCustomApisInput = z.infer; + +export const ListCustomApisResponseSchema = z + .object({ + customApis: z.array(z.record(z.string(), z.unknown())).optional(), + apis: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(); +export type ListCustomApisResponse = z.infer< + typeof ListCustomApisResponseSchema +>; + +// 8.3 Delete Custom API (DIFFBOT_DELETE_CUSTOM_API) +export const DeleteCustomApiInputSchema = z.object({ + api: z.string().describe('Name of the custom API to delete'), + url: z + .string() + .optional() + .describe('URL pattern or test URL of the custom API'), +}); +export type DeleteCustomApiInput = z.infer; + +export const DeleteCustomApiResponseSchema = z + .object({ + response: z.string().optional(), + api: z.string().optional(), + status: z.string().optional(), + }) + .passthrough(); +export type DeleteCustomApiResponse = z.infer< + typeof DeleteCustomApiResponseSchema +>; + +// --------------------------------------------------------------------------- +// Aggregated type maps (35 operations keyed by camelCase endpoint name) +// --------------------------------------------------------------------------- + +export type DiffbotEndpointInputs = { + // Account + getAccount: GetAccountInput; + + // Extract + getArticle: GetArticleInput; + getProduct: GetProductInput; + getAnalyze: GetAnalyzeInput; + getImage: GetImageInput; + getVideo: GetVideoInput; + getDiscussion: GetDiscussionInput; + getEvent: GetEventInput; + extractList: ExtractListInput; + extractJob: ExtractJobInput; + + // Search + search: SearchInput; + searchCrawlData: SearchCrawlDataInput; + + // Enhance + enhanceEntity: EnhanceEntityInput; + combineEntityProfiles: CombineEntityProfilesInput; + resolveLostId: ResolveLostIdInput; + getKgCoverageReportById: GetKgCoverageReportByIdInput; + + // KG Bulk Enhance + createKgBulkEnhance: CreateKgBulkEnhanceInput; + getBulkJobStatus: GetBulkJobStatusInput; + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenInput; + getBulkResults: GetBulkResultsInput; + downloadBulkResults: DownloadBulkResultsInput; + getBulkSingleResult: GetBulkSingleResultInput; + stopKgBulkJobById: StopKgBulkJobByIdInput; + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobInput; + + // Bulk Extract + createBulk: CreateBulkInput; + startBulk: StartBulkInput; + stopBulkJob: StopBulkJobInput; + getBulkData: GetBulkDataInput; + listBulkJobs: ListBulkJobsInput; + + // Crawl + startCrawl: StartCrawlInput; + manageCrawl: ManageCrawlInput; + getCrawlData: GetCrawlDataInput; + + // Custom API + createCustomApi: CreateCustomApiInput; + listCustomApis: ListCustomApisInput; + deleteCustomApi: DeleteCustomApiInput; +}; + +export type DiffbotEndpointOutputs = { + // Account + getAccount: GetAccountResponse; + + // Extract + getArticle: GetArticleResponse; + getProduct: GetProductResponse; + getAnalyze: GetAnalyzeResponse; + getImage: GetImageResponse; + getVideo: GetVideoResponse; + getDiscussion: GetDiscussionResponse; + getEvent: GetEventResponse; + extractList: ExtractListResponse; + extractJob: ExtractJobResponse; + + // Search + search: SearchResponse; + searchCrawlData: SearchCrawlDataResponse; + + // Enhance + enhanceEntity: EnhanceEntityResponse; + combineEntityProfiles: CombineEntityProfilesResponse; + resolveLostId: ResolveLostIdResponse; + getKgCoverageReportById: GetKgCoverageReportByIdResponse; + + // KG Bulk Enhance + createKgBulkEnhance: CreateKgBulkEnhanceResponse; + getBulkJobStatus: GetBulkJobStatusResponse; + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenResponse; + getBulkResults: GetBulkResultsResponse; + downloadBulkResults: DownloadBulkResultsResponse; + getBulkSingleResult: GetBulkSingleResultResponse; + stopKgBulkJobById: StopKgBulkJobByIdResponse; + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobResponse; + + // Bulk Extract + createBulk: CreateBulkResponse; + startBulk: StartBulkResponse; + stopBulkJob: StopBulkJobResponse; + getBulkData: GetBulkDataResponse; + listBulkJobs: ListBulkJobsResponse; + + // Crawl + startCrawl: StartCrawlResponse; + manageCrawl: ManageCrawlResponse; + getCrawlData: GetCrawlDataResponse; + + // Custom API + createCustomApi: CreateCustomApiResponse; + listCustomApis: ListCustomApisResponse; + deleteCustomApi: DeleteCustomApiResponse; +}; + +export const DiffbotEndpointInputSchemas = { + getAccount: GetAccountInputSchema, + getArticle: GetArticleInputSchema, + getProduct: GetProductInputSchema, + getAnalyze: GetAnalyzeInputSchema, + getImage: GetImageInputSchema, + getVideo: GetVideoInputSchema, + getDiscussion: GetDiscussionInputSchema, + getEvent: GetEventInputSchema, + extractList: ExtractListInputSchema, + extractJob: ExtractJobInputSchema, + search: SearchInputSchema, + searchCrawlData: SearchCrawlDataInputSchema, + enhanceEntity: EnhanceEntityInputSchema, + combineEntityProfiles: CombineEntityProfilesInputSchema, + resolveLostId: ResolveLostIdInputSchema, + getKgCoverageReportById: GetKgCoverageReportByIdInputSchema, + createKgBulkEnhance: CreateKgBulkEnhanceInputSchema, + getBulkJobStatus: GetBulkJobStatusInputSchema, + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenInputSchema, + getBulkResults: GetBulkResultsInputSchema, + downloadBulkResults: DownloadBulkResultsInputSchema, + getBulkSingleResult: GetBulkSingleResultInputSchema, + stopKgBulkJobById: StopKgBulkJobByIdInputSchema, + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobInputSchema, + createBulk: CreateBulkInputSchema, + startBulk: StartBulkInputSchema, + stopBulkJob: StopBulkJobInputSchema, + getBulkData: GetBulkDataInputSchema, + listBulkJobs: ListBulkJobsInputSchema, + startCrawl: StartCrawlInputSchema, + manageCrawl: ManageCrawlInputSchema, + getCrawlData: GetCrawlDataInputSchema, + createCustomApi: CreateCustomApiInputSchema, + listCustomApis: ListCustomApisInputSchema, + deleteCustomApi: DeleteCustomApiInputSchema, +} as const; + +export const DiffbotEndpointOutputSchemas = { + getAccount: GetAccountResponseSchema, + getArticle: GetArticleResponseSchema, + getProduct: GetProductResponseSchema, + getAnalyze: GetAnalyzeResponseSchema, + getImage: GetImageResponseSchema, + getVideo: GetVideoResponseSchema, + getDiscussion: GetDiscussionResponseSchema, + getEvent: GetEventResponseSchema, + extractList: ExtractListResponseSchema, + extractJob: ExtractJobResponseSchema, + search: SearchResponseSchema, + searchCrawlData: SearchCrawlDataResponseSchema, + enhanceEntity: EnhanceEntityResponseSchema, + combineEntityProfiles: CombineEntityProfilesResponseSchema, + resolveLostId: ResolveLostIdResponseSchema, + getKgCoverageReportById: GetKgCoverageReportByIdResponseSchema, + createKgBulkEnhance: CreateKgBulkEnhanceResponseSchema, + getBulkJobStatus: GetBulkJobStatusResponseSchema, + listBulkJobsStatusForToken: ListBulkJobsStatusForTokenResponseSchema, + getBulkResults: GetBulkResultsResponseSchema, + downloadBulkResults: DownloadBulkResultsResponseSchema, + getBulkSingleResult: GetBulkSingleResultResponseSchema, + stopKgBulkJobById: StopKgBulkJobByIdResponseSchema, + deleteKgEnhanceBulkjob: DeleteKgEnhanceBulkjobResponseSchema, + createBulk: CreateBulkResponseSchema, + startBulk: StartBulkResponseSchema, + stopBulkJob: StopBulkJobResponseSchema, + getBulkData: GetBulkDataResponseSchema, + listBulkJobs: ListBulkJobsResponseSchema, + startCrawl: StartCrawlResponseSchema, + manageCrawl: ManageCrawlResponseSchema, + getCrawlData: GetCrawlDataResponseSchema, + createCustomApi: CreateCustomApiResponseSchema, + listCustomApis: ListCustomApisResponseSchema, + deleteCustomApi: DeleteCustomApiResponseSchema, +} as const; diff --git a/packages/diffbot/error-handlers.ts b/packages/diffbot/error-handlers.ts new file mode 100644 index 000000000..5899cb599 --- /dev/null +++ b/packages/diffbot/error-handlers.ts @@ -0,0 +1,70 @@ +import type { CorsairErrorHandler } from 'corsair/core'; + +type DiffbotError = Error & { + status?: number; + retryAfter?: number; +}; + +function hasStatus(error: Error, status: number): boolean { + return (error as DiffbotError).status === status; +} + +function retryAfter(error: Error): number | undefined { + return (error as DiffbotError).retryAfter; +} + +export const errorHandlers = { + RATE_LIMIT_ERROR: { + match: (error: Error) => { + if (hasStatus(error, 429)) return true; + const msg = error.message.toLowerCase(); + return ( + msg.includes('rate_limited') || + msg.includes('429') || + msg.includes('too many requests') + ); + }, + handler: async (error: Error) => { + return { maxRetries: 5, headersRetryAfterMs: retryAfter(error) }; + }, + }, + AUTH_ERROR: { + match: (error: Error) => { + if (hasStatus(error, 401) || hasStatus(error, 403)) return true; + const msg = error.message.toLowerCase(); + return ( + msg.includes('unauthorized') || + msg.includes('invalid_auth') || + msg.includes('invalid token') || + msg.includes('forbidden') + ); + }, + handler: async (_error?: Error) => ({ maxRetries: 0 }), + }, + NOT_FOUND_ERROR: { + match: (error: Error) => { + if (hasStatus(error, 404)) return true; + const msg = error.message.toLowerCase(); + return msg.includes('not found') || msg.includes('404'); + }, + handler: async (_error?: Error) => ({ maxRetries: 0 }), + }, + SERVER_ERROR: { + match: (error: Error) => { + if ( + hasStatus(error, 500) || + hasStatus(error, 502) || + hasStatus(error, 503) + ) { + return true; + } + const msg = error.message.toLowerCase(); + return msg.includes('internal server error') || msg.includes('500'); + }, + handler: async (_error?: Error) => ({ maxRetries: 2 }), + }, + DEFAULT: { + match: () => true, + handler: async (_error?: Error) => ({ maxRetries: 0 }), + }, +} satisfies CorsairErrorHandler; diff --git a/packages/diffbot/index.ts b/packages/diffbot/index.ts new file mode 100644 index 000000000..feefb7b9f --- /dev/null +++ b/packages/diffbot/index.ts @@ -0,0 +1,538 @@ +import type { + AuthTypes, + BindEndpoints, + CorsairEndpoint, + CorsairErrorHandler, + CorsairPlugin, + CorsairPluginContext, + KeyBuilderContext, + PickAuth, + PluginAuthConfig, + PluginPermissionsConfig, + RequiredPluginEndpointMeta, + RequiredPluginEndpointSchemas, +} from 'corsair/core'; +import { + Account, + Bulk, + Crawl, + CustomApi, + Enhance, + Extract, + KgBulkEnhance, + Search, +} from './endpoints'; +import type { + DiffbotEndpointInputs, + DiffbotEndpointOutputs, +} from './endpoints/types'; +import { + DiffbotEndpointInputSchemas, + DiffbotEndpointOutputSchemas, +} from './endpoints/types'; +import { errorHandlers } from './error-handlers'; +import { DiffbotSchema } from './schema'; + +export type DiffbotPluginOptions = { + authType?: PickAuth<'api_key'>; + key?: string; + hooks?: InternalDiffbotPlugin['hooks']; + errorHandlers?: CorsairErrorHandler; + permissions?: PluginPermissionsConfig; +}; + +export type DiffbotContext = CorsairPluginContext< + typeof DiffbotSchema, + DiffbotPluginOptions +>; + +export type DiffbotKeyBuilderContext = KeyBuilderContext; + +export type DiffbotBoundEndpoints = BindEndpoints< + typeof diffbotEndpointsNested +>; + +type DiffbotEndpoint = CorsairEndpoint< + DiffbotContext, + DiffbotEndpointInputs[K], + DiffbotEndpointOutputs[K] +>; + +export type DiffbotEndpoints = { + // Account + getAccount: DiffbotEndpoint<'getAccount'>; + + // Extract + getArticle: DiffbotEndpoint<'getArticle'>; + getProduct: DiffbotEndpoint<'getProduct'>; + getAnalyze: DiffbotEndpoint<'getAnalyze'>; + getImage: DiffbotEndpoint<'getImage'>; + getVideo: DiffbotEndpoint<'getVideo'>; + getDiscussion: DiffbotEndpoint<'getDiscussion'>; + getEvent: DiffbotEndpoint<'getEvent'>; + extractList: DiffbotEndpoint<'extractList'>; + extractJob: DiffbotEndpoint<'extractJob'>; + + // Search + search: DiffbotEndpoint<'search'>; + searchCrawlData: DiffbotEndpoint<'searchCrawlData'>; + + // Enhance + enhanceEntity: DiffbotEndpoint<'enhanceEntity'>; + combineEntityProfiles: DiffbotEndpoint<'combineEntityProfiles'>; + resolveLostId: DiffbotEndpoint<'resolveLostId'>; + getKgCoverageReportById: DiffbotEndpoint<'getKgCoverageReportById'>; + + // KG Bulk Enhance + createKgBulkEnhance: DiffbotEndpoint<'createKgBulkEnhance'>; + getBulkJobStatus: DiffbotEndpoint<'getBulkJobStatus'>; + listBulkJobsStatusForToken: DiffbotEndpoint<'listBulkJobsStatusForToken'>; + getBulkResults: DiffbotEndpoint<'getBulkResults'>; + downloadBulkResults: DiffbotEndpoint<'downloadBulkResults'>; + getBulkSingleResult: DiffbotEndpoint<'getBulkSingleResult'>; + stopKgBulkJobById: DiffbotEndpoint<'stopKgBulkJobById'>; + deleteKgEnhanceBulkjob: DiffbotEndpoint<'deleteKgEnhanceBulkjob'>; + + // Bulk Extract + createBulk: DiffbotEndpoint<'createBulk'>; + startBulk: DiffbotEndpoint<'startBulk'>; + stopBulkJob: DiffbotEndpoint<'stopBulkJob'>; + getBulkData: DiffbotEndpoint<'getBulkData'>; + listBulkJobs: DiffbotEndpoint<'listBulkJobs'>; + + // Crawl + startCrawl: DiffbotEndpoint<'startCrawl'>; + manageCrawl: DiffbotEndpoint<'manageCrawl'>; + getCrawlData: DiffbotEndpoint<'getCrawlData'>; + + // Custom API + createCustomApi: DiffbotEndpoint<'createCustomApi'>; + listCustomApis: DiffbotEndpoint<'listCustomApis'>; + deleteCustomApi: DiffbotEndpoint<'deleteCustomApi'>; +}; + +const diffbotEndpointsNested = { + account: { + getAccount: Account.getAccount, + }, + extract: { + getArticle: Extract.getArticle, + getProduct: Extract.getProduct, + getAnalyze: Extract.getAnalyze, + getImage: Extract.getImage, + getVideo: Extract.getVideo, + getDiscussion: Extract.getDiscussion, + getEvent: Extract.getEvent, + extractList: Extract.extractList, + extractJob: Extract.extractJob, + }, + search: { + search: Search.search, + searchCrawlData: Search.searchCrawlData, + }, + enhance: { + enhanceEntity: Enhance.enhanceEntity, + combineEntityProfiles: Enhance.combineEntityProfiles, + resolveLostId: Enhance.resolveLostId, + getKgCoverageReportById: Enhance.getKgCoverageReportById, + }, + kgBulkEnhance: { + createKgBulkEnhance: KgBulkEnhance.createKgBulkEnhance, + getBulkJobStatus: KgBulkEnhance.getBulkJobStatus, + listBulkJobsStatusForToken: KgBulkEnhance.listBulkJobsStatusForToken, + getBulkResults: KgBulkEnhance.getBulkResults, + downloadBulkResults: KgBulkEnhance.downloadBulkResults, + getBulkSingleResult: KgBulkEnhance.getBulkSingleResult, + stopKgBulkJobById: KgBulkEnhance.stopKgBulkJobById, + deleteKgEnhanceBulkjob: KgBulkEnhance.deleteKgEnhanceBulkjob, + }, + bulk: { + createBulk: Bulk.createBulk, + startBulk: Bulk.startBulk, + stopBulkJob: Bulk.stopBulkJob, + getBulkData: Bulk.getBulkData, + listBulkJobs: Bulk.listBulkJobs, + }, + crawl: { + startCrawl: Crawl.startCrawl, + manageCrawl: Crawl.manageCrawl, + getCrawlData: Crawl.getCrawlData, + }, + customApi: { + createCustomApi: CustomApi.createCustomApi, + listCustomApis: CustomApi.listCustomApis, + deleteCustomApi: CustomApi.deleteCustomApi, + }, +} as const; + +export const diffbotEndpointSchemas = { + 'account.getAccount': { + input: DiffbotEndpointInputSchemas.getAccount, + output: DiffbotEndpointOutputSchemas.getAccount, + }, + 'extract.getArticle': { + input: DiffbotEndpointInputSchemas.getArticle, + output: DiffbotEndpointOutputSchemas.getArticle, + }, + 'extract.getProduct': { + input: DiffbotEndpointInputSchemas.getProduct, + output: DiffbotEndpointOutputSchemas.getProduct, + }, + 'extract.getAnalyze': { + input: DiffbotEndpointInputSchemas.getAnalyze, + output: DiffbotEndpointOutputSchemas.getAnalyze, + }, + 'extract.getImage': { + input: DiffbotEndpointInputSchemas.getImage, + output: DiffbotEndpointOutputSchemas.getImage, + }, + 'extract.getVideo': { + input: DiffbotEndpointInputSchemas.getVideo, + output: DiffbotEndpointOutputSchemas.getVideo, + }, + 'extract.getDiscussion': { + input: DiffbotEndpointInputSchemas.getDiscussion, + output: DiffbotEndpointOutputSchemas.getDiscussion, + }, + 'extract.getEvent': { + input: DiffbotEndpointInputSchemas.getEvent, + output: DiffbotEndpointOutputSchemas.getEvent, + }, + 'extract.extractList': { + input: DiffbotEndpointInputSchemas.extractList, + output: DiffbotEndpointOutputSchemas.extractList, + }, + 'extract.extractJob': { + input: DiffbotEndpointInputSchemas.extractJob, + output: DiffbotEndpointOutputSchemas.extractJob, + }, + 'search.search': { + input: DiffbotEndpointInputSchemas.search, + output: DiffbotEndpointOutputSchemas.search, + }, + 'search.searchCrawlData': { + input: DiffbotEndpointInputSchemas.searchCrawlData, + output: DiffbotEndpointOutputSchemas.searchCrawlData, + }, + 'enhance.enhanceEntity': { + input: DiffbotEndpointInputSchemas.enhanceEntity, + output: DiffbotEndpointOutputSchemas.enhanceEntity, + }, + 'enhance.combineEntityProfiles': { + input: DiffbotEndpointInputSchemas.combineEntityProfiles, + output: DiffbotEndpointOutputSchemas.combineEntityProfiles, + }, + 'enhance.resolveLostId': { + input: DiffbotEndpointInputSchemas.resolveLostId, + output: DiffbotEndpointOutputSchemas.resolveLostId, + }, + 'enhance.getKgCoverageReportById': { + input: DiffbotEndpointInputSchemas.getKgCoverageReportById, + output: DiffbotEndpointOutputSchemas.getKgCoverageReportById, + }, + 'kgBulkEnhance.createKgBulkEnhance': { + input: DiffbotEndpointInputSchemas.createKgBulkEnhance, + output: DiffbotEndpointOutputSchemas.createKgBulkEnhance, + }, + 'kgBulkEnhance.getBulkJobStatus': { + input: DiffbotEndpointInputSchemas.getBulkJobStatus, + output: DiffbotEndpointOutputSchemas.getBulkJobStatus, + }, + 'kgBulkEnhance.listBulkJobsStatusForToken': { + input: DiffbotEndpointInputSchemas.listBulkJobsStatusForToken, + output: DiffbotEndpointOutputSchemas.listBulkJobsStatusForToken, + }, + 'kgBulkEnhance.getBulkResults': { + input: DiffbotEndpointInputSchemas.getBulkResults, + output: DiffbotEndpointOutputSchemas.getBulkResults, + }, + 'kgBulkEnhance.downloadBulkResults': { + input: DiffbotEndpointInputSchemas.downloadBulkResults, + output: DiffbotEndpointOutputSchemas.downloadBulkResults, + }, + 'kgBulkEnhance.getBulkSingleResult': { + input: DiffbotEndpointInputSchemas.getBulkSingleResult, + output: DiffbotEndpointOutputSchemas.getBulkSingleResult, + }, + 'kgBulkEnhance.stopKgBulkJobById': { + input: DiffbotEndpointInputSchemas.stopKgBulkJobById, + output: DiffbotEndpointOutputSchemas.stopKgBulkJobById, + }, + 'kgBulkEnhance.deleteKgEnhanceBulkjob': { + input: DiffbotEndpointInputSchemas.deleteKgEnhanceBulkjob, + output: DiffbotEndpointOutputSchemas.deleteKgEnhanceBulkjob, + }, + 'bulk.createBulk': { + input: DiffbotEndpointInputSchemas.createBulk, + output: DiffbotEndpointOutputSchemas.createBulk, + }, + 'bulk.startBulk': { + input: DiffbotEndpointInputSchemas.startBulk, + output: DiffbotEndpointOutputSchemas.startBulk, + }, + 'bulk.stopBulkJob': { + input: DiffbotEndpointInputSchemas.stopBulkJob, + output: DiffbotEndpointOutputSchemas.stopBulkJob, + }, + 'bulk.getBulkData': { + input: DiffbotEndpointInputSchemas.getBulkData, + output: DiffbotEndpointOutputSchemas.getBulkData, + }, + 'bulk.listBulkJobs': { + input: DiffbotEndpointInputSchemas.listBulkJobs, + output: DiffbotEndpointOutputSchemas.listBulkJobs, + }, + 'crawl.startCrawl': { + input: DiffbotEndpointInputSchemas.startCrawl, + output: DiffbotEndpointOutputSchemas.startCrawl, + }, + 'crawl.manageCrawl': { + input: DiffbotEndpointInputSchemas.manageCrawl, + output: DiffbotEndpointOutputSchemas.manageCrawl, + }, + 'crawl.getCrawlData': { + input: DiffbotEndpointInputSchemas.getCrawlData, + output: DiffbotEndpointOutputSchemas.getCrawlData, + }, + 'customApi.createCustomApi': { + input: DiffbotEndpointInputSchemas.createCustomApi, + output: DiffbotEndpointOutputSchemas.createCustomApi, + }, + 'customApi.listCustomApis': { + input: DiffbotEndpointInputSchemas.listCustomApis, + output: DiffbotEndpointOutputSchemas.listCustomApis, + }, + 'customApi.deleteCustomApi': { + input: DiffbotEndpointInputSchemas.deleteCustomApi, + output: DiffbotEndpointOutputSchemas.deleteCustomApi, + }, +} as const satisfies RequiredPluginEndpointSchemas< + typeof diffbotEndpointsNested +>; + +const defaultAuthType: AuthTypes = 'api_key' as const; + +const diffbotEndpointMeta = { + 'account.getAccount': { + riskLevel: 'read', + description: + 'Retrieve Diffbot account details, credit balance, and plan usage', + }, + 'extract.getArticle': { + riskLevel: 'read', + description: + 'Extract article title, text, author, date, and metadata from any URL', + }, + 'extract.getProduct': { + riskLevel: 'read', + description: + 'Extract product price, availability, images, and specifications from any e-commerce URL', + }, + 'extract.getAnalyze': { + riskLevel: 'read', + description: + 'Automatically analyze web page to determine its type and extract structured data', + }, + 'extract.getImage': { + riskLevel: 'read', + description: + 'Extract detailed image information including dimensions and recognition data', + }, + 'extract.getVideo': { + riskLevel: 'read', + description: + 'Extract structured metadata from videos including embed HTML and durations', + }, + 'extract.getDiscussion': { + riskLevel: 'read', + description: + 'Extract structured discussion threads, forum posts, and comments from web pages', + }, + 'extract.getEvent': { + riskLevel: 'read', + description: + 'Extract event details including dates, venues, organizers, and descriptions', + }, + 'extract.extractList': { + riskLevel: 'read', + description: + 'Extract structured items from list-style pages, catalogs, and news indexes', + }, + 'extract.extractJob': { + riskLevel: 'read', + description: + 'Extract structured job posting data including compensation, requirements, and company info', + }, + 'search.search': { + riskLevel: 'read', + description: + 'Search the Diffbot Knowledge Graph using DQL (Diffbot Query Language)', + }, + 'search.searchCrawlData': { + riskLevel: 'read', + description: 'Query crawl job collections using DQL or keyword search', + }, + 'enhance.enhanceEntity': { + riskLevel: 'read', + description: + 'Enrich person or organization data with Knowledge Graph records', + }, + 'enhance.combineEntityProfiles': { + riskLevel: 'read', + description: + 'Combine entity profiles into a unified view with organization affiliations', + }, + 'enhance.resolveLostId': { + riskLevel: 'read', + description: + 'Resolve lost or legacy identifiers to canonical Knowledge Graph entities', + }, + 'enhance.getKgCoverageReportById': { + riskLevel: 'read', + description: 'Download Knowledge Graph coverage report by report ID', + }, + 'kgBulkEnhance.createKgBulkEnhance': { + riskLevel: 'write', + description: + 'Submit an asynchronous bulk enhance job for multiple entities', + }, + 'kgBulkEnhance.getBulkJobStatus': { + riskLevel: 'read', + description: + 'Poll the status and progress of a Knowledge Graph bulk enhance job', + }, + 'kgBulkEnhance.listBulkJobsStatusForToken': { + riskLevel: 'read', + description: + 'List all Knowledge Graph bulk enhance jobs and their statuses for token', + }, + 'kgBulkEnhance.getBulkResults': { + riskLevel: 'read', + description: + 'Download results of a completed Knowledge Graph bulk enhance job', + }, + 'kgBulkEnhance.downloadBulkResults': { + riskLevel: 'read', + description: + 'Download bulk enhance results with filtering and custom output formats', + }, + 'kgBulkEnhance.getBulkSingleResult': { + riskLevel: 'read', + description: + 'Download single enriched entity result from a bulk enhance job by index', + }, + 'kgBulkEnhance.stopKgBulkJobById': { + riskLevel: 'write', + description: + 'Stop or pause an active Knowledge Graph bulk enhance job by ID', + }, + 'kgBulkEnhance.deleteKgEnhanceBulkjob': { + riskLevel: 'destructive', + description: 'Delete a Knowledge Graph bulk enhance job and its results', + }, + 'bulk.createBulk': { + riskLevel: 'write', + description: + 'Submit an asynchronous bulk extract job to process multiple URLs', + }, + 'bulk.startBulk': { + riskLevel: 'write', + description: 'Start a bulk extract job using query parameters', + }, + 'bulk.stopBulkJob': { + riskLevel: 'write', + description: 'Pause or stop an active bulk extract job', + }, + 'bulk.getBulkData': { + riskLevel: 'read', + description: 'Download extracted results from a completed bulk extract job', + }, + 'bulk.listBulkJobs': { + riskLevel: 'read', + description: 'List all bulk extract jobs associated with the token', + }, + 'crawl.startCrawl': { + riskLevel: 'write', + description: 'Initiate a website crawl job starting from seed URLs', + }, + 'crawl.manageCrawl': { + riskLevel: 'write', + description: 'Inspect, pause, restart, or delete crawl jobs', + }, + 'crawl.getCrawlData': { + riskLevel: 'read', + description: 'Download extracted data from a completed crawl job', + }, + 'customApi.createCustomApi': { + riskLevel: 'write', + description: + 'Create or update custom API rules and selectors for URL patterns', + }, + 'customApi.listCustomApis': { + riskLevel: 'read', + description: 'List all custom API definitions configured on the account', + }, + 'customApi.deleteCustomApi': { + riskLevel: 'destructive', + description: 'Delete custom API definitions for a given URL pattern', + }, +} as const satisfies RequiredPluginEndpointMeta; + +export const diffbotAuthConfig = { + api_key: { + account: ['tenant_external_id'] as const, + }, +} as const satisfies PluginAuthConfig; + +export type BaseDiffbotPlugin = CorsairPlugin< + 'diffbot', + typeof DiffbotSchema, + typeof diffbotEndpointsNested, + Record, + T, + typeof defaultAuthType +>; + +export type InternalDiffbotPlugin = BaseDiffbotPlugin; + +export type ExternalDiffbotPlugin = + BaseDiffbotPlugin; + +export function diffbot( + incomingOptions: DiffbotPluginOptions & T = {} as DiffbotPluginOptions & T, +): ExternalDiffbotPlugin { + const options = { + ...incomingOptions, + authType: incomingOptions.authType ?? defaultAuthType, + }; + return { + id: 'diffbot', + authConfig: diffbotAuthConfig, + schema: DiffbotSchema, + options: options, + hooks: options.hooks, + endpoints: diffbotEndpointsNested, + webhooks: {}, + endpointMeta: diffbotEndpointMeta, + endpointSchemas: diffbotEndpointSchemas, + errorHandlers: { + ...errorHandlers, + ...options.errorHandlers, + }, + keyBuilder: async (ctx: DiffbotKeyBuilderContext, source) => { + if (source === 'endpoint' && options.key) { + return options.key; + } + + if (source === 'endpoint' && ctx.authType === 'api_key') { + const res = await ctx.keys.get_api_key(); + return res ?? ''; + } + + return ''; + }, + } satisfies InternalDiffbotPlugin; +} + +export * from './endpoints/types'; +export * from './schema'; diff --git a/packages/diffbot/jest.config.cjs b/packages/diffbot/jest.config.cjs new file mode 100644 index 000000000..8c6218f64 --- /dev/null +++ b/packages/diffbot/jest.config.cjs @@ -0,0 +1,55 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: [''], + testMatch: [ + '**/*.test.ts', + '**/tests/**/*.test.ts', + '**/plugins/**/*.test.ts', + '**/setup/**/*.test.ts', + ], + collectCoverageFrom: [ + '**/*.ts', + '!**/*.d.ts', + '!**/node_modules/**', + '!**/dist/**', + '!jest.config.ts', + '!tests/**', + ], + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'], + transform: { + '^.+\\.yaml$': '/../corsair/jest-yaml-transform.cjs', + '^.+\\.ts$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + verbatimModuleSyntax: false, + module: 'ESNext', + moduleResolution: 'Bundler', + }, + }, + ], + '.*\\.js$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + esModuleInterop: true, + allowSyntheticDefaultImports: true, + }, + }, + ], + }, + moduleNameMapper: { + '^corsair/core$': '/../corsair/core.ts', + '^corsair/http$': '/../corsair/http.ts', + '^(\\.\\.?/.*)\\.js$': '$1', + }, + transformIgnorePatterns: ['node_modules/(?!.*uuid.*)'], + extensionsToTreatAsEsm: ['.ts'], + testTimeout: 30000, + verbose: true, +}; diff --git a/packages/diffbot/package.json b/packages/diffbot/package.json new file mode 100644 index 000000000..1cf46f5af --- /dev/null +++ b/packages/diffbot/package.json @@ -0,0 +1,44 @@ +{ + "name": "@corsair-dev/diffbot", + "version": "0.1.0", + "description": "Diffbot plugin for Corsair", + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "dev-source": "./index.ts", + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "rm -rf dist && tsc --build --force && tsup", + "typecheck": "tsc --noEmit", + "test": "jest" + }, + "peerDependencies": { + "corsair": ">=0.1.0", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "corsair": "workspace:*", + "jest": "^29.7.0", + "ts-jest": "^29.4.9", + "tsup": "^8.0.1", + "typescript": "catalog:", + "zod": "^4.1.13" + }, + "keywords": [ + "corsair", + "diffbot", + "plugin" + ], + "author": "", + "license": "Apache-2.0", + "files": [ + "dist" + ] +} diff --git a/packages/diffbot/schema.test.ts b/packages/diffbot/schema.test.ts new file mode 100644 index 000000000..c9b9a6ff4 --- /dev/null +++ b/packages/diffbot/schema.test.ts @@ -0,0 +1,33 @@ +import { DiffbotSchema } from './schema'; + +describe('Diffbot schema', () => { + it('declares a semver version', () => { + expect(DiffbotSchema.version).toBeDefined(); + expect(DiffbotSchema.version).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it('declares an entities map with all official ontology tables', () => { + expect(typeof DiffbotSchema.entities).toBe('object'); + expect(DiffbotSchema.entities).not.toBeNull(); + const entityKeys = Object.keys(DiffbotSchema.entities); + expect(entityKeys.length).toBeGreaterThanOrEqual(10); + expect(entityKeys).toContain('articles'); + expect(entityKeys).toContain('products'); + expect(entityKeys).toContain('discussions'); + expect(entityKeys).toContain('images'); + expect(entityKeys).toContain('videos'); + expect(entityKeys).toContain('events'); + expect(entityKeys).toContain('jobs'); + expect(entityKeys).toContain('lists'); + expect(entityKeys).toContain('organizations'); + expect(entityKeys).toContain('people'); + expect(entityKeys).toContain('crawlJobs'); + expect(entityKeys).toContain('bulkJobs'); + expect(entityKeys).toContain('customApis'); + expect(entityKeys).toContain('accounts'); + + for (const entity of Object.values(DiffbotSchema.entities)) { + expect(entity).toBeDefined(); + } + }); +}); diff --git a/packages/diffbot/schema/database.ts b/packages/diffbot/schema/database.ts new file mode 100644 index 000000000..24f765dee --- /dev/null +++ b/packages/diffbot/schema/database.ts @@ -0,0 +1,593 @@ +import { z } from 'zod'; + +/** + * DiffbotArticle — cached article entity. + * Represents structured article data extracted by Diffbot Article API / Knowledge Graph. + * @see https://docs.diffbot.com/docs/ontology/article + */ +export const DiffbotArticle = z.object({ + id: z.string().optional(), + type: z.literal('article').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + text: z.string().optional(), + html: z.string().optional(), + date: z.string().optional(), + estimatedDate: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + siteName: z.string().optional(), + humanLanguage: z.string().optional(), + numPages: z.number().optional(), + nextPage: z.string().optional(), + nextPages: z.array(z.string()).optional(), + images: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + videos: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + duration: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + tags: z + .array( + z + .object({ + id: z.number().optional(), + label: z.string(), + uri: z.string().optional(), + score: z.number().optional(), + types: z.array(z.string()).optional(), + }) + .passthrough(), + ) + .optional(), + links: z.array(z.string()).optional(), + breadcrumb: z + .array( + z.object({ link: z.string().optional(), name: z.string().optional() }), + ) + .optional(), + publisherRegion: z.string().optional(), + publisherCountry: z.string().optional(), + sentiment: z.number().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotArticle = z.infer; + +/** + * DiffbotProduct — cached product entity. + * Represents structured product data extracted by Diffbot Product API / Knowledge Graph. + * @see https://docs.diffbot.com/docs/ontology/product + */ +export const DiffbotProduct = z.object({ + id: z.string().optional(), + type: z.literal('product').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + text: z.string().optional(), + brand: z.string().optional(), + offerPrice: z.string().optional(), + offerPriceDetails: z + .object({ + amount: z.number().optional(), + symbol: z.string().optional(), + text: z.string().optional(), + }) + .passthrough() + .optional(), + regularPrice: z.string().optional(), + saveAmount: z.string().optional(), + shippingAmount: z.string().optional(), + availability: z.boolean().optional(), + sku: z.string().optional(), + mpn: z.string().optional(), + upc: z.string().optional(), + isbn: z.string().optional(), + images: z + .array( + z + .object({ + url: z.string().optional(), + title: z.string().optional(), + primary: z.boolean().optional(), + }) + .passthrough(), + ) + .optional(), + offers: z + .array( + z + .object({ + price: z.string().optional(), + priceCurrency: z.string().optional(), + seller: z.string().optional(), + availability: z.boolean().optional(), + }) + .passthrough(), + ) + .optional(), + colors: z.array(z.string()).optional(), + humanLanguage: z.string().optional(), + tags: z + .array( + z + .object({ + label: z.string(), + score: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotProduct = z.infer; + +/** + * DiffbotDiscussion — cached discussion thread entity. + * @see https://docs.diffbot.com/docs/ontology/discussion + */ +export const DiffbotDiscussion = z.object({ + id: z.string().optional(), + type: z.literal('discussion').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + numPosts: z.number().optional(), + numParticipants: z.number().optional(), + participants: z.array(z.string()).optional(), + rssUrl: z.string().optional(), + posts: z + .array( + z + .object({ + id: z.number().optional(), + text: z.string().optional(), + html: z.string().optional(), + author: z.string().optional(), + authorUrl: z.string().optional(), + date: z.string().optional(), + parentId: z.number().optional(), + voteCount: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotDiscussion = z.infer; + +/** + * DiffbotImage — cached image extraction entity. + * @see https://docs.diffbot.com/docs/ontology/image + */ +export const DiffbotImage = z.object({ + id: z.string().optional(), + type: z.literal('image').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + width: z.number().optional(), + height: z.number().optional(), + primary: z.boolean().optional(), + xpath: z.string().optional(), + attrTitle: z.string().optional(), + attrAlt: z.string().optional(), + caption: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotImage = z.infer; + +/** + * DiffbotVideo — cached video extraction entity. + * @see https://docs.diffbot.com/docs/ontology/video + */ +export const DiffbotVideo = z.object({ + id: z.string().optional(), + type: z.literal('video').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + url: z.string().optional(), + title: z.string().optional(), + naturalHeight: z.number().optional(), + naturalWidth: z.number().optional(), + duration: z.number().optional(), + viewCount: z.number().optional(), + uploadDate: z.string().optional(), + author: z.string().optional(), + embedUrl: z.string().optional(), + html: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotVideo = z.infer; + +/** + * DiffbotEvent — cached event entity. + * @see https://docs.diffbot.com/docs/ontology/event + */ +export const DiffbotEvent = z.object({ + id: z.string().optional(), + type: z.literal('event').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + startDate: z.string().optional(), + endDate: z.string().optional(), + location: z.string().optional(), + venue: z + .object({ + name: z.string().optional(), + address: z.string().optional(), + city: z.string().optional(), + state: z.string().optional(), + country: z.string().optional(), + }) + .passthrough() + .optional(), + organizer: z.string().optional(), + ticketUrl: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotEvent = z.infer; + +/** + * DiffbotJob — cached job posting entity. + * @see https://docs.diffbot.com/docs/ontology/jobpost + */ +export const DiffbotJob = z.object({ + id: z.string().optional(), + type: z.literal('job').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + company: z + .object({ + name: z.string().optional(), + url: z.string().optional(), + }) + .passthrough() + .optional(), + locations: z.array(z.string()).optional(), + employmentType: z.string().optional(), + compensation: z + .object({ + min: z.number().optional(), + max: z.number().optional(), + currency: z.string().optional(), + interval: z.string().optional(), + }) + .passthrough() + .optional(), + requirements: z.array(z.string()).optional(), + skills: z.array(z.string()).optional(), + postedDate: z.string().optional(), + humanLanguage: z.string().optional(), + diffbotUri: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotJob = z.infer; + +/** + * DiffbotList — cached list extraction entity. + * @see https://docs.diffbot.com/docs/extract/list + */ +export const DiffbotList = z.object({ + id: z.string().optional(), + type: z.literal('list').optional(), + pageUrl: z.string(), + resolvedPageUrl: z.string().optional(), + title: z.string().optional(), + numItems: z.number().optional(), + items: z + .array( + z + .object({ + title: z.string().optional(), + link: z.string().optional(), + description: z.string().optional(), + image: z.string().optional(), + price: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + humanLanguage: z.string().optional(), + extractedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotList = z.infer; + +/** + * DiffbotOrganization — Knowledge Graph organization entity. + * @see https://docs.diffbot.com/docs/ontology/organization + */ +export const DiffbotOrganization = z.object({ + id: z.string(), + name: z.string(), + type: z.literal('Organization').optional(), + types: z.array(z.string()).optional(), + diffbotUri: z.string().optional(), + homepageUri: z.string().optional(), + description: z.string().optional(), + summary: z.string().optional(), + logo: z.string().optional(), + image: z.string().optional(), + images: z.array(z.string()).optional(), + nbEmployees: z.number().optional(), + nbEmployeesMin: z.number().optional(), + nbEmployeesMax: z.number().optional(), + revenue: z.number().optional(), + yearlyRevenues: z + .array( + z + .object({ + year: z.number().optional(), + revenue: z.number().optional(), + }) + .passthrough(), + ) + .optional(), + isPublic: z.boolean().optional(), + isNonProfit: z.boolean().optional(), + isAcquired: z.boolean().optional(), + isDissolved: z.boolean().optional(), + founders: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + ceo: z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough() + .optional(), + boardMembers: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + competitors: z + .array( + z + .object({ + name: z.string().optional(), + id: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + totalInvestment: z.number().optional(), + location: z + .object({ + city: z.string().optional(), + region: z.string().optional(), + country: z.string().optional(), + address: z.string().optional(), + }) + .passthrough() + .optional(), + locations: z.array(z.record(z.string(), z.unknown())).optional(), + emailAddresses: z.array(z.string()).optional(), + phoneNumbers: z.array(z.string()).optional(), + linkedInUri: z.string().optional(), + twitterUri: z.string().optional(), + facebookUri: z.string().optional(), + githubUri: z.string().optional(), + wikipediaUri: z.string().optional(), + crunchbaseUri: z.string().optional(), + angellistUri: z.string().optional(), + categories: z.array(z.string()).optional(), + industries: z.array(z.string()).optional(), + naicsClassification: z.string().optional(), + sicClassification: z.string().optional(), + naceClassification: z.string().optional(), + crawlTimestamp: z.number().optional(), + importance: z.number().optional(), + origin: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotOrganization = z.infer; + +/** + * DiffbotPerson — Knowledge Graph person entity. + * @see https://docs.diffbot.com/docs/ontology/person + */ +export const DiffbotPerson = z.object({ + id: z.string(), + name: z.string(), + type: z.literal('Person').optional(), + types: z.array(z.string()).optional(), + diffbotUri: z.string().optional(), + description: z.string().optional(), + summary: z.string().optional(), + image: z.string().optional(), + images: z.array(z.string()).optional(), + gender: z.string().optional(), + birthDate: z.string().optional(), + deathDate: z.string().optional(), + educations: z.array(z.record(z.string(), z.unknown())).optional(), + employments: z.array(z.record(z.string(), z.unknown())).optional(), + awards: z.array(z.record(z.string(), z.unknown())).optional(), + skills: z.array(z.string()).optional(), + interests: z.array(z.string()).optional(), + location: z + .object({ + city: z.string().optional(), + region: z.string().optional(), + country: z.string().optional(), + }) + .passthrough() + .optional(), + emailAddresses: z.array(z.string()).optional(), + phoneNumbers: z.array(z.string()).optional(), + linkedInUri: z.string().optional(), + twitterUri: z.string().optional(), + facebookUri: z.string().optional(), + githubUri: z.string().optional(), + wikipediaUri: z.string().optional(), + crunchbaseUri: z.string().optional(), + angellistUri: z.string().optional(), + homepageUri: z.string().optional(), + crawlTimestamp: z.number().optional(), + importance: z.number().optional(), + origin: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotPerson = z.infer; + +/** + * DiffbotCrawlJob — Crawl job record. + * @see https://docs.diffbot.com/docs/crawl/ + */ +export const DiffbotCrawlJob = z.object({ + name: z.string(), + jobStatus: z + .object({ + status: z.number().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + sentToCrawler: z.number().optional(), + objectsHarvested: z.number().optional(), + urlsHarvested: z.number().optional(), + pageRounds: z.number().optional(), + maxRounds: z.number().optional(), + maxHops: z.number().optional(), + pause: z.number().optional(), + roundProxy: z.number().optional(), + seeds: z.string().optional(), + apiUrl: z.string().optional(), + downloadUrl: z.string().optional(), + maxTags: z.number().optional(), + status: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotCrawlJob = z.infer; + +/** + * DiffbotBulkJob — Bulk extract or Bulk enhance job record. + * @see https://docs.diffbot.com/docs/bulk/ + */ +export const DiffbotBulkJob = z.object({ + id: z.string(), + name: z.string().optional(), + bulkjobId: z.string().optional(), + kind: z.enum(['extract', 'enhance']).optional(), + status: z.string().optional(), + jobStatus: z + .object({ + status: z.number().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + total: z.number().optional(), + completed: z.number().optional(), + failed: z.number().optional(), + format: z.string().optional(), + apiUrl: z.string().optional(), + urls: z.string().optional(), + downloadUrl: z.string().optional(), + createdAt: z.string().optional(), + updatedAt: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotBulkJob = z.infer; + +/** + * DiffbotCustomApi — Custom API extraction configuration. + * @see https://docs.diffbot.com/docs/custom-api/ + */ +export const DiffbotCustomApi = z.object({ + id: z.string(), + api: z.string(), + url: z.string(), + pattern: z.string().optional(), + ruleset: z.record(z.string(), z.unknown()).optional(), + selectors: z.record(z.string(), z.unknown()).optional(), + testUrl: z.string().optional(), + createdAt: z.string().optional(), + updatedAt: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotCustomApi = z.infer; + +/** + * DiffbotAccount — Account details and API quota. + * @see https://docs.diffbot.com/docs/account + */ +export const DiffbotAccount = z.object({ + id: z.string(), + token: z.string(), + name: z.string().optional(), + email: z.string().optional(), + plan: z.string().optional(), + planStart: z.string().optional(), + planCalls: z.number().optional(), + apiCalls: z.number().optional(), + status: z.string().optional(), + fetchedAt: z.coerce.date().nullable().optional(), +}); + +export type DiffbotAccount = z.infer; diff --git a/packages/diffbot/schema/index.ts b/packages/diffbot/schema/index.ts new file mode 100644 index 000000000..cf55a97e2 --- /dev/null +++ b/packages/diffbot/schema/index.ts @@ -0,0 +1,38 @@ +import { + DiffbotAccount, + DiffbotArticle, + DiffbotBulkJob, + DiffbotCrawlJob, + DiffbotCustomApi, + DiffbotDiscussion, + DiffbotEvent, + DiffbotImage, + DiffbotJob, + DiffbotList, + DiffbotOrganization, + DiffbotPerson, + DiffbotProduct, + DiffbotVideo, +} from './database'; + +export const DiffbotSchema = { + version: '1.0.0', + entities: { + articles: DiffbotArticle, + products: DiffbotProduct, + discussions: DiffbotDiscussion, + images: DiffbotImage, + videos: DiffbotVideo, + events: DiffbotEvent, + jobs: DiffbotJob, + lists: DiffbotList, + organizations: DiffbotOrganization, + people: DiffbotPerson, + crawlJobs: DiffbotCrawlJob, + bulkJobs: DiffbotBulkJob, + customApis: DiffbotCustomApi, + accounts: DiffbotAccount, + }, +} as const; + +export * from './database'; diff --git a/packages/diffbot/tsconfig.json b/packages/diffbot/tsconfig.json new file mode 100644 index 000000000..15e507a13 --- /dev/null +++ b/packages/diffbot/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["esnext"], + "types": ["node", "jest"], + "module": "ESNext", + "moduleResolution": "Bundler", + "outDir": "./dist", + "rootDir": "./", + "composite": true, + "incremental": true, + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["./**/*"], + "exclude": ["dist", "node_modules"], + "references": [] +} diff --git a/packages/diffbot/tsup.config.ts b/packages/diffbot/tsup.config.ts new file mode 100644 index 000000000..3ec221e23 --- /dev/null +++ b/packages/diffbot/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + clean: false, + dts: false, + format: ['esm'], + target: 'esnext', + platform: 'node', + bundle: true, + splitting: true, + minify: true, + outDir: 'dist', + external: ['corsair', 'zod'], + entry: ['index.ts'], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 461835684..fd3d10ddc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2428,6 +2428,30 @@ importers: specifier: 4.4.3 version: 4.4.3 + packages/diffbot: + devDependencies: + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + corsair: + specifier: workspace:* + version: link:../corsair + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)) + ts-jest: + specifier: ^29.4.9 + version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@30.4.1)(babel-jest@29.7.0(@babel/core@7.29.7))(esbuild@0.27.0)(jest-util@30.4.1)(jest@29.7.0(@types/node@24.10.1)(ts-node@10.9.2(@types/node@24.10.1)(typescript@5.9.3)))(typescript@5.9.3) + tsup: + specifier: ^8.0.1 + version: 8.5.1(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.9.0) + typescript: + specifier: 'catalog:' + version: 5.9.3 + zod: + specifier: 4.4.3 + version: 4.4.3 + packages/digitalocean: devDependencies: '@types/jest': @@ -5137,6 +5161,7 @@ importers: zod: specifier: 4.4.3 version: 4.4.3 + packages/zoominfo: devDependencies: '@types/jest':