From 621c968f3f8bb46940c3bfb70468eac589b23563 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 14:00:43 -0700 Subject: [PATCH 01/27] Hold Shift to delete without confirmation: fixes #537 (cherry picked from commit 002df7b0f9ab73f60bcbf4f560a5cdc717ffc025) --- src/apps/chat/components/ChatDrawer.tsx | 2 +- src/apps/chat/components/ChatDrawerItem.tsx | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/apps/chat/components/ChatDrawer.tsx b/src/apps/chat/components/ChatDrawer.tsx index 4692eee0a..ccc998411 100644 --- a/src/apps/chat/components/ChatDrawer.tsx +++ b/src/apps/chat/components/ChatDrawer.tsx @@ -310,7 +310,7 @@ function ChatDrawer(props: { bottomBarBasis={filteredChatsBarBasis} onConversationActivate={handleConversationActivate} onConversationBranch={onConversationBranch} - onConversationDelete={handleConversationDeleteNoConfirmation} + onConversationDeleteNoConfirmation={handleConversationDeleteNoConfirmation} onConversationExport={onConversationsExportDialog} onConversationFolderChange={handleConversationFolderChange} /> diff --git a/src/apps/chat/components/ChatDrawerItem.tsx b/src/apps/chat/components/ChatDrawerItem.tsx index cf2c83cfc..b99d9f17a 100644 --- a/src/apps/chat/components/ChatDrawerItem.tsx +++ b/src/apps/chat/components/ChatDrawerItem.tsx @@ -42,7 +42,7 @@ export const ChatDrawerItemMemo = React.memo(ChatDrawerItem, (prev, next) => prev.bottomBarBasis === next.bottomBarBasis && prev.onConversationActivate === next.onConversationActivate && prev.onConversationBranch === next.onConversationBranch && - prev.onConversationDelete === next.onConversationDelete && + prev.onConversationDeleteNoConfirmation === next.onConversationDeleteNoConfirmation && prev.onConversationExport === next.onConversationExport && prev.onConversationFolderChange === next.onConversationFolderChange, ); @@ -76,7 +76,7 @@ function ChatDrawerItem(props: { bottomBarBasis: number, onConversationActivate: (conversationId: DConversationId, closeMenu: boolean) => void, onConversationBranch: (conversationId: DConversationId, messageId: string | null) => void, - onConversationDelete: (conversationId: DConversationId) => void, + onConversationDeleteNoConfirmation: (conversationId: DConversationId) => void, onConversationExport: (conversationId: DConversationId, exportAll: boolean) => void, onConversationFolderChange: (folderChangeRequest: FolderChangeRequest) => void, }) { @@ -155,7 +155,16 @@ function ChatDrawerItem(props: { // Delete - const handleDeleteButtonShow = React.useCallback(() => setDeleteArmed(true), []); + const { onConversationDeleteNoConfirmation } = props; + const handleDeleteButtonShow = React.useCallback((event: React.MouseEvent) => { + // special case: if 'Shift' is pressed, delete immediately + if (event.shiftKey) { + event.stopPropagation(); + onConversationDeleteNoConfirmation(conversationId); + return; + } + setDeleteArmed(true); + }, [conversationId, onConversationDeleteNoConfirmation]); const handleDeleteButtonHide = React.useCallback(() => setDeleteArmed(false), []); @@ -163,9 +172,9 @@ function ChatDrawerItem(props: { if (deleteArmed) { setDeleteArmed(false); event.stopPropagation(); - props.onConversationDelete(conversationId); + onConversationDeleteNoConfirmation(conversationId); } - }, [conversationId, deleteArmed, props]); + }, [conversationId, deleteArmed, onConversationDeleteNoConfirmation]); const textSymbol = SystemPurposes[systemPurposeId]?.symbol || 'โ“'; From f4b39071f06c3e391e3bf8cf9049448b847805b7 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 14:34:25 -0700 Subject: [PATCH 02/27] Browse: support transform (skel) (cherry picked from commit 51b6e30986177f9b7fa40c19ffcb926efee6c66f) --- src/modules/browse/browse.client.ts | 5 ++++- src/modules/browse/browse.router.ts | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/modules/browse/browse.client.ts b/src/modules/browse/browse.client.ts index 8776177c8..785066b18 100644 --- a/src/modules/browse/browse.client.ts +++ b/src/modules/browse/browse.client.ts @@ -26,7 +26,10 @@ export async function callBrowseFetchPage(url: string) { dialect: 'browse-wss', ...(!!clientWssEndpoint && { wssEndpoint: clientWssEndpoint }), }, - subjects: [{ url }], + subjects: [{ + url, + transform: 'markdown', + }], screenshot: DEBUG_SHOW_SCREENSHOT ? { width: 512, height: 512, diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 063391cb4..435eae49d 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -17,10 +17,14 @@ const browseAccessSchema = z.object({ wssEndpoint: z.string().trim().optional(), }); +const pageTransformSchema = z.enum(['html', 'text', 'markdown']); +type PageTransformSchema = z.infer; + const fetchPageInputSchema = z.object({ access: browseAccessSchema, subjects: z.array(z.object({ url: z.string().url(), + transform: pageTransformSchema, })), screenshot: z.object({ width: z.number(), @@ -60,7 +64,7 @@ export const browseRouter = createTRPCRouter({ for (const subject of subjects) { try { - pages.push(await workerPuppeteer(access, subject.url, screenshot?.width, screenshot?.height, screenshot?.quality)); + pages.push(await workerPuppeteer(access, subject.url, subject.transform, screenshot?.width, screenshot?.height, screenshot?.quality)); } catch (error: any) { pages.push({ url: subject.url, @@ -80,7 +84,11 @@ export const browseRouter = createTRPCRouter({ type BrowseAccessSchema = z.infer; type FetchPageWorkerOutputSchema = z.infer; -async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, ssWidth: number | undefined, ssHeight: number | undefined, ssQuality: number | undefined): Promise { + +/** + * Puppeteer implementation of the worker + */ +async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, transform: PageTransformSchema, ssWidth: number | undefined, ssHeight: number | undefined, ssQuality: number | undefined): Promise { // access const browserWSEndpoint = (access.wssEndpoint || env.PUPPETEER_WSS_ENDPOINT || '').trim(); From 6f82e2c3edc57375df2ea456b149c55c2799565c Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 14:49:29 -0700 Subject: [PATCH 03/27] Browse: markdown transform as default (cherry picked from commit 7946cd661462cba9cbfc40f50d5aa36e6cb96539) --- package-lock.json | 21 +++++++++++++++++++++ package.json | 2 ++ src/modules/browse/browse.router.ts | 28 +++++++++++++++++++++------- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 734a60c91..43d2c4409 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "superjson": "^2.2.1", "tesseract.js": "^5.1.0", "tiktoken": "^1.0.14", + "turndown": "^7.1.3", "uuid": "^9.0.1", "zod": "^3.23.8", "zustand": "^4.5.2" @@ -68,6 +69,7 @@ "@types/react-dom": "^18.3.0", "@types/react-katex": "^3.0.4", "@types/react-timeago": "^4.1.7", + "@types/turndown": "^5.0.4", "@types/uuid": "^9.0.8", "eslint": "^8.57.0", "eslint-config-next": "^14.2.3", @@ -2082,6 +2084,12 @@ "@types/react": "*" } }, + "node_modules/@types/turndown": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.4.tgz", + "integrity": "sha512-28GI33lCCkU4SGH1GvjDhFgOVr+Tym4PXGBIU1buJUa6xQolniPArtUT+kv42RR2N9MsMLInkr904Aq+ESHBJg==", + "dev": true + }, "node_modules/@types/unist": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", @@ -3214,6 +3222,11 @@ "csstype": "^3.0.2" } }, + "node_modules/domino": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", + "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -8269,6 +8282,14 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, + "node_modules/turndown": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz", + "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==", + "dependencies": { + "domino": "^2.1.6" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index d02516f6d..90e2dae3b 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "superjson": "^2.2.1", "tesseract.js": "^5.1.0", "tiktoken": "^1.0.14", + "turndown": "^7.1.3", "uuid": "^9.0.1", "zod": "^3.23.8", "zustand": "^4.5.2" @@ -77,6 +78,7 @@ "@types/react-dom": "^18.3.0", "@types/react-katex": "^3.0.4", "@types/react-timeago": "^4.1.7", + "@types/turndown": "^5.0.4", "@types/uuid": "^9.0.8", "eslint": "^8.57.0", "eslint-config-next": "^14.2.3", diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 435eae49d..689d33c4e 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -1,7 +1,10 @@ import { z } from 'zod'; import { TRPCError } from '@trpc/server'; + import { BrowserContext, connect, ScreenshotOptions, TimeoutError } from '@cloudflare/puppeteer'; +import TurndownService from 'turndown'; + import { createTRPCRouter, publicProcedure } from '~/server/api/trpc.server'; import { env } from '~/server/env.mjs'; @@ -125,8 +128,9 @@ async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, tr if (!isWebPage) { // noinspection ExceptionCaughtLocallyJS throw new Error(`Invalid content-type: ${contentType}`); - } else + } else { result.stopReason = 'end'; + } } catch (error: any) { const isTimeout: boolean = error instanceof TimeoutError; result.stopReason = isTimeout ? 'timeout' : 'error'; @@ -137,12 +141,22 @@ async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, tr // transform the content of the page as text try { if (result.stopReason !== 'error') { - result.content = await page.evaluate(() => { - const content = document.body.innerText || document.textContent; - if (!content) - throw new Error('No content'); - return content; - }); + switch (transform) { + case 'html': + result.content = await page.evaluate(() => document.documentElement.innerHTML); + break; + case 'markdown': + const html = await page.evaluate(() => document.documentElement.innerHTML); + const turndownService = new TurndownService(); + result.content = turndownService.turndown(html); + break; + case 'text': + default: + result.content = await page.evaluate(() => document.body.innerText || document.textContent || ''); + break; + } + if (!result.content) + result.error = '[Puppeteer] Empty content'; } } catch (error: any) { result.error = '[Puppeteer] ' + error?.message || error?.toString() || 'Unknown evaluate error'; From ad24c8771aa750a51c594cc1837f2af9a3456856 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 15:38:08 -0700 Subject: [PATCH 04/27] Browse: full support for markdown transform (cherry picked from commit 7b07bb788449f19bf1105580aecd4d83ab9feaca) --- src/modules/browse/BrowseSettings.tsx | 73 ++++++++++++++------ src/modules/browse/browse.client.ts | 8 +-- src/modules/browse/browse.router.ts | 46 +++++++----- src/modules/browse/store-module-browsing.tsx | 8 +++ 4 files changed, 94 insertions(+), 41 deletions(-) diff --git a/src/modules/browse/BrowseSettings.tsx b/src/modules/browse/BrowseSettings.tsx index 40215394b..0bed7ea5c 100644 --- a/src/modules/browse/BrowseSettings.tsx +++ b/src/modules/browse/BrowseSettings.tsx @@ -1,62 +1,95 @@ import * as React from 'react'; -import { shallow } from 'zustand/shallow'; +import { useShallow } from 'zustand/react/shallow'; -import { Checkbox, FormControl, FormHelperText } from '@mui/joy'; +import { Checkbox, FormControl, FormHelperText, Option, Select, Typography } from '@mui/joy'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; -import { Link } from '~/common/components/Link'; import { platformAwareKeystrokes } from '~/common/components/KeyStroke'; import { useBrowseCapability, useBrowseStore } from './store-module-browsing'; +import { ExternalLink } from '~/common/components/ExternalLink'; +import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; export function BrowseSettings() { // external state - const { mayWork, isServerConfig, isClientValid, inCommand, inComposer, inReact } = useBrowseCapability(); - const { wssEndpoint, setWssEndpoint, setEnableCommandBrowse, setEnableComposerAttach, setEnableReactTool } = useBrowseStore(state => ({ + const { mayWork, isServerConfig, isClientValid, inCommand, inComposer, inReact, inPersonas } = useBrowseCapability(); + const { + wssEndpoint, setWssEndpoint, + pageTransform, setPageTransform, + setEnableCommandBrowse, setEnableComposerAttach, setEnableReactTool, setEnablePersonaTool, + } = useBrowseStore(useShallow(state => ({ wssEndpoint: state.wssEndpoint, + pageTransform: state.pageTransform, + setPageTransform: state.setPageTransform, setWssEndpoint: state.setWssEndpoint, setEnableCommandBrowse: state.setEnableCommandBrowse, setEnableComposerAttach: state.setEnableComposerAttach, setEnableReactTool: state.setEnableReactTool, - }), shallow); + setEnablePersonaTool: state.setEnablePersonaTool, + }))); + + const handlePageTransformChange = (_event: any, value: typeof pageTransform | null) => value && setPageTransform(value); + return <> - - Configure a browsing service to enable loading links and pages. See the - browse configuration guide for more information. - + + Configure Browsing to enable loading links and web pages. + Learn more. + + + + + + + + + Browsing enablement: + - setEnableComposerAttach(event.target.checked)} /> - {platformAwareKeystrokes('Load and attach a page when pasting a URL')} + setEnableComposerAttach(event.target.checked)} /> + {platformAwareKeystrokes('Load and attach when pasting a URL')} - setEnableCommandBrowse(event.target.checked)} /> + setEnableCommandBrowse(event.target.checked)} /> {platformAwareKeystrokes('Use /browse to load a web page')} - setEnableReactTool(event.target.checked)} /> + setEnableReactTool(event.target.checked)} /> Enables loadURL() in ReAct - {/**/} - {/* setEnablePersonaTool(event.target.checked)} />*/} - {/* Enable loading URLs by Personas*/} - {/**/} + + setEnablePersonaTool(event.target.checked)} /> + Not yet available + {/*Enable loading URLs by Personas*/} + ; } \ No newline at end of file diff --git a/src/modules/browse/browse.client.ts b/src/modules/browse/browse.client.ts index 785066b18..806aa6de5 100644 --- a/src/modules/browse/browse.client.ts +++ b/src/modules/browse/browse.client.ts @@ -1,4 +1,4 @@ -import { useBrowseStore } from '~/modules/browse/store-module-browsing'; +import { BrowsePageTransform, useBrowseStore } from '~/modules/browse/store-module-browsing'; import { apiAsyncNode } from '~/common/util/trpc.client'; @@ -7,7 +7,7 @@ import { apiAsyncNode } from '~/common/util/trpc.client'; const DEBUG_SHOW_SCREENSHOT = false; -export async function callBrowseFetchPage(url: string) { +export async function callBrowseFetchPage(url: string, forceTransform?: BrowsePageTransform) { // thow if no URL is provided url = url?.trim() || ''; @@ -19,7 +19,7 @@ export async function callBrowseFetchPage(url: string) { if (!url.startsWith('http://') && !url.startsWith('https://')) url = 'https://' + url; - const clientWssEndpoint = useBrowseStore.getState().wssEndpoint; + const { wssEndpoint: clientWssEndpoint, pageTransform } = useBrowseStore.getState(); const { pages } = await apiAsyncNode.browse.fetchPages.mutate({ access: { @@ -28,7 +28,7 @@ export async function callBrowseFetchPage(url: string) { }, subjects: [{ url, - transform: 'markdown', + transform: pageTransform || 'text', }], screenshot: DEBUG_SHOW_SCREENSHOT ? { width: 512, diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 689d33c4e..16a566151 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -3,6 +3,9 @@ import { TRPCError } from '@trpc/server'; import { BrowserContext, connect, ScreenshotOptions, TimeoutError } from '@cloudflare/puppeteer'; +/** + * Puppeteer implementation of the worker + */ import TurndownService from 'turndown'; import { createTRPCRouter, publicProcedure } from '~/server/api/trpc.server'; @@ -88,12 +91,14 @@ type BrowseAccessSchema = z.infer; type FetchPageWorkerOutputSchema = z.infer; -/** - * Puppeteer implementation of the worker - */ -async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, transform: PageTransformSchema, ssWidth: number | undefined, ssHeight: number | undefined, ssQuality: number | undefined): Promise { - - // access +async function workerPuppeteer( + access: BrowseAccessSchema, + targetUrl: string, + transform: PageTransformSchema, + ssWidth: number | undefined, + ssHeight: number | undefined, + ssQuality: number | undefined, +): Promise { const browserWSEndpoint = (access.wssEndpoint || env.PUPPETEER_WSS_ENDPOINT || '').trim(); const isLocalBrowser = browserWSEndpoint.startsWith('ws://'); if (!browserWSEndpoint || (!browserWSEndpoint.startsWith('wss://') && !isLocalBrowser)) @@ -132,10 +137,11 @@ async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, tr result.stopReason = 'end'; } } catch (error: any) { - const isTimeout: boolean = error instanceof TimeoutError; + const isTimeout = error instanceof TimeoutError; result.stopReason = isTimeout ? 'timeout' : 'error'; - if (!isTimeout) - result.error = '[Puppeteer] ' + error?.message || error?.toString() || 'Unknown goto error'; + if (!isTimeout) { + result.error = '[Puppeteer] ' + (error?.message || error?.toString() || 'Unknown goto error'); + } } // transform the content of the page as text @@ -143,23 +149,29 @@ async function workerPuppeteer(access: BrowseAccessSchema, targetUrl: string, tr if (result.stopReason !== 'error') { switch (transform) { case 'html': - result.content = await page.evaluate(() => document.documentElement.innerHTML); - break; - case 'markdown': - const html = await page.evaluate(() => document.documentElement.innerHTML); - const turndownService = new TurndownService(); - result.content = turndownService.turndown(html); + result.content = await page.content(); break; case 'text': - default: result.content = await page.evaluate(() => document.body.innerText || document.textContent || ''); break; + case 'markdown': + await page.evaluate(() => { + // Remove unnecessary elements + document.querySelectorAll('script, style, nav, footer, aside, header, .ads, .comments') + .forEach(el => el.remove()); + }); + const cleanedHtml = await page.content(); + const turndownService = new TurndownService({ + headingStyle: 'atx', + }); + result.content = turndownService.turndown(cleanedHtml); + break; } if (!result.content) result.error = '[Puppeteer] Empty content'; } } catch (error: any) { - result.error = '[Puppeteer] ' + error?.message || error?.toString() || 'Unknown evaluate error'; + result.error = '[Puppeteer] ' + (error?.message || error?.toString() || 'Unknown evaluate error'); } // get a screenshot of the page diff --git a/src/modules/browse/store-module-browsing.tsx b/src/modules/browse/store-module-browsing.tsx index c6869d3fb..8a0a2f93d 100644 --- a/src/modules/browse/store-module-browsing.tsx +++ b/src/modules/browse/store-module-browsing.tsx @@ -5,11 +5,16 @@ import { CapabilityBrowsing } from '~/common/components/useCapabilities'; import { getBackendCapabilities } from '~/modules/backend/store-backend-capabilities'; +export type BrowsePageTransform = 'html' | 'text' | 'markdown'; + interface BrowseState { wssEndpoint: string; setWssEndpoint: (url: string) => void; + pageTransform: BrowsePageTransform; + setPageTransform: (transform: BrowsePageTransform) => void; + enableCommandBrowse: boolean; setEnableCommandBrowse: (value: boolean) => void; @@ -31,6 +36,9 @@ export const useBrowseStore = create()( wssEndpoint: '', // default WSS endpoint setWssEndpoint: (wssEndpoint: string) => set(() => ({ wssEndpoint })), + pageTransform: 'text', + setPageTransform: (pageTransform: BrowsePageTransform) => set(() => ({ pageTransform })), + enableCommandBrowse: true, setEnableCommandBrowse: (enableCommandBrowse: boolean) => set(() => ({ enableCommandBrowse })), From f751c91c68053b847bc2af873a14c302a0caae61 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 15:59:46 -0700 Subject: [PATCH 05/27] Browse: improve markdown transform (cherry picked from commit 3623eef47fdd7e0f2755a7a1009afc4aa8a9bb01) --- package-lock.json | 160 ++++++++++++++++++++++++++++ package.json | 1 + src/modules/browse/browse.router.ts | 51 ++++++--- 3 files changed, 198 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 43d2c4409..ad4c4762d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "@vercel/analytics": "^1.2.2", "@vercel/speed-insights": "^1.0.10", "browser-fs-access": "^0.35.0", + "cheerio": "^1.0.0-rc.12", "eventsource-parser": "^1.1.2", "idb-keyval": "^6.2.1", "next": "~14.1.4", @@ -2671,6 +2672,11 @@ "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==" }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2836,6 +2842,42 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chownr": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", @@ -2994,6 +3036,32 @@ "tiny-invariant": "^1.0.6" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", @@ -3222,11 +3290,62 @@ "csstype": "^3.0.2" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, "node_modules/domino": { "version": "2.1.6", "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -4673,6 +4792,24 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -6559,6 +6696,17 @@ "resolved": "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz", "integrity": "sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==" }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -6818,6 +6966,18 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", diff --git a/package.json b/package.json index 90e2dae3b..1961b2ed8 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@vercel/analytics": "^1.2.2", "@vercel/speed-insights": "^1.0.10", "browser-fs-access": "^0.35.0", + "cheerio": "^1.0.0-rc.12", "eventsource-parser": "^1.1.2", "idb-keyval": "^6.2.1", "next": "~14.1.4", diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 16a566151..280b2e7bd 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -2,11 +2,8 @@ import { z } from 'zod'; import { TRPCError } from '@trpc/server'; import { BrowserContext, connect, ScreenshotOptions, TimeoutError } from '@cloudflare/puppeteer'; - -/** - * Puppeteer implementation of the worker - */ -import TurndownService from 'turndown'; +import { default as TurndownService } from 'turndown'; +import { load as cheerioLoad } from 'cheerio'; import { createTRPCRouter, publicProcedure } from '~/server/api/trpc.server'; import { env } from '~/server/env.mjs'; @@ -155,15 +152,9 @@ async function workerPuppeteer( result.content = await page.evaluate(() => document.body.innerText || document.textContent || ''); break; case 'markdown': - await page.evaluate(() => { - // Remove unnecessary elements - document.querySelectorAll('script, style, nav, footer, aside, header, .ads, .comments') - .forEach(el => el.remove()); - }); - const cleanedHtml = await page.content(); - const turndownService = new TurndownService({ - headingStyle: 'atx', - }); + const html = await page.content(); + const cleanedHtml = cleanHtml(html); + const turndownService = new TurndownService({ headingStyle: 'atx' }); result.content = turndownService.turndown(cleanedHtml); break; } @@ -226,3 +217,35 @@ async function workerPuppeteer( return result; } + + +function cleanHtml(html: string) { + const $ = cheerioLoad(html); + + // Remove standard unwanted elements + $('script, style, nav, aside, noscript, iframe, svg, canvas, .ads, .comments, link[rel="stylesheet"]').remove(); + + // Remove elements that might be specific to proxy services or injected by them + $('[id^="brightdata-"], [class^="brightdata-"]').remove(); + + // Remove comments + $('*').contents().filter(function() { + return this.type === 'comment'; + }).remove(); + + // Remove empty elements + $('p, div, span').each(function() { + if ($(this).text().trim() === '' && $(this).children().length === 0) { + $(this).remove(); + } + }); + + // Merge consecutive paragraphs + $('p + p').each(function() { + $(this).prev().append(' ' + $(this).text()); + $(this).remove(); + }); + + // Return the cleaned HTML + return $.html(); +} \ No newline at end of file From 463ea35d7ca651b8a80c35b953f3e9abc20ff863 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 16:24:30 -0700 Subject: [PATCH 06/27] Default to the full context window (cherry picked from commit fd30baafb821f434c1bd7e8a8a7b3dd2dfe93b9f) --- src/modules/llms/llm.client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index 92a6f2cb6..55b70dd09 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -69,7 +69,7 @@ function modelDescriptionToDLLMOpenAIOptions(model: M // null means unknown contenxt/output tokens const contextTokens = model.contextWindow || null; const maxOutputTokens = model.maxCompletionTokens || (contextTokens ? Math.round(contextTokens / 2) : null); - const llmResponseTokensRatio = model.maxCompletionTokens ? 1 / 2 : 1 / 4; + const llmResponseTokensRatio = model.maxCompletionTokens ? 1 : 1 / 4; const llmResponseTokens = maxOutputTokens ? Math.round(maxOutputTokens * llmResponseTokensRatio) : null; return { From e1a723a39fb295282bc3363fdcea5026452ac657 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Mon, 13 May 2024 16:24:42 -0700 Subject: [PATCH 07/27] (bits) (cherry picked from commit 2db74867f591c8cc781ce27e0fbd0cb691e01a6d) --- src/apps/news/news.data.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/apps/news/news.data.tsx b/src/apps/news/news.data.tsx index 11792a855..30f5f6fba 100644 --- a/src/apps/news/news.data.tsx +++ b/src/apps/news/news.data.tsx @@ -52,11 +52,12 @@ interface NewsItem { // news and feature surfaces export const NewsItems: NewsItem[] = [ /*{ - versionCode: '1.16.0', + versionCode: '1.17.0', items: [ + Screen Capture (when removed from labs) + Auto-Merge Draw ... - Screen Capture (when removed from labs) ] }*/ { From 54caa3e01add2eb35481f3643aca9701e963f5ec Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Tue, 14 May 2024 15:15:53 -0700 Subject: [PATCH 08/27] Gemini: improve support (incl. interfaces, cost, visibility) (cherry picked from commit 9eb0cc0b62a66f06f03c65ff648fc0e2b663d635) --- .../llms/server/gemini/gemini.models.ts | 137 ++++++++++++++++-- .../llms/server/gemini/gemini.router.ts | 2 +- src/modules/llms/server/openai/models.data.ts | 8 +- 3 files changed, 129 insertions(+), 18 deletions(-) diff --git a/src/modules/llms/server/gemini/gemini.models.ts b/src/modules/llms/server/gemini/gemini.models.ts index 703ee1dec..4ec9402a6 100644 --- a/src/modules/llms/server/gemini/gemini.models.ts +++ b/src/modules/llms/server/gemini/gemini.models.ts @@ -1,16 +1,117 @@ import type { GeminiModelSchema } from './gemini.wiretypes'; import type { ModelDescriptionSchema } from '../llm.server.types'; -import { LLM_IF_OAI_Chat, LLM_IF_OAI_Vision } from '../../store-llms'; +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Json, LLM_IF_OAI_Vision } from '../../store-llms'; +// supported interfaces +const geminiChatInterfaces: GeminiModelSchema['supportedGenerationMethods'] = ['generateContent']; + +// unsupported interfaces const filterUnallowedNames = ['Legacy']; const filterUnallowedInterfaces: GeminiModelSchema['supportedGenerationMethods'] = ['generateAnswer', 'embedContent', 'embedText']; -const geminiLinkModels = ['models/gemini-pro', 'models/gemini-pro-vision']; -// interfaces mapping -const geminiChatInterfaces: GeminiModelSchema['supportedGenerationMethods'] = ['generateContent']; -const geminiVisionNames = ['-vision']; +/* Manual models details + Gemini Name Mapping example: + - Latest version gemini-1.0-pro-latest ---latest + - Latest stable version gemini-1.0-pro -- + - Stable versions gemini-1.0-pro-001 --- +*/ +const _knownGeminiModels: ({ + id: string, + isLatest?: boolean, + isPreview?: boolean + symLink?: string +} & Pick)[] = [ + + // Generation 1.5 + { + id: 'models/gemini-1.5-flash-latest', + isLatest: true, + isPreview: true, + pricing: { + chatIn: 0.70, // 0.35 up to 128k tokens, 0.70 prompts > 128k tokens + chatOut: 1.05, // 0.53 up to 128k tokens, 1.05 prompts > 128k tokens + }, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text + }, + { + id: 'models/gemini-1.5-pro-latest', + // NOTE: no 'models/gemini-1.5-pro' (latest stable) as of 2024-05-14 + isLatest: true, + pricing: { + chatIn: 7.00, // $3.50 / 1 million tokens (for prompts up to 128K tokens), $7.00 / 1 million tokens (for prompts longer than 128K) + chatOut: 21.00, // $10.50 / 1 million tokens (128K or less), $21.00 / 1 million tokens (128K+) + }, + trainingDataCutoff: 'Apr 2024', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text + }, + + + // Generation 1.0 + { + id: 'models/gemini-1.0-pro-latest', + isLatest: true, + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat], + }, + { + id: 'models/gemini-1.0-pro', + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + { + id: 'models/gemini-1.0-pro-001', + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + + // Generation 1.0 + Vision + { + id: 'models/gemini-1.0-pro-vision-latest', + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision], // Text and Images + hidden: true, + }, + + // Older symlinks + { + id: 'models/gemini-pro', + symLink: 'models/gemini-1.0-pro', + // copied from symlinked + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + { + id: 'models/gemini-pro-vision', + // copied from symlinked + symLink: 'models/gemini-1.0-pro-vision', + pricing: { + chatIn: 0.50, + chatOut: 1.50, + }, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision], // Text and Images + hidden: true, + }, +]; export function geminiFilterModels(geminiModel: GeminiModelSchema): boolean { @@ -26,17 +127,20 @@ export function geminiSortModels(a: ModelDescriptionSchema, b: ModelDescriptionS return b.label.localeCompare(a.label); } -export function geminiModelToModelDescription(geminiModel: GeminiModelSchema, allModels: GeminiModelSchema[]): ModelDescriptionSchema { +export function geminiModelToModelDescription(geminiModel: GeminiModelSchema): ModelDescriptionSchema { const { description, displayName, name: modelId, supportedGenerationMethods } = geminiModel; + // find known manual mapping + const knownModel = _knownGeminiModels.find(m => m.id === modelId); + // handle symlinks - const isSymlink = geminiLinkModels.includes(modelId); - const symlinked = isSymlink ? allModels.find(m => m.displayName === displayName && m.name !== modelId) : null; - const label = isSymlink ? `๐Ÿ”— ${displayName.replace('1.0', '')} โ†’ ${symlinked ? symlinked.name : '?'}` : displayName; + const label = knownModel?.symLink + ? `๐Ÿ”— ${displayName.replace('1.0', '')} โ†’ ${knownModel.symLink}` + : displayName; // handle hidden models const hasChatInterfaces = supportedGenerationMethods.some(iface => geminiChatInterfaces.includes(iface)); - const hidden = isSymlink || !hasChatInterfaces; + const hidden = knownModel?.hidden || !!knownModel?.symLink || !hasChatInterfaces; // context window const { inputTokenLimit, outputTokenLimit } = geminiModel; @@ -46,11 +150,12 @@ export function geminiModelToModelDescription(geminiModel: GeminiModelSchema, al const { version, topK, topP, temperature } = geminiModel; const descriptionLong = description + ` (Version: ${version}, Defaults: temperature=${temperature}, topP=${topP}, topK=${topK}, interfaces=[${supportedGenerationMethods.join(',')}])`; - const interfaces: ModelDescriptionSchema['interfaces'] = []; - if (hasChatInterfaces) { + // use known interfaces, or add chat if this is a generateContent model + const interfaces: ModelDescriptionSchema['interfaces'] = knownModel?.interfaces || []; + if (!interfaces.length && hasChatInterfaces) { interfaces.push(LLM_IF_OAI_Chat); - if (geminiVisionNames.some(name => modelId.includes(name))) - interfaces.push(LLM_IF_OAI_Vision); + // if (geminiVisionNames.some(name => modelId.includes(name))) + // interfaces.push(LLM_IF_OAI_Vision); } return { @@ -61,11 +166,11 @@ export function geminiModelToModelDescription(geminiModel: GeminiModelSchema, al description: descriptionLong, contextWindow: contextWindow, maxCompletionTokens: outputTokenLimit, - // trainingDataCutoff: '...', + trainingDataCutoff: knownModel?.trainingDataCutoff, interfaces, // rateLimits: isGeminiPro ? { reqPerMinute: 60 } : undefined, // benchmarks: ... - // pricing: isGeminiPro ? { needs per-character and per-image pricing } : undefined, + pricing: knownModel?.pricing, // TODO: needs <>128k, and per-character and per-image pricing hidden, }; } diff --git a/src/modules/llms/server/gemini/gemini.router.ts b/src/modules/llms/server/gemini/gemini.router.ts index 80fa9245a..e398b5b6d 100644 --- a/src/modules/llms/server/gemini/gemini.router.ts +++ b/src/modules/llms/server/gemini/gemini.router.ts @@ -147,7 +147,7 @@ export const llmGeminiRouter = createTRPCRouter({ // map to our output schema const models = detailedModels .filter(geminiFilterModels) - .map(geminiModel => geminiModelToModelDescription(geminiModel, detailedModels)) + .map(geminiModel => geminiModelToModelDescription(geminiModel)) .sort(geminiSortModels); return { diff --git a/src/modules/llms/server/openai/models.data.ts b/src/modules/llms/server/openai/models.data.ts index 719e948d8..707daab03 100644 --- a/src/modules/llms/server/openai/models.data.ts +++ b/src/modules/llms/server/openai/models.data.ts @@ -875,7 +875,13 @@ export function groqModelSortFn(a: ModelDescriptionSchema, b: ModelDescriptionSc // Helpers -type ManualMapping = ({ idPrefix: string, isLatest?: boolean, isPreview?: boolean, isLegacy?: boolean, symLink?: string } & Omit); +type ManualMapping = ({ + idPrefix: string, + isLatest?: boolean, + isPreview?: boolean, + isLegacy?: boolean, + symLink?: string +} & Omit); type ManualMappings = ManualMapping[]; function fromManualMapping(mappings: ManualMappings, id: string, created?: number, updated?: number, fallback?: ManualMapping): ModelDescriptionSchema { From 409a3ee1940d34a8ca27f65c2ba20a03c67d34e9 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Tue, 14 May 2024 17:57:53 -0700 Subject: [PATCH 09/27] DChat: remove IDB migration (cherry picked from commit 44ab0483b626ada24817c6f9e0ad9d7c0971e09d) --- src/common/state/store-chats.ts | 33 ++------------------------------- src/common/util/idbUtils.ts | 15 --------------- 2 files changed, 2 insertions(+), 46 deletions(-) diff --git a/src/common/state/store-chats.ts b/src/common/state/store-chats.ts index a1d3f226e..9119beeba 100644 --- a/src/common/state/store-chats.ts +++ b/src/common/state/store-chats.ts @@ -5,7 +5,7 @@ import { v4 as uuidv4 } from 'uuid'; import { DLLMId, getChatLLMId } from '~/modules/llms/store-llms'; -import { IDB_MIGRATION_INITIAL, idbStateStorage } from '../util/idbUtils'; +import { idbStateStorage } from '../util/idbUtils'; import { countModelTokens } from '../util/token-counter'; import { defaultSystemPurposeId, SystemPurposeId } from '../../data'; @@ -407,10 +407,7 @@ export const useChatStore = create()(devtools( storage: createJSONStorage(() => idbStateStorage), // Migrations - migrate: (persistedState: unknown, fromVersion: number): ConversationsStore => { - // -1 -> 3: migration loading from localStorage to IndexedDB - if (fromVersion === IDB_MIGRATION_INITIAL) - return _migrateLocalStorageData() as any; + migrate: (persistedState: unknown, _fromVersion: number): ConversationsStore => { // other: just proceed return persistedState as any; @@ -465,32 +462,6 @@ function getNextBranchTitle(currentTitle: string): string { return `(1) ${currentTitle}`; } -/** - * Returns the chats stored in the localStorage, and rename the key for - * backup/data loss prevention purposes - */ -function _migrateLocalStorageData(): ChatState | {} { - const key = 'app-chats'; - const value = localStorage.getItem(key); - if (!value) return {}; - try { - // parse the localStorage state - const localStorageState = JSON.parse(value)?.state; - - // backup and delete the localStorage key - const backupKey = `${key}-v2`; - localStorage.setItem(backupKey, value); - localStorage.removeItem(key); - - // match the state from localstorage - return { - conversations: localStorageState?.conversations ?? [], - }; - } catch (error) { - console.error('LocalStorage migration error', error); - return {}; - } -} /** * Convenience function to count the tokens in a DMessage object diff --git a/src/common/util/idbUtils.ts b/src/common/util/idbUtils.ts index b900a8ecf..fd42178fd 100644 --- a/src/common/util/idbUtils.ts +++ b/src/common/util/idbUtils.ts @@ -1,10 +1,6 @@ import type { StateStorage } from 'zustand/middleware'; import { del as idbDel, get as idbGet, set as idbSet } from 'idb-keyval'; -// used by the state storage middleware to detect data migration from the old state storage (localStorage) -// NOTE: remove past 2024-03-19 (6 months past release of this utility conversion) -export const IDB_MIGRATION_INITIAL = -1; - // set to true to enable debugging const DEBUG_SCHEDULER = false; @@ -130,17 +126,6 @@ export const idbStateStorage: StateStorage = { if (DEBUG_SCHEDULER) console.warn(' (read bytes:', value?.length?.toLocaleString(), ')'); - /* IMPORTANT! - * We modify the default behavior of `getItem` to return a {version: -1} object if a key is not found. - * This is to trigger the migration across state storage implementations, as Zustand would not call the - * 'migrate' function otherwise. - * See 'https://github.com/enricoros/big-agi/pull/158' for more details - */ - if (value === undefined) { - return JSON.stringify({ - version: IDB_MIGRATION_INITIAL, - }); - } return value || null; }, setItem: (name: string, value: string): void => { From 29a784c6c614702fa8cbbc4f4d44c724522c3b16 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 00:53:08 -0700 Subject: [PATCH 10/27] Update TikToken for perfect token computation on 'o' models. (cherry picked from commit 21d045be597c1fb0dc4903cee95d3985d4b8262b) --- package-lock.json | 8 +++---- package.json | 2 +- src/common/util/token-counter.ts | 38 ++++++++++++++++++++------------ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index ad4c4762d..3aa41654f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,7 +52,7 @@ "sharp": "^0.33.3", "superjson": "^2.2.1", "tesseract.js": "^5.1.0", - "tiktoken": "^1.0.14", + "tiktoken": "^1.0.15", "turndown": "^7.1.3", "uuid": "^9.0.1", "zod": "^3.23.8", @@ -8353,9 +8353,9 @@ } }, "node_modules/tiktoken": { - "version": "1.0.14", - "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.14.tgz", - "integrity": "sha512-g5zd5r/DoH8Kw0fiYbYpVhb6WO8BHO1unXqmBBWKwoT17HwSounnDtMDFUKm2Pko8U47sjQarOe+9aUrnqmmTg==" + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.15.tgz", + "integrity": "sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw==" }, "node_modules/tiny-invariant": { "version": "1.3.3", diff --git a/package.json b/package.json index 1961b2ed8..d32a68e05 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "sharp": "^0.33.3", "superjson": "^2.2.1", "tesseract.js": "^5.1.0", - "tiktoken": "^1.0.14", + "tiktoken": "^1.0.15", "turndown": "^7.1.3", "uuid": "^9.0.1", "zod": "^3.23.8", diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts index adfc06b78..b6429fef5 100644 --- a/src/common/util/token-counter.ts +++ b/src/common/util/token-counter.ts @@ -6,14 +6,17 @@ import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms'; // Do not set this to true in production, it's very verbose const DEBUG_TOKEN_COUNT = false; - -// global symbols to dynamically load the Tiktoken library +// Global symbols to dynamically load the Tiktoken library let get_encoding: ((encoding: TiktokenEncoding) => Tiktoken) | null = null; let encoding_for_model: ((model: TiktokenModel) => Tiktoken) | null = null; let preloadPromise: Promise | null = null; let informTheUser = false; -export function preloadTiktokenLibrary() { +/** + * Preloads the Tiktoken library if not already loaded. + * @returns {Promise} A promise that resolves when the library is loaded. + */ +export function preloadTiktokenLibrary(): Promise { if (!preloadPromise) { preloadPromise = import('tiktoken') .then(tiktoken => { @@ -33,16 +36,21 @@ export function preloadTiktokenLibrary() { /** - * Wrapper around the Tiktoken library, to keep tokenizers for all models in a cache - * - * We also preload the tokenizer for the default model, so that the first time a user types - * a message, it doesn't stall loading the tokenizer. + * Wrapper around the Tiktoken library to keep tokenizers for all models in a cache. + * Also, preloads the tokenizer for the default model to avoid initial stall. */ export const countModelTokens: (text: string, llmId: DLLMId, debugFrom: string) => number | null = (() => { // return () => 0; const tokenEncoders: { [modelId: string]: Tiktoken } = {}; - let encodingCL100K: Tiktoken | null = null; + let encodingDefault: Tiktoken | null = null; + /** + * Counts the tokens in the given text for the specified model. + * @param {string} text - The text to tokenize. + * @param {DLLMId} llmId - The ID of the LLM. + * @param {string} debugFrom - Debug information. + * @returns {number | null} The token count or null if not ready. + */ function _tokenCount(text: string, llmId: DLLMId, debugFrom: string): number | null { // The library shall have been preloaded - if not, attempt to start its loading and return null to indicate we're not ready to count @@ -55,21 +63,23 @@ export const countModelTokens: (text: string, llmId: DLLMId, debugFrom: string) return null; } - const { options: { llmRef: openaiModel } } = findLLMOrThrow(llmId); + const openaiModel = findLLMOrThrow(llmId)?.options?.llmRef; if (!openaiModel) throw new Error(`LLM ${llmId} has no LLM reference id`); + if (!(openaiModel in tokenEncoders)) { try { tokenEncoders[openaiModel] = encoding_for_model(openaiModel as TiktokenModel); } catch (e) { - // make sure we recycle the default encoding across all models - if (!encodingCL100K) - encodingCL100K = get_encoding('cl100k_base'); - tokenEncoders[openaiModel] = encodingCL100K; + // fallback to the default encoding across all models (not just OpenAI - this will be used everywhere..) + if (!encodingDefault) + encodingDefault = get_encoding('cl100k_base'); + tokenEncoders[openaiModel] = encodingDefault; } } - let count: number = 0; + // Note: the try/catch shouldn't be necessary, but there could be corner cases where the tiktoken library throws // https://github.com/enricoros/big-agi/issues/182 + let count = 0; try { count = tokenEncoders[openaiModel]?.encode(text, 'all', [])?.length || 0; } catch (e) { From 8a0e7a4e3d5a20710fe086a7be92f9b814dbf59b Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 00:55:58 -0700 Subject: [PATCH 11/27] Tiktoken: in the future, show tokens (cherry picked from commit d5c3f5012b316bd5d93080060e5f6790b734c2fa) --- src/common/util/token-counter.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts index b6429fef5..de21026b6 100644 --- a/src/common/util/token-counter.ts +++ b/src/common/util/token-counter.ts @@ -6,6 +6,9 @@ import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms'; // Do not set this to true in production, it's very verbose const DEBUG_TOKEN_COUNT = false; +// Globals +// const tokenEncodings: string[] = ['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base'] satisfies TiktokenEncoding[]; + // Global symbols to dynamically load the Tiktoken library let get_encoding: ((encoding: TiktokenEncoding) => Tiktoken) | null = null; let encoding_for_model: ((model: TiktokenModel) => Tiktoken) | null = null; From 1fed2fb18c1103bc2907c942954a1f1579f13262 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 01:00:21 -0700 Subject: [PATCH 12/27] Beam: if auto-start, give the chance to change merge model (cherry picked from commit b607e3c034fe5c99d0ca4d84d2ee6efbbeb9d850) --- src/modules/beam/gather/BeamGatherPane.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/modules/beam/gather/BeamGatherPane.tsx b/src/modules/beam/gather/BeamGatherPane.tsx index ff591b196..2b82361fa 100644 --- a/src/modules/beam/gather/BeamGatherPane.tsx +++ b/src/modules/beam/gather/BeamGatherPane.tsx @@ -13,6 +13,7 @@ import { BeamStoreApi, useBeamStore } from '../store-beam.hooks'; import { FFactoryId, FUSION_FACTORIES } from './instructions/beam.gather.factories'; import { GATHER_COLOR } from '../beam.config'; import { beamPaneSx } from '../BeamCard'; +import { useModuleBeamStore } from '../store-module-beam'; const gatherPaneClasses = { @@ -79,8 +80,9 @@ export function BeamGatherPane(props: { setCurrentFactoryId: state.setCurrentFactoryId, setCurrentGatherLlmId: state.setCurrentGatherLlmId, }))); + const gatherAutoStartAfterScatter = useModuleBeamStore(state => state.gatherAutoStartAfterScatter); const [_, gatherLlmComponent/*, gatherLlmIcon*/] = useLLMSelect( - currentGatherLlmId, setCurrentGatherLlmId, props.isMobile ? '' : 'Merge Model', true, !props.canGather, + currentGatherLlmId, setCurrentGatherLlmId, props.isMobile ? '' : 'Merge Model', true, !props.canGather && !gatherAutoStartAfterScatter, ); // derived state From 5cc0b0a011a012cef67d9cf569c1ed0fcfcaf758 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 01:30:37 -0700 Subject: [PATCH 13/27] Beam: fix reactive bug (cherry picked from commit e513b4278626135324ba3c3bdf831d4c755c609f) --- src/modules/beam/BeamView.tsx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/modules/beam/BeamView.tsx b/src/modules/beam/BeamView.tsx index 2cef13c55..d3d50304a 100644 --- a/src/modules/beam/BeamView.tsx +++ b/src/modules/beam/BeamView.tsx @@ -30,6 +30,7 @@ export function BeamView(props: { // external state const { novel: explainerUnseen, touch: explainerCompleted, forget: explainerShow } = useUICounter('beam-wizard'); + const gatherAutoStartAfterScatter = useModuleBeamStore(state => state.gatherAutoStartAfterScatter); const { /* root */ editInputHistoryMessage, /* scatter */ setRayCount, startScatteringAll, stopScatteringAll, @@ -38,24 +39,21 @@ export function BeamView(props: { /* root */ inputHistory, inputIssues, inputReady, /* scatter */ isScattering, raysReady, /* gather (composite) */ canGather, - /* IDs */ rayIds, fusionIds, } = useBeamStore(props.beamStore, useShallow(state => ({ // input inputHistory: state.inputHistory, inputIssues: state.inputIssues, inputReady: state.inputReady, // scatter + hadImportedRays: state.hadImportedRays, isScattering: state.isScattering, raysReady: state.raysReady, // gather (composite) canGather: state.raysReady >= 2 && state.currentFactoryId !== null && state.currentGatherLlmId !== null, - // IDs - rayIds: state.rays.map(ray => ray.rayId), - fusionIds: state.fusions.map(fusion => fusion.fusionId), - }))); - const { gatherAutoStartAfterScatter } = useModuleBeamStore(useShallow(state => ({ - gatherAutoStartAfterScatter: state.gatherAutoStartAfterScatter, }))); + // the following are independent because of useShallow, which would break in the above call + const rayIds = useBeamStore(props.beamStore, useShallow(state => state.rays.map(ray => ray.rayId))); + const fusionIds = useBeamStore(props.beamStore, useShallow(state => state.fusions.map(fusion => fusion.fusionId))); // derived state const raysCount = rayIds.length; From 6247b5411b345cb5591ae7cc3c5d75bffc376dbf Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 01:35:04 -0700 Subject: [PATCH 14/27] Beam: recall importing rays (cherry picked from commit 454a4257da4a9c25ee3e1774e5ce4a84aa6c681f) --- src/modules/beam/BeamView.tsx | 3 ++- src/modules/beam/scatter/BeamRay.tsx | 3 ++- src/modules/beam/scatter/BeamRayGrid.tsx | 2 ++ src/modules/beam/scatter/beam.scatter.ts | 3 +++ src/modules/beam/store-beam-vanilla.ts | 3 ++- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/modules/beam/BeamView.tsx b/src/modules/beam/BeamView.tsx index d3d50304a..81ecbd01c 100644 --- a/src/modules/beam/BeamView.tsx +++ b/src/modules/beam/BeamView.tsx @@ -37,7 +37,7 @@ export function BeamView(props: { } = props.beamStore.getState(); const { /* root */ inputHistory, inputIssues, inputReady, - /* scatter */ isScattering, raysReady, + /* scatter */ hadImportedRays, isScattering, raysReady, /* gather (composite) */ canGather, } = useBeamStore(props.beamStore, useShallow(state => ({ // input @@ -171,6 +171,7 @@ export function BeamView(props: { beamStore={props.beamStore} isMobile={props.isMobile} rayIds={rayIds} + hadImportedRays={hadImportedRays} onIncreaseRayCount={handleRayIncreaseCount} // linkedLlmId={currentGatherLlmId} /> diff --git a/src/modules/beam/scatter/BeamRay.tsx b/src/modules/beam/scatter/BeamRay.tsx index 4be86aa02..66e2c8521 100644 --- a/src/modules/beam/scatter/BeamRay.tsx +++ b/src/modules/beam/scatter/BeamRay.tsx @@ -109,7 +109,8 @@ function RayControls(props: { export function BeamRay(props: { beamStore: BeamStoreApi, - isRemovable: boolean + hadImportedRays: boolean + isRemovable: boolean, rayId: string, rayIndexWeak: number, // linkedLlmId: DLLMId | null, diff --git a/src/modules/beam/scatter/BeamRayGrid.tsx b/src/modules/beam/scatter/BeamRayGrid.tsx index 1ef6ff662..8c5d6500a 100644 --- a/src/modules/beam/scatter/BeamRayGrid.tsx +++ b/src/modules/beam/scatter/BeamRayGrid.tsx @@ -27,6 +27,7 @@ const rayGridMobileSx: SxProps = { export function BeamRayGrid(props: { beamStore: BeamStoreApi, + hadImportedRays: boolean isMobile: boolean, onIncreaseRayCount: () => void, rayIds: string[], @@ -44,6 +45,7 @@ export function BeamRayGrid(props: { key={'ray-' + rayId} rayIndexWeak={index} beamStore={props.beamStore} + hadImportedRays={props.hadImportedRays} isRemovable={raysCount > SCATTER_RAY_MIN} rayId={rayId} // linkedLlmId={props.linkedLlmId} diff --git a/src/modules/beam/scatter/beam.scatter.ts b/src/modules/beam/scatter/beam.scatter.ts index 2d611fe62..d229a6d30 100644 --- a/src/modules/beam/scatter/beam.scatter.ts +++ b/src/modules/beam/scatter/beam.scatter.ts @@ -134,6 +134,7 @@ export function rayIsImported(ray: BRay | null): boolean { interface ScatterStateSlice { rays: BRay[]; + hadImportedRays: boolean; // derived state isScattering: boolean; // true if any ray is scattering at the moment @@ -148,6 +149,7 @@ export const reInitScatterStateSlice = (prevRays: BRay[]): ScatterStateSlice => return { // (remember) keep the same quantity of rays and same llms rays: prevRays.map(prevRay => createBRay(prevRay.rayLlmId)), + hadImportedRays: false, isScattering: false, raysReady: 0, @@ -238,6 +240,7 @@ export const createScatterSlice: StateCreator !raysToRemove.includes(ray)), ], + hadImportedRays: messages.length > 0, }); _storeLastScatterConfig(); _syncRaysStateToScatter(); diff --git a/src/modules/beam/store-beam-vanilla.ts b/src/modules/beam/store-beam-vanilla.ts index 92528275a..a286d5501 100644 --- a/src/modules/beam/store-beam-vanilla.ts +++ b/src/modules/beam/store-beam-vanilla.ts @@ -70,7 +70,7 @@ const createRootSlice: StateCreator = (_set, open: (chatHistory: Readonly, initialChatLlmId: DLLMId | null, callback: BeamSuccessCallback) => { - const { isOpen: wasAlreadyOpen, terminateKeepingSettings, loadBeamConfig, setRayLlmIds, setCurrentGatherLlmId } = _get(); + const { isOpen: wasAlreadyOpen, terminateKeepingSettings, loadBeamConfig, hadImportedRays, setRayLlmIds, setCurrentGatherLlmId } = _get(); // reset pending operations terminateKeepingSettings(); @@ -89,6 +89,7 @@ const createRootSlice: StateCreator = (_set, onSuccessCallback: callback, // rays already reset + hadImportedRays, // update the model only if the dialog was not already open ...(!wasAlreadyOpen && initialChatLlmId && { From acfe0aba21749b02a0a7785887c386bca2db759f Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 01:41:44 -0700 Subject: [PATCH 15/27] Beam: bits (cherry picked from commit 81d99f19d465b5a15ae2850bafd35641d78f5e62) --- src/modules/beam/gather/Fusion.tsx | 9 +++++---- src/modules/beam/scatter/BeamRay.tsx | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/modules/beam/gather/Fusion.tsx b/src/modules/beam/gather/Fusion.tsx index bb6dcb7d7..57782e83e 100644 --- a/src/modules/beam/gather/Fusion.tsx +++ b/src/modules/beam/gather/Fusion.tsx @@ -170,23 +170,24 @@ export function Fusion(props: { } sx={{ // ...BEAM_BTN_SX, - // fontSize: 'xs', + fontSize: 'xs', + // '--Icon-fontSize': 'var(--joy-fontSize-xl)', // backgroundColor: 'background.popup', // border: '1px solid', // borderColor: `${GATHER_COLOR}.outlinedBorder`, // boxShadow: `0 4px 16px -4px rgb(var(--joy-palette-${GATHER_COLOR}-mainChannel) / 20%)`, animation: `${animationEnterBelow} 0.1s ease-out`, - // whiteSpace: 'nowrap', + whiteSpace: 'nowrap', }} > - {/*Ok*/} + {/*Use*/} diff --git a/src/modules/beam/scatter/BeamRay.tsx b/src/modules/beam/scatter/BeamRay.tsx index 66e2c8521..1df103ab6 100644 --- a/src/modules/beam/scatter/BeamRay.tsx +++ b/src/modules/beam/scatter/BeamRay.tsx @@ -16,6 +16,7 @@ import type { DLLMId } from '~/modules/llms/store-llms'; import { GoodTooltip } from '~/common/components/GoodTooltip'; import { InlineError } from '~/common/components/InlineError'; +import { animationEnterBelow } from '~/common/util/animUtils'; import { copyToClipboard } from '~/common/util/clipboardUtils'; import { useLLMSelect } from '~/common/components/forms/useLLMSelect'; @@ -241,16 +242,20 @@ export function BeamRay(props: { : null} sx={{ fontSize: 'xs', + // '--Icon-fontSize': 'var(--joy-fontSize-xl)', px: isImported ? 1 : undefined, + animation: `${animationEnterBelow} 0.1s ease-out`, whiteSpace: 'nowrap', }} > - {isImported ? 'From Chat' : /*'Use'*/ } + {isImported ? 'From Chat' : /*props.hadImportedRays ? 'Replace' : 'Use'*/ } From e3957bf08b2c2e4a4e10efbdb931ee27234e6208 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 02:57:50 -0700 Subject: [PATCH 16/27] Page download: improve (cherry picked from commit f9d33d4888a1937d437e409ccf0eb4864f8cca48) --- pages/link/share_target.tsx | 9 +- .../composer/attachments/pipeline.tsx | 16 +-- src/apps/chat/editors/browse-load.ts | 3 +- src/modules/aifn/react/react.ts | 3 +- src/modules/browse/browse.client.ts | 37 +++--- src/modules/browse/browse.router.ts | 105 +++++++++--------- 6 files changed, 90 insertions(+), 83 deletions(-) diff --git a/pages/link/share_target.tsx b/pages/link/share_target.tsx index 5614121cf..be29c7e65 100644 --- a/pages/link/share_target.tsx +++ b/pages/link/share_target.tsx @@ -77,9 +77,12 @@ function AppShareTarget() { setIsDownloading(true); callBrowseFetchPage(intentURL) .then(page => { - if (page.stopReason !== 'error') - queueComposerTextAndLaunchApp('\n\n```' + intentURL + '\n' + page.content + '\n```\n'); - else + if (page.stopReason !== 'error') { + let pageContent = page.content.markdown || page.content.text || page.content.html || ''; + if (pageContent) + pageContent = '\n\n```' + intentURL + '\n' + pageContent + '\n```\n'; + queueComposerTextAndLaunchApp(pageContent); + } else setErrorMessage('Could not read any data' + page.error ? ': ' + page.error : ''); }) .catch(error => setErrorMessage(error?.message || error || 'Unknown error')) diff --git a/src/apps/chat/components/composer/attachments/pipeline.tsx b/src/apps/chat/components/composer/attachments/pipeline.tsx index f77eef785..4aeda14b3 100644 --- a/src/apps/chat/components/composer/attachments/pipeline.tsx +++ b/src/apps/chat/components/composer/attachments/pipeline.tsx @@ -58,16 +58,12 @@ export async function attachmentLoadInputAsync(source: Readonly { async function browse(url: string): Promise { try { const page = await callBrowseFetchPage(url); - return JSON.stringify(page.content ? { text: page.content } : { error: 'Issue reading the page' }); + const pageContent = page.content.markdown || page.content.text || page.content.html || ''; + return JSON.stringify(pageContent ? { text: pageContent } : { error: 'Issue reading the page' }); } catch (error) { console.error('Error browsing:', (error as Error).message); return 'An error occurred while browsing to the URL. Missing WSS Key?'; diff --git a/src/modules/browse/browse.client.ts b/src/modules/browse/browse.client.ts index 806aa6de5..9e68dcf05 100644 --- a/src/modules/browse/browse.client.ts +++ b/src/modules/browse/browse.client.ts @@ -1,4 +1,4 @@ -import { BrowsePageTransform, useBrowseStore } from '~/modules/browse/store-module-browsing'; +import { useBrowseStore } from '~/modules/browse/store-module-browsing'; import { apiAsyncNode } from '~/common/util/trpc.client'; @@ -7,34 +7,39 @@ import { apiAsyncNode } from '~/common/util/trpc.client'; const DEBUG_SHOW_SCREENSHOT = false; -export async function callBrowseFetchPage(url: string, forceTransform?: BrowsePageTransform) { +// export function - // thow if no URL is provided +export async function callBrowseFetchPage( + url: string, + // transforms?: BrowsePageTransform[], + // screenshotOptions?: { width: number, height: number, quality?: number }, +) { + + // validate url url = url?.trim() || ''; if (!url) throw new Error('Browsing error: Invalid URL'); - // assume https if no protocol is provided - // noinspection HttpUrlsUsage + // noinspection HttpUrlsUsage: assume https if no protocol is provided if (!url.startsWith('http://') && !url.startsWith('https://')) url = 'https://' + url; - const { wssEndpoint: clientWssEndpoint, pageTransform } = useBrowseStore.getState(); + const { wssEndpoint, pageTransform } = useBrowseStore.getState(); const { pages } = await apiAsyncNode.browse.fetchPages.mutate({ access: { dialect: 'browse-wss', - ...(!!clientWssEndpoint && { wssEndpoint: clientWssEndpoint }), + ...(!!wssEndpoint && { wssEndpoint }), }, - subjects: [{ + requests: [{ url, - transform: pageTransform || 'text', + transforms: /*transforms ? transforms :*/ [pageTransform], + screenshot: /*screenshotOptions ? screenshotOptions :*/ !DEBUG_SHOW_SCREENSHOT ? undefined : { + width: 512, + height: 512, + // quality: 100, + }, }], - screenshot: DEBUG_SHOW_SCREENSHOT ? { - width: 512, - height: 512, - // quality: 100, - } : undefined, }); if (pages.length !== 1) @@ -45,7 +50,7 @@ export async function callBrowseFetchPage(url: string, forceTransform?: BrowsePa // DEBUG: if there's a screenshot, append it to the dom if (DEBUG_SHOW_SCREENSHOT && page.screenshot) { const img = document.createElement('img'); - img.src = page.screenshot.imageDataUrl; + img.src = page.screenshot.webpDataUrl; img.style.width = `${page.screenshot.width}px`; img.style.height = `${page.screenshot.height}px`; document.body.appendChild(img); @@ -54,7 +59,7 @@ export async function callBrowseFetchPage(url: string, forceTransform?: BrowsePa // throw if there's an error if (page.error) { console.warn('Browsing service error:', page.error); - if (!page.content) + if (!Object.keys(page.content).length) throw new Error(page.error); } diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 280b2e7bd..3e11247d0 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -19,21 +19,22 @@ const browseAccessSchema = z.object({ dialect: z.enum(['browse-wss']), wssEndpoint: z.string().trim().optional(), }); +type BrowseAccessSchema = z.infer; const pageTransformSchema = z.enum(['html', 'text', 'markdown']); type PageTransformSchema = z.infer; const fetchPageInputSchema = z.object({ access: browseAccessSchema, - subjects: z.array(z.object({ + requests: z.array(z.object({ url: z.string().url(), - transform: pageTransformSchema, + transforms: z.array(pageTransformSchema), + screenshot: z.object({ + width: z.number(), + height: z.number(), + quality: z.number().optional(), + }).optional(), })), - screenshot: z.object({ - width: z.number(), - height: z.number(), - quality: z.number().optional(), - }).optional(), }); @@ -41,16 +42,18 @@ const fetchPageInputSchema = z.object({ const fetchPageWorkerOutputSchema = z.object({ url: z.string(), - content: z.string(), + content: z.record(pageTransformSchema, z.string()), error: z.string().optional(), stopReason: z.enum(['end', 'timeout', 'error']), screenshot: z.object({ - imageDataUrl: z.string().startsWith('data:image/'), + webpDataUrl: z.string().startsWith('data:image/webp'), mimeType: z.string().startsWith('image/'), width: z.number(), height: z.number(), }).optional(), }); +type FetchPageWorkerOutputSchema = z.infer; + const fetchPagesOutputSchema = z.object({ pages: z.array(fetchPageWorkerOutputSchema), @@ -62,21 +65,23 @@ export const browseRouter = createTRPCRouter({ fetchPages: publicProcedure .input(fetchPageInputSchema) .output(fetchPagesOutputSchema) - .mutation(async ({ input: { access, subjects, screenshot } }) => { - const pages: FetchPageWorkerOutputSchema[] = []; - - for (const subject of subjects) { - try { - pages.push(await workerPuppeteer(access, subject.url, subject.transform, screenshot?.width, screenshot?.height, screenshot?.quality)); - } catch (error: any) { - pages.push({ - url: subject.url, - content: '', - error: error?.message || JSON.stringify(error) || 'Unknown fetch error', + .mutation(async ({ input: { access, requests } }) => { + + const pagePromises = requests.map(request => + workerPuppeteer(access, request.url, request.transforms, request.screenshot)); + + const results = await Promise.allSettled(pagePromises); + + const pages: FetchPageWorkerOutputSchema[] = results.map((result, index) => + result.status === 'fulfilled' + ? result.value + : { + url: requests[index].url, + content: {}, + error: result.reason?.message || 'Unknown fetch error', stopReason: 'error', - }); - } - } + }, + ); return { pages }; }), @@ -84,18 +89,13 @@ export const browseRouter = createTRPCRouter({ }); -type BrowseAccessSchema = z.infer; -type FetchPageWorkerOutputSchema = z.infer; - - async function workerPuppeteer( access: BrowseAccessSchema, targetUrl: string, - transform: PageTransformSchema, - ssWidth: number | undefined, - ssHeight: number | undefined, - ssQuality: number | undefined, + transforms: PageTransformSchema[], + screenshotOptions?: { width: number, height: number, quality?: number }, ): Promise { + const browserWSEndpoint = (access.wssEndpoint || env.PUPPETEER_WSS_ENDPOINT || '').trim(); const isLocalBrowser = browserWSEndpoint.startsWith('ws://'); if (!browserWSEndpoint || (!browserWSEndpoint.startsWith('wss://') && !isLocalBrowser)) @@ -106,7 +106,7 @@ async function workerPuppeteer( const result: FetchPageWorkerOutputSchema = { url: targetUrl, - content: '', + content: {}, error: undefined, stopReason: 'error', screenshot: undefined, @@ -144,21 +144,23 @@ async function workerPuppeteer( // transform the content of the page as text try { if (result.stopReason !== 'error') { - switch (transform) { - case 'html': - result.content = await page.content(); - break; - case 'text': - result.content = await page.evaluate(() => document.body.innerText || document.textContent || ''); - break; - case 'markdown': - const html = await page.content(); - const cleanedHtml = cleanHtml(html); - const turndownService = new TurndownService({ headingStyle: 'atx' }); - result.content = turndownService.turndown(cleanedHtml); - break; + for (const transform of transforms) { + switch (transform) { + case 'html': + result.content.html = cleanHtml(await page.content()); + break; + case 'text': + result.content.text = await page.evaluate(() => document.body.innerText || document.textContent || ''); + break; + case 'markdown': + const html = await page.content(); + const cleanedHtml = cleanHtml(html); + const turndownService = new TurndownService({ headingStyle: 'atx' }); + result.content.markdown = turndownService.turndown(cleanedHtml); + break; + } } - if (!result.content) + if (!Object.keys(result.content).length) result.error = '[Puppeteer] Empty content'; } } catch (error: any) { @@ -167,10 +169,9 @@ async function workerPuppeteer( // get a screenshot of the page try { - if (ssWidth && ssHeight) { - const width = ssWidth; - const height = ssHeight; - const scale = Math.round(100 * ssWidth / 1024) / 100; + if (screenshotOptions?.width && screenshotOptions?.height) { + const { width, height, quality } = screenshotOptions; + const scale = Math.round(100 * width / 1024) / 100; await page.setViewport({ width: width / scale, height: height / scale, deviceScaleFactor: scale }); @@ -181,10 +182,10 @@ async function workerPuppeteer( type: imageType, encoding: 'base64', clip: { x: 0, y: 0, width: width / scale, height: height / scale }, - ...(ssQuality && { quality: ssQuality }), + ...(quality && { quality }), }) as string; - result.screenshot = { imageDataUrl: `data:${mimeType};base64,${dataString}`, mimeType, width, height }; + result.screenshot = { webpDataUrl: `data:${mimeType};base64,${dataString}`, mimeType, width, height }; } } catch (error: any) { console.error('workerPuppeteer: page.screenshot', error); From 44d8c301879c5705aeb477d69cb40f10ee5a3150 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 03:03:57 -0700 Subject: [PATCH 17/27] Start opened (cherry picked from commit 952bd2bd93d6bd44598649e1632fd456afef3dc3) --- src/apps/settings-modal/SettingsModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/settings-modal/SettingsModal.tsx b/src/apps/settings-modal/SettingsModal.tsx index a4202dffc..7ec93a831 100644 --- a/src/apps/settings-modal/SettingsModal.tsx +++ b/src/apps/settings-modal/SettingsModal.tsx @@ -200,7 +200,7 @@ export function SettingsModal(props: { - } title='Browsing' startCollapsed> + } title='Browsing'> } title='Google Search API' startCollapsed> From 97b6fc5e2bc46bc8cd152c32bfc8c64b564d84f7 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 03:25:25 -0700 Subject: [PATCH 18/27] Already Set (cherry picked from commit a7ce5c1ca664e5753697470f0dbb516dec99cc15) --- src/common/components/AlreadySet.tsx | 15 +++++++++++++++ src/modules/browse/BrowseSettings.tsx | 7 ++++--- src/modules/elevenlabs/ElevenlabsSettings.tsx | 3 ++- .../vendors/anthropic/AnthropicSourceSetup.tsx | 3 ++- .../llms/vendors/azure/AzureSourceSetup.tsx | 3 ++- .../llms/vendors/gemini/GeminiSourceSetup.tsx | 3 ++- src/modules/llms/vendors/groq/GroqSourceSetup.tsx | 3 ++- .../llms/vendors/localai/LocalAISourceSetup.tsx | 3 ++- .../llms/vendors/mistral/MistralSourceSetup.tsx | 3 ++- .../llms/vendors/openai/OpenAISourceSetup.tsx | 3 ++- .../vendors/openrouter/OpenRouterSourceSetup.tsx | 3 ++- .../vendors/perplexity/PerplexitySourceSetup.tsx | 3 ++- .../vendors/togetherai/TogetherAISourceSetup.tsx | 3 ++- src/modules/t2i/prodia/ProdiaSettings.tsx | 3 ++- 14 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 src/common/components/AlreadySet.tsx diff --git a/src/common/components/AlreadySet.tsx b/src/common/components/AlreadySet.tsx new file mode 100644 index 000000000..ccb4c9fee --- /dev/null +++ b/src/common/components/AlreadySet.tsx @@ -0,0 +1,15 @@ +import * as React from 'react'; + +import { Typography } from '@mui/joy'; + +import CheckRoundedIcon from '@mui/icons-material/CheckRounded'; + + +export function AlreadySet(props: { required?: boolean }) { + return ( + }> + {/*Installed Already*/} + {props.required ? 'required' : 'Already set on server'} + + ); +} \ No newline at end of file diff --git a/src/modules/browse/BrowseSettings.tsx b/src/modules/browse/BrowseSettings.tsx index 0bed7ea5c..de3e406b0 100644 --- a/src/modules/browse/BrowseSettings.tsx +++ b/src/modules/browse/BrowseSettings.tsx @@ -3,12 +3,13 @@ import { useShallow } from 'zustand/react/shallow'; import { Checkbox, FormControl, FormHelperText, Option, Select, Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; +import { ExternalLink } from '~/common/components/ExternalLink'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; +import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { platformAwareKeystrokes } from '~/common/components/KeyStroke'; import { useBrowseCapability, useBrowseStore } from './store-module-browsing'; -import { ExternalLink } from '~/common/components/ExternalLink'; -import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; export function BrowseSettings() { @@ -44,7 +45,7 @@ export function BrowseSettings() { } required={!isServerConfig} isError={!isClientValid && !isServerConfig} placeholder='wss://...' /> diff --git a/src/modules/elevenlabs/ElevenlabsSettings.tsx b/src/modules/elevenlabs/ElevenlabsSettings.tsx index 6c2abfbdc..51b07db94 100644 --- a/src/modules/elevenlabs/ElevenlabsSettings.tsx +++ b/src/modules/elevenlabs/ElevenlabsSettings.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { FormControl } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { useCapabilityElevenLabs } from '~/common/components/useCapabilities'; @@ -31,7 +32,7 @@ export function ElevenlabsSettings() { {!isConfiguredServerSide && } value={apiKey} onChange={setApiKey} required={!isConfiguredServerSide} isError={!isValidKey} />} diff --git a/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx b/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx index c3afef6a2..6c31f648c 100644 --- a/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx +++ b/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Alert } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { ExternalLink } from '~/common/components/ExternalLink'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormTextField } from '~/common/components/forms/FormTextField'; @@ -49,7 +50,7 @@ export function AnthropicSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='anthropic-key' label={!!anthropicHost ? 'API Key' : 'Anthropic API Key'} rightLabel={<>{needsUserKey ? !anthropicKey && request Key - : 'โœ”๏ธ already set in server' + : } {anthropicKey && keyValid && show tokens usage} } value={anthropicKey} onChange={value => updateSetup({ anthropicKey: value })} diff --git a/src/modules/llms/vendors/azure/AzureSourceSetup.tsx b/src/modules/llms/vendors/azure/AzureSourceSetup.tsx index 24fc0d79e..cfd6572f1 100644 --- a/src/modules/llms/vendors/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/vendors/azure/AzureSourceSetup.tsx @@ -1,5 +1,6 @@ import * as React from 'react'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormTextField } from '~/common/components/forms/FormTextField'; import { InlineError } from '~/common/components/InlineError'; @@ -49,7 +50,7 @@ export function AzureSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='azure-key' label='Azure Key' rightLabel={<>{needsUserKey ? !azureKey && request Key - : 'โœ”๏ธ already set in server'} + : } } value={azureKey} onChange={value => updateSetup({ azureKey: value })} required={needsUserKey} isError={keyError} diff --git a/src/modules/llms/vendors/gemini/GeminiSourceSetup.tsx b/src/modules/llms/vendors/gemini/GeminiSourceSetup.tsx index b0f89ab1d..2044e7c53 100644 --- a/src/modules/llms/vendors/gemini/GeminiSourceSetup.tsx +++ b/src/modules/llms/vendors/gemini/GeminiSourceSetup.tsx @@ -3,6 +3,7 @@ import * as React from 'react'; import { FormControl, FormHelperText, Option, Select } from '@mui/joy'; import HealthAndSafetyIcon from '@mui/icons-material/HealthAndSafety'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { InlineError } from '~/common/components/InlineError'; @@ -50,7 +51,7 @@ export function GeminiSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='gemini-key' label='Gemini API Key' rightLabel={<>{needsUserKey ? !geminiKey && request Key - : 'โœ”๏ธ already set in server'} + : } } value={geminiKey} onChange={value => updateSetup({ geminiKey: value.trim() })} required={needsUserKey} isError={showKeyError} diff --git a/src/modules/llms/vendors/groq/GroqSourceSetup.tsx b/src/modules/llms/vendors/groq/GroqSourceSetup.tsx index bbce7101e..6b73dff95 100644 --- a/src/modules/llms/vendors/groq/GroqSourceSetup.tsx +++ b/src/modules/llms/vendors/groq/GroqSourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; @@ -42,7 +43,7 @@ export function GroqSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='groq-key' label='Groq API Key' rightLabel={<>{needsUserKey ? !groqKey && API keys - : 'โœ”๏ธ already set in server'} + : } } value={groqKey} onChange={value => updateSetup({ groqKey: value })} required={needsUserKey} isError={showKeyError} diff --git a/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx b/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx index 4d0b3a64c..c62858be5 100644 --- a/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx +++ b/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx @@ -6,6 +6,7 @@ import CheckBoxOutlinedIcon from '@mui/icons-material/CheckBoxOutlined'; import { getBackendCapabilities } from '~/modules/backend/store-backend-capabilities'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { ExpanderAccordion } from '~/common/components/ExpanderAccordion'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { InlineError } from '~/common/components/InlineError'; @@ -81,7 +82,7 @@ export function LocalAISourceSetup(props: { sourceId: DModelSourceId }) { noKey required={userHostRequired} isError={userHostError} - rightLabel={backendHasHost ? 'โœ”๏ธ already set in server' : Learn more} + rightLabel={backendHasHost ? : Learn more} value={localAIHost} onChange={value => updateSetup({ localAIHost: value })} /> diff --git a/src/modules/llms/vendors/mistral/MistralSourceSetup.tsx b/src/modules/llms/vendors/mistral/MistralSourceSetup.tsx index 02c8fe5df..fb02d2cf6 100644 --- a/src/modules/llms/vendors/mistral/MistralSourceSetup.tsx +++ b/src/modules/llms/vendors/mistral/MistralSourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; @@ -39,7 +40,7 @@ export function MistralSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='mistral-key' label='Mistral Key' rightLabel={<>{needsUserKey ? !mistralKey && request Key - : 'โœ”๏ธ already set in server'} + : } } value={mistralKey} onChange={value => updateSetup({ oaiKey: value })} required={needsUserKey} isError={showKeyError} diff --git a/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx b/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx index a62c53620..613a0868d 100644 --- a/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx +++ b/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Alert } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { Brand } from '~/common/app.config'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl'; @@ -48,7 +49,7 @@ export function OpenAISourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='openai-key' label='API Key' rightLabel={<>{needsUserKey ? !oaiKey && <>create key and request access to GPT-4 - : 'โœ”๏ธ already set in server' + : } {oaiKey && keyValid && check usage} } value={oaiKey} onChange={value => updateSetup({ oaiKey: value })} diff --git a/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx b/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx index 1ec664025..a6d3804c2 100644 --- a/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx +++ b/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Button, Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; @@ -56,7 +57,7 @@ export function OpenRouterSourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='openrouter-key' label='OpenRouter API Key' rightLabel={<>{needsUserKey ? !oaiKey && your keys - : 'โœ”๏ธ already set in server' + : } {oaiKey && keyValid && check usage} } value={oaiKey} onChange={value => updateSetup({ oaiKey: value })} diff --git a/src/modules/llms/vendors/perplexity/PerplexitySourceSetup.tsx b/src/modules/llms/vendors/perplexity/PerplexitySourceSetup.tsx index 5b47d404d..2e2c269b8 100644 --- a/src/modules/llms/vendors/perplexity/PerplexitySourceSetup.tsx +++ b/src/modules/llms/vendors/perplexity/PerplexitySourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; @@ -42,7 +43,7 @@ export function PerplexitySourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='perplexity-key' label='Perplexity API Key' rightLabel={<>{needsUserKey ? !perplexityKey && API keys - : 'โœ”๏ธ already set in server'} + : } } value={perplexityKey} onChange={value => updateSetup({ perplexityKey: value })} required={needsUserKey} isError={showKeyError} diff --git a/src/modules/llms/vendors/togetherai/TogetherAISourceSetup.tsx b/src/modules/llms/vendors/togetherai/TogetherAISourceSetup.tsx index bd1f53b67..898134e87 100644 --- a/src/modules/llms/vendors/togetherai/TogetherAISourceSetup.tsx +++ b/src/modules/llms/vendors/togetherai/TogetherAISourceSetup.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Alert, Typography } from '@mui/joy'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl'; import { InlineError } from '~/common/components/InlineError'; @@ -48,7 +49,7 @@ export function TogetherAISourceSetup(props: { sourceId: DModelSourceId }) { autoCompleteId='togetherai-key' label='Together AI Key' rightLabel={<>{needsUserKey ? !togetherKey && request Key - : 'โœ”๏ธ already set in server'} + : } } value={togetherKey} onChange={value => updateSetup({ togetherKey: value })} required={needsUserKey} isError={showKeyError} diff --git a/src/modules/t2i/prodia/ProdiaSettings.tsx b/src/modules/t2i/prodia/ProdiaSettings.tsx index 3a1117e91..99a104615 100644 --- a/src/modules/t2i/prodia/ProdiaSettings.tsx +++ b/src/modules/t2i/prodia/ProdiaSettings.tsx @@ -10,6 +10,7 @@ import StayPrimaryPortraitIcon from '@mui/icons-material/StayPrimaryPortrait'; import { getBackendCapabilities } from '~/modules/backend/store-backend-capabilities'; +import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { FormRadioControl } from '~/common/components/forms/FormRadioControl'; @@ -80,7 +81,7 @@ export function ProdiaSettings(props: { noSkipKey?: boolean }) { {!backendHasProdia && !!props.noSkipKey && } value={apiKey} onChange={setApiKey} required={!backendHasProdia} isError={!isValidKey} />} From 20f1c4c0ae68e02fe22c4c9b1da79d71f9cf039e Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 03:36:14 -0700 Subject: [PATCH 19/27] Mistral: update #518 (cherry picked from commit 6afb61d25dc9703d4eaf0d88b60e696b1b8278a8) --- src/modules/llms/server/openai/models.data.ts | 125 +++++++++++------- 1 file changed, 78 insertions(+), 47 deletions(-) diff --git a/src/modules/llms/server/openai/models.data.ts b/src/modules/llms/server/openai/models.data.ts index 707daab03..8695bf281 100644 --- a/src/modules/llms/server/openai/models.data.ts +++ b/src/modules/llms/server/openai/models.data.ts @@ -403,97 +403,137 @@ const _knownMistralChatModels: ManualMappings = [ { idPrefix: 'mistral-large', label: 'Mistral Large (?)', - description: 'Flagship model, with top-tier reasoning capabilities and language support (English, French, German, Italian, Spanish, and Code)', + description: 'Top-tier reasoning for high-complexity tasks.', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], hidden: true, }, - // Medium - not updated on 2024-02-26 + // Open Mixtral (8x22B) + { + idPrefix: 'open-mixtral-8x22b-2404', + label: 'Open Mixtral 8x22B (2404)', + description: 'Mixtral 8x22B model', + contextWindow: 65536, + interfaces: [LLM_IF_OAI_Chat], + pricing: { chatIn: 1.5, chatOut: 4.5 }, + }, + { + idPrefix: 'open-mixtral-8x22b', + label: 'Open Mixtral 8x22B', + symLink: 'open-mixtral-8x22b-2404', + hidden: true, + // copied + description: 'Mixtral 8x22B model', + contextWindow: 65536, + interfaces: [LLM_IF_OAI_Chat], + pricing: { chatIn: 1.5, chatOut: 4.5 }, + }, + // Medium (Deprecated) { idPrefix: 'mistral-medium-2312', label: 'Mistral Medium (2312)', - description: 'Mistral internal prototype model.', + description: 'Ideal for intermediate tasks that require moderate reasoning (Data extraction, Summarizing a Document, Writing emails, Writing a Job Description, or Writing Product Descriptions)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], pricing: { chatIn: 2.7, chatOut: 8.1 }, benchmark: { cbaElo: 1148 }, + isLegacy: true, + hidden: true, }, { idPrefix: 'mistral-medium-latest', label: 'Mistral Medium (latest)', symLink: 'mistral-medium-2312', - hidden: true, // copied - description: 'Mistral internal prototype model.', + description: 'Ideal for intermediate tasks that require moderate reasoning (Data extraction, Summarizing a Document, Writing emails, Writing a Job Description, or Writing Product Descriptions)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], pricing: { chatIn: 2.7, chatOut: 8.1 }, benchmark: { cbaElo: 1148 }, + isLegacy: true, + hidden: true, }, { idPrefix: 'mistral-medium', label: 'Mistral Medium', - description: 'Mistral internal prototype model.', + symLink: 'mistral-medium-2312', + // copied + description: 'Ideal for intermediate tasks that require moderate reasoning (Data extraction, Summarizing a Document, Writing emails, Writing a Job Description, or Writing Product Descriptions)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], pricing: { chatIn: 2.7, chatOut: 8.1 }, benchmark: { cbaElo: 1148 }, + isLegacy: true, hidden: true, }, - // Small (8x7B) + // Open Mixtral (8x7B) -> currenty points to `mistral-small-2312` (as per the docs) { - idPrefix: 'mistral-small-2402', - label: 'Mistral Small (2402)', - description: 'Optimized endpoint. Cost-efficient reasoning for low-latency workloads. Mistral Small outperforms Mixtral 8x7B and has lower latency', + idPrefix: 'open-mixtral-8x7b', + label: 'Open Mixtral (8x7B)', + description: 'A sparse mixture of experts model. As such, it leverages up to 45B parameters but only uses about 12B during inference, leading to better inference throughput at the cost of more vRAM.', contextWindow: 32768, - interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], - pricing: { chatIn: 2, chatOut: 6 }, + interfaces: [LLM_IF_OAI_Chat], + pricing: { chatIn: 0.7, chatOut: 0.7 }, }, + // Small (deprecated) { - idPrefix: 'mistral-small-2312', - label: 'Mistral Small (2312)', - description: 'Aka open-mixtral-8x7b. Cost-efficient reasoning for low-latency workloads. Mistral Small outperforms Mixtral 8x7B and has lower latency', + idPrefix: 'mistral-small-2402', + label: 'Mistral Small (2402)', + description: 'Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, - interfaces: [LLM_IF_OAI_Chat], + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], pricing: { chatIn: 2, chatOut: 6 }, hidden: true, + isLegacy: true, }, { idPrefix: 'mistral-small-latest', label: 'Mistral Small (latest)', symLink: 'mistral-small-2402', - hidden: true, // copied - description: 'Cost-efficient reasoning for low-latency workloads. Mistral Small outperforms Mixtral 8x7B and has lower latency', + description: 'Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], pricing: { chatIn: 2, chatOut: 6 }, + hidden: true, + isLegacy: true, + }, + { + idPrefix: 'mistral-small-2312', + label: 'Mistral Small (2312)', + description: 'Aka open-mixtral-8x7b. Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat], + pricing: { chatIn: 2, chatOut: 6 }, + hidden: true, + isLegacy: true, }, { idPrefix: 'mistral-small', label: 'Mistral Small', - description: 'Cost-efficient reasoning for low-latency workloads.', + symLink: 'mistral-small-2312', + // copied + description: 'Aka open-mixtral-8x7b. Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], pricing: { chatIn: 2, chatOut: 6 }, hidden: true, + isLegacy: true, }, - // Open Mixtral (8x7B) + + // Open Mistral (7B) -> currently points to mistral-tiny-2312 (as per the docs) { - idPrefix: 'open-mixtral-8x7b', - label: 'Open Mixtral (8x7B)', - description: 'Mixtral 8x7B model, aka mistral-small-2312', - // symLink: 'mistral-small-2312', - // copied + idPrefix: 'open-mistral-7b', + label: 'Open Mistral (7B)', + description: 'The first dense model released by Mistral AI, perfect for experimentation, customization, and quick iteration. At the time of the release, it matched the capabilities of models up to 30B parameters.', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 0.7, chatOut: 0.7 }, + pricing: { chatIn: 0.25, chatOut: 0.25 }, }, - - // Tiny (7B) + // Tiny (deprecated) { idPrefix: 'mistral-tiny-2312', label: 'Mistral Tiny (2312)', @@ -501,43 +541,34 @@ const _knownMistralChatModels: ManualMappings = [ contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], hidden: true, + isLegacy: true, }, { idPrefix: 'mistral-tiny', label: 'Mistral Tiny', - description: 'Used for large batch processing tasks where cost is a significant factor but reasoning capabilities are not crucial', - contextWindow: 32768, - interfaces: [LLM_IF_OAI_Chat], - hidden: true, - }, - // Open Mistral (7B) - { - idPrefix: 'open-mistral-7b', - label: 'Open Mistral (7B)', - description: 'Mistral 7B model, aka mistral-tiny-2312', - // symLink: 'mistral-tiny-2312', + symLink: 'mistral-tiny-2312', // copied + description: 'Aka open-mistral-7b. Used for large batch processing tasks where cost is a significant factor but reasoning capabilities are not crucial', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 0.25, chatOut: 0.25 }, + hidden: true, + isLegacy: true, }, { idPrefix: 'mistral-embed', label: 'Mistral Embed', - description: 'State-of-the-art semantic for extracting representation of text extracts.', - // output: 1024 dimensions + description: 'A model that converts text into numerical vectors of embeddings in 1024 dimensions. Embedding models enable retrieval and retrieval-augmented generation applications.', maxCompletionTokens: 1024, // HACK - it's 1024 dimensions, but those are not 'completion tokens' - contextWindow: 32768, // actually unknown, assumed from the other models + contextWindow: 8192, // Updated context window interfaces: [], hidden: true, }, ]; - const mistralModelFamilyOrder = [ - 'mistral-large', 'mistral-medium', 'mistral-small', 'open-mixtral-8x7b', 'mistral-tiny', 'open-mistral-7b', 'mistral-embed', '๐Ÿ”—', + 'mistral-large', 'open-mixtral-8x22b', 'mistral-medium', 'open-mixtral-8x7b', 'mistral-small', 'open-mistral-7b', 'mistral-tiny', 'mistral-embed', '๐Ÿ”—', ]; export function mistralModelToModelDescription(_model: unknown): ModelDescriptionSchema { @@ -553,13 +584,13 @@ export function mistralModelToModelDescription(_model: unknown): ModelDescriptio } export function mistralModelsSort(a: ModelDescriptionSchema, b: ModelDescriptionSchema): number { + if (a.label.startsWith('๐Ÿ”—') && !b.label.startsWith('๐Ÿ”—')) return 1; + if (!a.label.startsWith('๐Ÿ”—') && b.label.startsWith('๐Ÿ”—')) return -1; const aPrefixIndex = mistralModelFamilyOrder.findIndex(prefix => a.id.startsWith(prefix)); const bPrefixIndex = mistralModelFamilyOrder.findIndex(prefix => b.id.startsWith(prefix)); if (aPrefixIndex !== -1 && bPrefixIndex !== -1) { if (aPrefixIndex !== bPrefixIndex) return aPrefixIndex - bPrefixIndex; - if (a.label.startsWith('๐Ÿ”—') && !b.label.startsWith('๐Ÿ”—')) return 1; - if (!a.label.startsWith('๐Ÿ”—') && b.label.startsWith('๐Ÿ”—')) return -1; return b.label.localeCompare(a.label); } return aPrefixIndex !== -1 ? 1 : -1; From 45b7ed32203f9be79d447b4ac80dd423bae03dec Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 16 May 2024 03:58:11 -0700 Subject: [PATCH 20/27] Mistral: update pricing (cherry picked from commit 05aa4b547fc38df9bf19bd1719a33da59cfcc3ca) --- src/modules/llms/server/openai/models.data.ts | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/modules/llms/server/openai/models.data.ts b/src/modules/llms/server/openai/models.data.ts index 8695bf281..23f6237a4 100644 --- a/src/modules/llms/server/openai/models.data.ts +++ b/src/modules/llms/server/openai/models.data.ts @@ -376,6 +376,8 @@ export function localAIModelToModelDescription(modelId: string): ModelDescriptio // [Mistral] +// updated from the models on: https://docs.mistral.ai/getting-started/models/ +// and the pricing available on: https://mistral.ai/technology/#pricing const _knownMistralChatModels: ManualMappings = [ // Large @@ -385,7 +387,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Top-tier reasoning for high-complexity tasks.', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], - pricing: { chatIn: 8, chatOut: 24 }, + pricing: { chatIn: 4, chatOut: 12 }, benchmark: { cbaElo: 1159 }, }, { @@ -397,17 +399,9 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Top-tier reasoning for high-complexity tasks.', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], - pricing: { chatIn: 8, chatOut: 24 }, + pricing: { chatIn: 4, chatOut: 12 }, benchmark: { cbaElo: 1159 }, }, - { - idPrefix: 'mistral-large', - label: 'Mistral Large (?)', - description: 'Top-tier reasoning for high-complexity tasks.', - contextWindow: 32768, - interfaces: [LLM_IF_OAI_Chat], - hidden: true, - }, // Open Mixtral (8x22B) { @@ -416,7 +410,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Mixtral 8x22B model', contextWindow: 65536, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 1.5, chatOut: 4.5 }, + pricing: { chatIn: 2, chatOut: 6 }, }, { idPrefix: 'open-mixtral-8x22b', @@ -427,7 +421,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Mixtral 8x22B model', contextWindow: 65536, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 1.5, chatOut: 4.5 }, + pricing: { chatIn: 2, chatOut: 6 }, }, // Medium (Deprecated) { @@ -468,7 +462,7 @@ const _knownMistralChatModels: ManualMappings = [ hidden: true, }, - // Open Mixtral (8x7B) -> currenty points to `mistral-small-2312` (as per the docs) + // Open Mixtral (8x7B) -> currently points to `mistral-small-2312` (as per the docs) { idPrefix: 'open-mixtral-8x7b', label: 'Open Mixtral (8x7B)', @@ -484,7 +478,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], - pricing: { chatIn: 2, chatOut: 6 }, + pricing: { chatIn: 1, chatOut: 3 }, hidden: true, isLegacy: true, }, @@ -496,7 +490,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], - pricing: { chatIn: 2, chatOut: 6 }, + pricing: { chatIn: 1, chatOut: 3 }, hidden: true, isLegacy: true, }, @@ -506,7 +500,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Aka open-mixtral-8x7b. Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 2, chatOut: 6 }, + pricing: { chatIn: 1, chatOut: 3 }, hidden: true, isLegacy: true, }, @@ -518,7 +512,7 @@ const _knownMistralChatModels: ManualMappings = [ description: 'Aka open-mixtral-8x7b. Suitable for simple tasks that one can do in bulk (Classification, Customer Support, or Text Generation)', contextWindow: 32768, interfaces: [LLM_IF_OAI_Chat], - pricing: { chatIn: 2, chatOut: 6 }, + pricing: { chatIn: 1, chatOut: 3 }, hidden: true, isLegacy: true, }, @@ -555,7 +549,6 @@ const _knownMistralChatModels: ManualMappings = [ isLegacy: true, }, - { idPrefix: 'mistral-embed', label: 'Mistral Embed', @@ -563,6 +556,7 @@ const _knownMistralChatModels: ManualMappings = [ maxCompletionTokens: 1024, // HACK - it's 1024 dimensions, but those are not 'completion tokens' contextWindow: 8192, // Updated context window interfaces: [], + pricing: { chatIn: 0.1, chatOut: 0.1 }, hidden: true, }, ]; From ba66fc30c5da2fb9ec613949f51991d8b76abdba Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Sat, 18 May 2024 03:07:04 -0700 Subject: [PATCH 21/27] Fix TimeoutError issue (cherry picked from commit 7c099cab949e3d039b25512c41cc42b23a7c6fca) --- src/modules/browse/browse.router.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 3e11247d0..943a89788 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import { TRPCError } from '@trpc/server'; -import { BrowserContext, connect, ScreenshotOptions, TimeoutError } from '@cloudflare/puppeteer'; +import { BrowserContext, connect, ScreenshotOptions } from '@cloudflare/puppeteer'; import { default as TurndownService } from 'turndown'; import { load as cheerioLoad } from 'cheerio'; @@ -134,7 +134,8 @@ async function workerPuppeteer( result.stopReason = 'end'; } } catch (error: any) { - const isTimeout = error instanceof TimeoutError; + // This was "error instanceof TimeoutError;" but threw some type error - trying the below instead + const isTimeout = error?.message?.includes('Navigation timeout') || false; result.stopReason = isTimeout ? 'timeout' : 'error'; if (!isTimeout) { result.error = '[Puppeteer] ' + (error?.message || error?.toString() || 'Unknown goto error'); From 7fbf6ee2e800148f88e735c1380ca2b2d86a1770 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Tue, 21 May 2024 23:50:32 -0700 Subject: [PATCH 22/27] Fix Domino issue (crash) by upgrading Turndown to 7.2.0 See: https://github.com/mixmark-io/turndown/issues/439 https://github.com/fgnass/domino/issues/146 (cherry picked from commit baad3ae1c35c49f08497cbfac1dad9eb2e021882) --- package-lock.json | 20 ++++++++++---------- package.json | 2 +- src/modules/browse/browse.router.ts | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3aa41654f..df14df364 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "superjson": "^2.2.1", "tesseract.js": "^5.1.0", "tiktoken": "^1.0.15", - "turndown": "^7.1.3", + "turndown": "^7.2.0", "uuid": "^9.0.1", "zod": "^3.23.8", "zustand": "^4.5.2" @@ -1027,6 +1027,11 @@ "node-pre-gyp": "bin/node-pre-gyp" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==" + }, "node_modules/@mui/base": { "version": "5.0.0-beta.42", "resolved": "https://registry.npmjs.org/@mui/base/-/base-5.0.0-beta.42.tgz", @@ -3328,11 +3333,6 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, - "node_modules/domino": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", - "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" - }, "node_modules/domutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", @@ -8443,11 +8443,11 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/turndown": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz", - "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz", + "integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==", "dependencies": { - "domino": "^2.1.6" + "@mixmark-io/domino": "^2.2.0" } }, "node_modules/type-check": { diff --git a/package.json b/package.json index d32a68e05..b7c6e17c8 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "superjson": "^2.2.1", "tesseract.js": "^5.1.0", "tiktoken": "^1.0.15", - "turndown": "^7.1.3", + "turndown": "^7.2.0", "uuid": "^9.0.1", "zod": "^3.23.8", "zustand": "^4.5.2" diff --git a/src/modules/browse/browse.router.ts b/src/modules/browse/browse.router.ts index 943a89788..128050563 100644 --- a/src/modules/browse/browse.router.ts +++ b/src/modules/browse/browse.router.ts @@ -165,7 +165,7 @@ async function workerPuppeteer( result.error = '[Puppeteer] Empty content'; } } catch (error: any) { - result.error = '[Puppeteer] ' + (error?.message || error?.toString() || 'Unknown evaluate error'); + result.error = '[Puppeteer] ' + (error?.message || error?.toString() || 'Unknown content error'); } // get a screenshot of the page From 07b62fe5c1261557ad380a3cf73a9f2142b638f5 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 6 Jun 2024 21:25:34 -0700 Subject: [PATCH 23/27] Streaming uplink: index sources for unification. --- src/apps/call/Telephone.tsx | 8 +++---- src/apps/chat/editors/chat-stream.ts | 8 +++++-- src/apps/personas/creator/Creator.tsx | 10 ++++++--- src/modules/aifn/digrams/DiagramsModal.tsx | 6 +++--- src/modules/aifn/flatten/FlattenerModal.tsx | 2 +- src/modules/aifn/useLLMChain.ts | 8 +++---- src/modules/aifn/useStreamChatText.ts | 6 +++--- .../instructions/ChatGenerateInstruction.tsx | 2 +- .../instructions/beam.gather.execution.tsx | 2 ++ src/modules/beam/scatter/beam.scatter.ts | 2 +- src/modules/llms/llm.client.ts | 14 ++++++++++++- .../llms/server/llm.server.streaming.ts | 21 ++++++++++++------- src/modules/llms/vendors/IModelVendor.ts | 3 ++- .../llms/vendors/unifiedStreamingClient.ts | 8 ++++++- 14 files changed, 68 insertions(+), 32 deletions(-) diff --git a/src/apps/call/Telephone.tsx b/src/apps/call/Telephone.tsx index 741a33ee5..5f333de2c 100644 --- a/src/apps/call/Telephone.tsx +++ b/src/apps/call/Telephone.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { shallow } from 'zustand/shallow'; +import { useShallow } from 'zustand/react/shallow'; import { Box, Card, ListDivider, ListItemDecorator, MenuItem, Switch, Typography } from '@mui/joy'; import ArrowBackIcon from '@mui/icons-material/ArrowBack'; @@ -99,7 +99,7 @@ export function Telephone(props: { // external state const { chatLLMId, chatLLMDropdown } = useChatLLMDropdown(); - const { chatTitle, reMessages } = useChatStore(state => { + const { chatTitle, reMessages } = useChatStore(useShallow(state => { const conversation = props.callIntent.conversationId ? state.conversations.find(conversation => conversation.id === props.callIntent.conversationId) ?? null : null; @@ -107,7 +107,7 @@ export function Telephone(props: { chatTitle: conversation ? conversationTitle(conversation) : null, reMessages: conversation ? conversation.messages : null, }; - }, shallow); + })); const persona = SystemPurposes[props.callIntent.personaId as SystemPurposeId] ?? undefined; const personaCallStarters = persona?.call?.starters ?? undefined; const personaVoiceId = overridePersonaVoice ? undefined : (persona?.voices?.elevenLabs?.voiceId ?? undefined); @@ -225,7 +225,7 @@ export function Telephone(props: { let finalText = ''; let error: any | null = null; setPersonaTextInterim('๐Ÿ’ญ...'); - llmStreamingChatGenerate(chatLLMId, callPrompt, null, null, responseAbortController.current.signal, ({ textSoFar }) => { + llmStreamingChatGenerate(chatLLMId, callPrompt, 'call', callMessages[0].id, null, null, responseAbortController.current.signal, ({ textSoFar }) => { const text = textSoFar?.trim(); if (text) { finalText = text; diff --git a/src/apps/chat/editors/chat-stream.ts b/src/apps/chat/editors/chat-stream.ts index 5b19bc961..f51290c59 100644 --- a/src/apps/chat/editors/chat-stream.ts +++ b/src/apps/chat/editors/chat-stream.ts @@ -2,7 +2,7 @@ import type { DLLMId } from '~/modules/llms/store-llms'; import type { StreamingClientUpdate } from '~/modules/llms/vendors/unifiedStreamingClient'; import { autoSuggestions } from '~/modules/aifn/autosuggestions/autoSuggestions'; import { conversationAutoTitle } from '~/modules/aifn/autotitle/autoTitle'; -import { llmStreamingChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; +import { llmStreamingChatGenerate, VChatContextRef, VChatContextName, VChatMessageIn } from '~/modules/llms/llm.client'; import { speakText } from '~/modules/elevenlabs/elevenlabs.client'; import type { DMessage } from '~/common/state/store-chats'; @@ -34,6 +34,8 @@ export async function runAssistantUpdatingState(conversationId: string, history: const messageStatus = await streamAssistantMessage( assistantLlmId, history.map((m): VChatMessageIn => ({ role: m.role, content: m.text })), + 'conversation', + conversationId, parallelViewCount, autoSpeak, (update) => cHandler.messageEdit(assistantMessageId, update, false), @@ -61,6 +63,8 @@ type StreamMessageStatus = { outcome: StreamMessageOutcome, errorMessage?: strin export async function streamAssistantMessage( llmId: DLLMId, messagesHistory: VChatMessageIn[], + contextName: VChatContextName, + contextRef: VChatContextRef, throttleUnits: number, // 0: disable, 1: default throttle (12Hz), 2+ reduce the message frequency with the square root autoSpeak: ChatAutoSpeakType, editMessage: (update: Partial) => void, @@ -92,7 +96,7 @@ export async function streamAssistantMessage( const incrementalAnswer: Partial = { text: '' }; try { - await llmStreamingChatGenerate(llmId, messagesHistory, null, null, abortSignal, (update: StreamingClientUpdate) => { + await llmStreamingChatGenerate(llmId, messagesHistory, contextName, contextRef, null, null, abortSignal, (update: StreamingClientUpdate) => { const textSoFar = update.textSoFar; // grow the incremental message diff --git a/src/apps/personas/creator/Creator.tsx b/src/apps/personas/creator/Creator.tsx index 7bc70d7b0..23a4e81c4 100644 --- a/src/apps/personas/creator/Creator.tsx +++ b/src/apps/personas/creator/Creator.tsx @@ -1,4 +1,5 @@ import * as React from 'react'; +import { v4 as uuidv4 } from 'uuid'; import { Alert, Box, Button, Card, CardContent, CircularProgress, Divider, FormLabel, Grid, IconButton, LinearProgress, Tab, tabClasses, TabList, TabPanel, Tabs, Typography } from '@mui/joy'; import AddIcon from '@mui/icons-material/Add'; @@ -102,8 +103,11 @@ export function Creator(props: { display: boolean }) { strings: editedInstructions, stringEditors: instructionEditors, } = useFormEditTextArray(Prompts, PromptTitles); - const creationChainSteps = React.useMemo(() => { - return createChain(editedInstructions, PromptTitles); + const { steps: creationChainSteps, id: chainId } = React.useMemo(() => { + return { + steps: createChain(editedInstructions, PromptTitles), + id: uuidv4(), + }; }, [editedInstructions]); const llmLabel = personaLlm?.label || undefined; @@ -122,7 +126,7 @@ export function Creator(props: { display: boolean }) { chainError, userCancelChain, restartChain, - } = useLLMChain(creationChainSteps, personaLlm?.id, chainInputText ?? undefined, savePersona); + } = useLLMChain(creationChainSteps, personaLlm?.id, chainInputText ?? undefined, savePersona, 'persona-extract', chainId); // Reset the relevant state when the selected tab changes diff --git a/src/modules/aifn/digrams/DiagramsModal.tsx b/src/modules/aifn/digrams/DiagramsModal.tsx index 55e092f5f..c50a3a295 100644 --- a/src/modules/aifn/digrams/DiagramsModal.tsx +++ b/src/modules/aifn/digrams/DiagramsModal.tsx @@ -68,7 +68,7 @@ export function DiagramsModal(props: { config: DiagramConfig, onClose: () => voi const [diagramLlm, llmComponent] = useFormRadioLlmType('Generator', 'chat'); // derived state - const { conversationId, text: subject } = props.config; + const { conversationId, messageId, text: subject } = props.config; const diagramLlmId = diagramLlm?.id; @@ -98,7 +98,7 @@ export function DiagramsModal(props: { config: DiagramConfig, onClose: () => voi const diagramPrompt = bigDiagramPrompt(diagramType, diagramLanguage, systemMessage.text, subject, customInstruction); try { - await llmStreamingChatGenerate(diagramLlm.id, diagramPrompt, null, null, stepAbortController.signal, + await llmStreamingChatGenerate(diagramLlm.id, diagramPrompt, 'ai-diagram', messageId, null, null, stepAbortController.signal, ({ textSoFar }) => textSoFar && setDiagramCode(diagramCode = textSoFar), ); } catch (error: any) { @@ -109,7 +109,7 @@ export function DiagramsModal(props: { config: DiagramConfig, onClose: () => voi setAbortController(null); } - }, [abortController, conversationId, diagramLanguage, diagramLlm, diagramType, subject, customInstruction]); + }, [abortController, conversationId, customInstruction, diagramLanguage, diagramLlm, diagramType, messageId, subject]); // [Effect] Auto-abort on unmount diff --git a/src/modules/aifn/flatten/FlattenerModal.tsx b/src/modules/aifn/flatten/FlattenerModal.tsx index 088498689..8a3723852 100644 --- a/src/modules/aifn/flatten/FlattenerModal.tsx +++ b/src/modules/aifn/flatten/FlattenerModal.tsx @@ -117,7 +117,7 @@ export function FlattenerModal(props: { await startStreaming(llm.id, [ { role: 'system', content: flattenProfile.systemPrompt }, { role: 'user', content: encodeConversationAsUserMessage(flattenProfile.userPrompt, messages) }, - ]); + ], 'ai-flattener', messages[0].id); }, [llm, props.conversationId, startStreaming]); diff --git a/src/modules/aifn/useLLMChain.ts b/src/modules/aifn/useLLMChain.ts index 091ca03fd..1e7a654ee 100644 --- a/src/modules/aifn/useLLMChain.ts +++ b/src/modules/aifn/useLLMChain.ts @@ -1,7 +1,7 @@ import * as React from 'react'; import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms'; -import { llmStreamingChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; +import { llmStreamingChatGenerate, VChatContextName, VChatContextRef, VChatMessageIn } from '~/modules/llms/llm.client'; // set to true to log to the console @@ -20,7 +20,7 @@ export interface LLMChainStep { /** * React hook to manage a chain of LLM transformations. */ -export function useLLMChain(steps: LLMChainStep[], llmId: DLLMId | undefined, chainInput: string | undefined, onSuccess?: (output: string, input: string) => void) { +export function useLLMChain(steps: LLMChainStep[], llmId: DLLMId | undefined, chainInput: string | undefined, onSuccess: (output: string, input: string) => void, contextName: VChatContextName, contextRef: VChatContextRef) { // state const [chain, setChain] = React.useState(null); @@ -114,7 +114,7 @@ export function useLLMChain(steps: LLMChainStep[], llmId: DLLMId | undefined, ch setChainStepInterimText(null); // LLM call (streaming, cancelable) - llmStreamingChatGenerate(llmId, llmChatInput, null, null, stepAbortController.signal, + llmStreamingChatGenerate(llmId, llmChatInput, contextName, contextRef, null, null, stepAbortController.signal, ({ textSoFar }) => { textSoFar && setChainStepInterimText(interimText = textSoFar); }) @@ -141,7 +141,7 @@ export function useLLMChain(steps: LLMChainStep[], llmId: DLLMId | undefined, ch stepAbortController.abort('step aborted'); _chainAbortController.signal.removeEventListener('abort', globalToStepListener); }; - }, [chain, llmId, onSuccess]); + }, [chain, contextRef, contextName, llmId, onSuccess]); return { diff --git a/src/modules/aifn/useStreamChatText.ts b/src/modules/aifn/useStreamChatText.ts index b7158e8dc..11b856f7e 100644 --- a/src/modules/aifn/useStreamChatText.ts +++ b/src/modules/aifn/useStreamChatText.ts @@ -1,7 +1,7 @@ import * as React from 'react'; import type { DLLMId } from '~/modules/llms/store-llms'; -import { llmStreamingChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; +import { llmStreamingChatGenerate, VChatContextName, VChatContextRef, VChatMessageIn } from '~/modules/llms/llm.client'; export function useStreamChatText() { @@ -13,7 +13,7 @@ export function useStreamChatText() { const abortControllerRef = React.useRef(null); - const startStreaming = React.useCallback(async (llmId: DLLMId, prompt: VChatMessageIn[]) => { + const startStreaming = React.useCallback(async (llmId: DLLMId, prompt: VChatMessageIn[], contextName: VChatContextName, contextRef: VChatContextRef) => { setStreamError(null); setPartialText(null); setText(null); @@ -24,7 +24,7 @@ export function useStreamChatText() { try { let lastText = ''; - await llmStreamingChatGenerate(llmId, prompt, null, null, abortControllerRef.current.signal, ({ textSoFar }) => { + await llmStreamingChatGenerate(llmId, prompt, contextName, contextRef, null, null, abortControllerRef.current.signal, ({ textSoFar }) => { if (textSoFar) { lastText = textSoFar; setPartialText(lastText); diff --git a/src/modules/beam/gather/instructions/ChatGenerateInstruction.tsx b/src/modules/beam/gather/instructions/ChatGenerateInstruction.tsx index 1f8452729..a00fd8384 100644 --- a/src/modules/beam/gather/instructions/ChatGenerateInstruction.tsx +++ b/src/modules/beam/gather/instructions/ChatGenerateInstruction.tsx @@ -96,7 +96,7 @@ export async function executeChatGenerate(_i: ChatGenerateInstruction, inputs: E }; // LLM Streaming generation - return streamAssistantMessage(inputs.llmId, history, getUXLabsHighPerformance() ? 0 : 1, 'off', onMessageUpdate, inputs.chainAbortController.signal) + return streamAssistantMessage(inputs.llmId, history, 'beam-gather', inputs.contextRef, getUXLabsHighPerformance() ? 0 : 1, 'off', onMessageUpdate, inputs.chainAbortController.signal) .then((status) => { // re-throw errors, as streamAssistantMessage catches internally if (status.outcome === 'aborted') { diff --git a/src/modules/beam/gather/instructions/beam.gather.execution.tsx b/src/modules/beam/gather/instructions/beam.gather.execution.tsx index 1836f9cf0..9419e6cc6 100644 --- a/src/modules/beam/gather/instructions/beam.gather.execution.tsx +++ b/src/modules/beam/gather/instructions/beam.gather.execution.tsx @@ -23,6 +23,7 @@ export interface ExecutionInputState { readonly chatMessages: DMessage[]; readonly rayMessages: DMessage[]; readonly llmId: DLLMId; + readonly contextRef: string; // not useful // interaction readonly chainAbortController: AbortController; readonly updateProgressComponent: (component: React.ReactNode) => void; @@ -67,6 +68,7 @@ export function gatherStartFusion( chatMessages: chatMessages, rayMessages: rayMessages, llmId: initialFusion.llmId, + contextRef: initialFusion.fusionId, // interaction chainAbortController: new AbortController(), updateProgressComponent: (component: React.ReactNode) => onUpdateBFusion({ fusingProgressComponent: component }), diff --git a/src/modules/beam/scatter/beam.scatter.ts b/src/modules/beam/scatter/beam.scatter.ts index d229a6d30..0b9cb51ac 100644 --- a/src/modules/beam/scatter/beam.scatter.ts +++ b/src/modules/beam/scatter/beam.scatter.ts @@ -67,7 +67,7 @@ function rayScatterStart(ray: BRay, llmId: DLLMId | null, inputHistory: DMessage // stream the assistant's messages const messagesHistory: VChatMessageIn[] = inputHistory.map(({ role, text }) => ({ role, content: text })); - streamAssistantMessage(llmId, messagesHistory, getUXLabsHighPerformance() ? 0 : rays.length, 'off', updateMessage, abortController.signal) + streamAssistantMessage(llmId, messagesHistory, 'beam-scatter', ray.rayId, getUXLabsHighPerformance() ? 0 : rays.length, 'off', updateMessage, abortController.signal) .then((status) => { _rayUpdate(ray.rayId, { status: (status.outcome === 'success') ? 'success' diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index 55b70dd09..3a6bf5804 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -21,6 +21,16 @@ export interface VChatMessageIn { export type VChatFunctionIn = OpenAIWire.ChatCompletion.RequestFunctionDef; +export type VChatContextName = + | 'conversation' + | 'ai-diagram' + | 'ai-flattener' + | 'beam-scatter' + | 'beam-gather' + | 'call' + | 'persona-extract'; +export type VChatContextRef = string; + export interface VChatMessageOut { role: 'assistant' | 'system' | 'user'; content: string; @@ -139,6 +149,8 @@ export async function llmChatGenerateOrThrow( llmId: DLLMId, messages: VChatMessageIn[], + contextName: VChatContextName, + contextRef: VChatContextRef, functions: VChatFunctionIn[] | null, forceFunctionName: string | null, abortSignal: AbortSignal, @@ -161,5 +173,5 @@ export async function llmStreamingChatGenerate setTimeout(resolve, delay)); // execute via the vendor - return await vendor.streamingChatGenerateOrThrow(access, llmId, llmOptions, messages, functions, forceFunctionName, abortSignal, onUpdate); + return await vendor.streamingChatGenerateOrThrow(access, llmId, llmOptions, messages, contextName, contextRef, functions, forceFunctionName, abortSignal, onUpdate); } diff --git a/src/modules/llms/server/llm.server.streaming.ts b/src/modules/llms/server/llm.server.streaming.ts index 88b7b4e2e..03c788782 100644 --- a/src/modules/llms/server/llm.server.streaming.ts +++ b/src/modules/llms/server/llm.server.streaming.ts @@ -19,7 +19,7 @@ import { OLLAMA_PATH_CHAT, ollamaAccess, ollamaAccessSchema, ollamaChatCompletio // OpenAI server imports import type { OpenAIWire } from './openai/openai.wiretypes'; -import { openAIAccess, openAIAccessSchema, openAIChatCompletionPayload, OpenAIHistorySchema, openAIHistorySchema, OpenAIModelSchema, openAIModelSchema } from './openai/openai.router'; +import { openAIAccess, openAIAccessSchema, openAIChatCompletionPayload, openAIHistorySchema, openAIModelSchema } from './openai/openai.router'; // configuration @@ -46,11 +46,17 @@ type MuxingFormat = 'sse' | 'json-nl'; */ type AIStreamParser = (data: string, eventType?: string) => { text: string, close: boolean }; +const streamingContextSchema = z.object({ + method: z.literal('chat-stream'), + name: z.enum(['conversation', 'ai-diagram', 'ai-flattener', 'call', 'beam-scatter', 'beam-gather', 'persona-extract']), + ref: z.string(), +}); const chatStreamingInputSchema = z.object({ access: z.union([anthropicAccessSchema, geminiAccessSchema, ollamaAccessSchema, openAIAccessSchema]), model: openAIModelSchema, history: openAIHistorySchema, + context: streamingContextSchema, }); export type ChatStreamingInputSchema = z.infer; @@ -72,14 +78,15 @@ export async function llmStreamingRelayHandler(req: NextRequest): Promise; try { - requestData = _prepareRequestData(access, model, history); + requestData = _prepareRequestData(_chatStreamingInput); } catch (error: any) { console.error(`[POST] /api/llms/stream: ${prettyDialect}: prepareRequestData issue:`, safeErrorString(error)); return new NextResponse(`**[Service Issue] ${prettyDialect}**: ${safeErrorString(error) || 'Unknown streaming error'}`, { @@ -103,7 +110,7 @@ export async function llmStreamingRelayHandler(req: NextRequest): Promise> { @@ -53,6 +53,7 @@ export interface IModelVendor void, diff --git a/src/modules/llms/vendors/unifiedStreamingClient.ts b/src/modules/llms/vendors/unifiedStreamingClient.ts index 07094baa6..c8aea7577 100644 --- a/src/modules/llms/vendors/unifiedStreamingClient.ts +++ b/src/modules/llms/vendors/unifiedStreamingClient.ts @@ -3,7 +3,7 @@ import { frontendSideFetch } from '~/common/util/clientFetchers'; import type { ChatStreamingInputSchema, ChatStreamingPreambleModelSchema, ChatStreamingPreambleStartSchema } from '../server/llm.server.streaming'; import type { DLLMId } from '../store-llms'; -import type { VChatFunctionIn, VChatMessageIn } from '../llm.client'; +import type { VChatContextName, VChatContextRef, VChatFunctionIn, VChatMessageIn } from '../llm.client'; import type { OpenAIAccessSchema } from '../server/openai/openai.router'; import type { OpenAIWire } from '../server/openai/openai.wiretypes'; @@ -29,6 +29,7 @@ export async function unifiedStreamingClient void, @@ -55,6 +56,11 @@ export async function unifiedStreamingClient Date: Thu, 6 Jun 2024 09:58:16 -0700 Subject: [PATCH 24/27] Add Codestral - Fixes #558 (cherry picked from commit 4075581acd83c4cb9c483e7818efa8a91e41a599) --- src/modules/llms/server/openai/models.data.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/modules/llms/server/openai/models.data.ts b/src/modules/llms/server/openai/models.data.ts index 23f6237a4..57400f69d 100644 --- a/src/modules/llms/server/openai/models.data.ts +++ b/src/modules/llms/server/openai/models.data.ts @@ -380,6 +380,27 @@ export function localAIModelToModelDescription(modelId: string): ModelDescriptio // and the pricing available on: https://mistral.ai/technology/#pricing const _knownMistralChatModels: ManualMappings = [ + // Codestral + { + idPrefix: 'codestral-2405', + label: 'Codestral (2405)', + description: 'Designed and optimized for code generation tasks.', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + pricing: { chatIn: 1, chatOut: 3 }, + }, + { + idPrefix: 'codestral-latest', + label: 'Mistral Large (latest)', + symLink: 'mistral-codestral-2405', + hidden: true, + // copied + description: 'Designed and optimized for code generation tasks.', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + pricing: { chatIn: 1, chatOut: 3 }, + }, + // Large { idPrefix: 'mistral-large-2402', @@ -562,7 +583,7 @@ const _knownMistralChatModels: ManualMappings = [ ]; const mistralModelFamilyOrder = [ - 'mistral-large', 'open-mixtral-8x22b', 'mistral-medium', 'open-mixtral-8x7b', 'mistral-small', 'open-mistral-7b', 'mistral-tiny', 'mistral-embed', '๐Ÿ”—', + 'codestral', 'mistral-large', 'open-mixtral-8x22b', 'mistral-medium', 'open-mixtral-8x7b', 'mistral-small', 'open-mistral-7b', 'mistral-tiny', 'mistral-embed', '๐Ÿ”—', ]; export function mistralModelToModelDescription(_model: unknown): ModelDescriptionSchema { From bd5bf6f94f179abeab75eaa556e1718786f7a023 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 6 Jun 2024 10:10:06 -0700 Subject: [PATCH 25/27] Gemini: update (cherry picked from commit 1429726ba6d2bbc8733558613054ff4db38ec328) --- .../llms/server/gemini/gemini.models.ts | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/modules/llms/server/gemini/gemini.models.ts b/src/modules/llms/server/gemini/gemini.models.ts index 4ec9402a6..899a1155f 100644 --- a/src/modules/llms/server/gemini/gemini.models.ts +++ b/src/modules/llms/server/gemini/gemini.models.ts @@ -26,26 +26,72 @@ const _knownGeminiModels: ({ // Generation 1.5 { - id: 'models/gemini-1.5-flash-latest', + id: 'models/gemini-1.5-flash-latest', // updated regularly and might be a preview version isLatest: true, isPreview: true, pricing: { chatIn: 0.70, // 0.35 up to 128k tokens, 0.70 prompts > 128k tokens - chatOut: 1.05, // 0.53 up to 128k tokens, 1.05 prompts > 128k tokens + chatOut: 2.10, // 1.05 up to 128k tokens, 2.10 prompts > 128k tokens }, + trainingDataCutoff: 'May 2024', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text }, { - id: 'models/gemini-1.5-pro-latest', - // NOTE: no 'models/gemini-1.5-pro' (latest stable) as of 2024-05-14 + id: 'models/gemini-1.5-flash', + // copied from above + pricing: { + chatIn: 0.70, // 0.35 up to 128k tokens, 0.70 prompts > 128k tokens + chatOut: 2.10, // 1.05 up to 128k tokens, 2.10 prompts > 128k tokens + }, + trainingDataCutoff: 'Apr 2024', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text + hidden: true, + }, + { + id: 'models/gemini-1.5-flash-001', + // copied from above + pricing: { + chatIn: 0.70, // 0.35 up to 128k tokens, 0.70 prompts > 128k tokens + chatOut: 2.10, // 1.05 up to 128k tokens, 2.10 prompts > 128k tokens + }, + trainingDataCutoff: 'Apr 2024', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text + hidden: true, + }, + + { + id: 'models/gemini-1.5-pro-latest', // updated regularly and might be a preview version isLatest: true, + isPreview: true, pricing: { chatIn: 7.00, // $3.50 / 1 million tokens (for prompts up to 128K tokens), $7.00 / 1 million tokens (for prompts longer than 128K) chatOut: 21.00, // $10.50 / 1 million tokens (128K or less), $21.00 / 1 million tokens (128K+) }, - trainingDataCutoff: 'Apr 2024', + trainingDataCutoff: 'May 2024', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], // input: audio, images and text }, + { + id: 'models/gemini-1.5-pro', // latest stable -> 001 + // copied from above + pricing: { + chatIn: 7.00, // $3.50 / 1 million tokens (for prompts up to 128K tokens), $7.00 / 1 million tokens (for prompts longer than 128K) + chatOut: 21.00, // $10.50 / 1 million tokens (128K or less), $21.00 / 1 million tokens (128K+) + }, + trainingDataCutoff: 'Apr 2024', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], + hidden: true, + }, + { + id: 'models/gemini-1.5-pro-001', // stable snapshot + // copied from above + pricing: { + chatIn: 7.00, // $3.50 / 1 million tokens (for prompts up to 128K tokens), $7.00 / 1 million tokens (for prompts longer than 128K) + chatOut: 21.00, // $10.50 / 1 million tokens (128K or less), $21.00 / 1 million tokens (128K+) + }, + trainingDataCutoff: 'Apr 2024', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Json], + hidden: true, + }, // Generation 1.0 From 94ef76c67e8c2be52664df033a939fd60e6d49dc Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 6 Jun 2024 10:13:37 -0700 Subject: [PATCH 26/27] Gemini: update (cherry picked from commit 3050b546ac97c5289dd2f7fadf24334a5dce036a) --- src/modules/llms/server/gemini/gemini.models.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/modules/llms/server/gemini/gemini.models.ts b/src/modules/llms/server/gemini/gemini.models.ts index 899a1155f..e147913a4 100644 --- a/src/modules/llms/server/gemini/gemini.models.ts +++ b/src/modules/llms/server/gemini/gemini.models.ts @@ -19,7 +19,7 @@ const filterUnallowedInterfaces: GeminiModelSchema['supportedGenerationMethods'] */ const _knownGeminiModels: ({ id: string, - isLatest?: boolean, + isNewest?: boolean, isPreview?: boolean symLink?: string } & Pick)[] = [ @@ -27,7 +27,7 @@ const _knownGeminiModels: ({ // Generation 1.5 { id: 'models/gemini-1.5-flash-latest', // updated regularly and might be a preview version - isLatest: true, + isNewest: true, isPreview: true, pricing: { chatIn: 0.70, // 0.35 up to 128k tokens, 0.70 prompts > 128k tokens @@ -61,7 +61,7 @@ const _knownGeminiModels: ({ { id: 'models/gemini-1.5-pro-latest', // updated regularly and might be a preview version - isLatest: true, + isNewest: true, isPreview: true, pricing: { chatIn: 7.00, // $3.50 / 1 million tokens (for prompts up to 128K tokens), $7.00 / 1 million tokens (for prompts longer than 128K) @@ -97,7 +97,6 @@ const _knownGeminiModels: ({ // Generation 1.0 { id: 'models/gemini-1.0-pro-latest', - isLatest: true, pricing: { chatIn: 0.50, chatOut: 1.50, @@ -206,7 +205,7 @@ export function geminiModelToModelDescription(geminiModel: GeminiModelSchema): M return { id: modelId, - label, + label: label, // + (knownModel?.isNewest ? ' ๐ŸŒŸ' : ''), // created: ... // updated: ... description: descriptionLong, From 0c15476dd263694e64dd5beb73606da4f19c38d0 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 6 Jun 2024 22:03:58 -0700 Subject: [PATCH 27/27] 1.16.2: release --- README.md | 10 +++++++++- docs/changelog.md | 10 +++++++++- src/apps/news/news.data.tsx | 6 ++++-- src/apps/news/news.version.tsx | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 21cf58f88..10996a82e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,15 @@ Or fork & run on Vercel [//]: # (big-AGI is an open book; see the **[ready-to-ship and future ideas](https://github.com/users/enricoros/projects/4/views/2)** in our open roadmap) -### What's New in 1.16.1 ยท May 13, 2024 (minor release, models support) +### What's New in 1.16.2 ยท Jun 7, 2024 (minor release) + +- Improve web downloads, as text, markdwon, or HTML +- Proper support for Gemini models +- Added the latest Mistral model +- Tokenizer support for gpt-4o +- Updates to Beam + +### What's New in 1.16.1 ยท May 13, 2024 (minor release) - Support for the new OpenAI GPT-4o 2024-05-13 model diff --git a/docs/changelog.md b/docs/changelog.md index 148e322ae..ba3a2a3ef 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,7 +10,15 @@ by release. - milestone: [1.17.0](https://github.com/enricoros/big-agi/milestone/17) - work in progress: [big-AGI open roadmap](https://github.com/users/enricoros/projects/4/views/2), [help here](https://github.com/users/enricoros/projects/4/views/4) -### What's New in 1.16.1 ยท May 13, 2024 (minor release, models support) +### What's New in 1.16.2 ยท Jun 7, 2024 (minor release) + +- Improve web downloads, as text, markdwon, or HTML +- Proper support for Gemini models +- Added the latest Mistral model +- Tokenizer support for gpt-4o +- Updates to Beam + +### What's New in 1.16.1 ยท May 13, 2024 (minor release) - Support for the new OpenAI GPT-4o 2024-05-13 model diff --git a/src/apps/news/news.data.tsx b/src/apps/news/news.data.tsx index 30f5f6fba..0fe6eee40 100644 --- a/src/apps/news/news.data.tsx +++ b/src/apps/news/news.data.tsx @@ -61,9 +61,10 @@ export const NewsItems: NewsItem[] = [ ] }*/ { - versionCode: '1.16.1', + versionCode: '1.16.2', versionName: 'Crystal Clear', - versionDate: new Date('2024-05-13T19:00:00Z'), + versionDate: new Date('2024-06-07T05:00:00Z'), + // versionDate: new Date('2024-05-13T19:00:00Z'), // versionDate: new Date('2024-05-09T00:00:00Z'), versionCoverImage: coverV116, items: [ @@ -77,6 +78,7 @@ export const NewsItems: NewsItem[] = [ { text: <>Updated Anthropic*, Groq, Ollama, OpenAI*, OpenRouter*, and Perplexity }, { text: <>Developers: update LLMs data structures, dev: true }, { text: <>1.16.1: Support for OpenAI GPT-4o (refresh your OpenAI models) }, + { text: <>1.16.2: Proper Gemini support, HTML/Markdown downloads, and latest Mistral }, ], }, { diff --git a/src/apps/news/news.version.tsx b/src/apps/news/news.version.tsx index 29d6f97e3..a4915d342 100644 --- a/src/apps/news/news.version.tsx +++ b/src/apps/news/news.version.tsx @@ -7,7 +7,7 @@ import { useAppStateStore } from '~/common/state/store-appstate'; // update this variable every time you want to broadcast a new version to clients -export const incrementalNewsVersion: number = 16.1; +export const incrementalNewsVersion: number = 16.1; // not notifying for 16.2 interface NewsState {