diff --git a/e2e/sample-apps/modular.js b/e2e/sample-apps/modular.js index 9e943e04494..292a11535a3 100644 --- a/e2e/sample-apps/modular.js +++ b/e2e/sample-apps/modular.js @@ -58,7 +58,12 @@ import { onValue, off } from 'firebase/database'; -import { getGenerativeModel, getVertexAI, VertexAI } from 'firebase/vertexai'; +import { + getGenerativeModel, + getVertexAI, + InferenceMode, + VertexAI +} from 'firebase/vertexai'; import { getDataConnect, DataConnect } from 'firebase/data-connect'; /** @@ -332,6 +337,17 @@ function callDataConnect(app) { console.log('[DATACONNECT] initialized'); } +async function callVertex(app) { + console.log('[VERTEX] start'); + const vertex = getVertexAI(app); + const model = getGenerativeModel(vertex, { + mode: InferenceMode.PREFER_ON_DEVICE + }); + const result = await model.generateContent("What is Roko's Basalisk?"); + console.log(result.response.text()); + console.log('[VERTEX] initialized'); +} + /** * Run smoke tests for all products. * Comment out any products you want to ignore. @@ -353,6 +369,7 @@ async function main() { await callVertexAI(app); callDataConnect(app); await authLogout(app); + await callVertex(app); console.log('DONE'); } diff --git a/packages/util/src/environment.ts b/packages/util/src/environment.ts index a0467b08c59..50d5f534106 100644 --- a/packages/util/src/environment.ts +++ b/packages/util/src/environment.ts @@ -173,6 +173,12 @@ export function isSafari(): boolean { ); } +export function isChrome(): boolean { + return ( + !isNode() && !!navigator.userAgent && navigator.userAgent.includes('Chrome') + ); +} + /** * This method checks if indexedDB is supported by current browser/service worker context * @return true if indexedDB is supported by current browser/service worker context diff --git a/packages/vertexai/src/methods/chrome-adapter.test.ts b/packages/vertexai/src/methods/chrome-adapter.test.ts new file mode 100644 index 00000000000..c350bd366c0 --- /dev/null +++ b/packages/vertexai/src/methods/chrome-adapter.test.ts @@ -0,0 +1,301 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, use } from 'chai'; +import sinonChai from 'sinon-chai'; +import chaiAsPromised from 'chai-as-promised'; +import { ChromeAdapter } from './chrome-adapter'; +import { stub } from 'sinon'; +import * as util from '@firebase/util'; + +use(sinonChai); +use(chaiAsPromised); + +describe('ChromeAdapter', () => { + describe('isAvailable', () => { + it('returns false if mode is only cloud', async () => { + const adapter = new ChromeAdapter(undefined, 'only_in_cloud'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if browser is not Chrome', async () => { + const chromeStub = stub(util, 'isChrome').returns(false); + const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + chromeStub.restore(); + }); + it('returns false if AI API is undefined', async () => { + const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if LanguageModel API is undefined', async () => { + const adapter = new ChromeAdapter({} as AI, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request contents empty', async () => { + const adapter = new ChromeAdapter({} as AI, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request content has function role', async () => { + const adapter = new ChromeAdapter({} as AI, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'function', + parts: [] + } + ] + }) + ).to.be.false; + }); + it('returns false if request content has multiple parts', async () => { + const adapter = new ChromeAdapter({} as AI, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'a' }, { text: 'b' }] + } + ] + }) + ).to.be.false; + }); + it('returns false if request content has non-text part', async () => { + const adapter = new ChromeAdapter({} as AI, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ inlineData: { mimeType: 'a', data: 'b' } }] + } + ] + }) + ).to.be.false; + }); + it('returns true if model is readily available', async () => { + const aiProvider = { + languageModel: { + capabilities: () => + Promise.resolve({ + available: 'readily' + }) + } + } as AI; + const adapter = new ChromeAdapter(aiProvider, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.true; + }); + it('returns false and triggers download when model is available after download', async () => { + const aiProvider = { + languageModel: { + capabilities: () => + Promise.resolve({ + available: 'after-download' + }), + create: () => Promise.resolve({}) + } + } as AI; + const createStub = stub(aiProvider.languageModel, 'create').resolves( + {} as AILanguageModel + ); + const onDeviceParams = {} as AILanguageModelCreateOptionsWithSystemPrompt; + const adapter = new ChromeAdapter( + aiProvider, + 'prefer_on_device', + onDeviceParams + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + expect(createStub).to.have.been.calledOnceWith(onDeviceParams); + }); + it('avoids redundant downloads', async () => { + const aiProvider = { + languageModel: { + capabilities: () => + Promise.resolve({ + available: 'after-download' + }), + create: () => {} + } + } as AI; + const downloadPromise = new Promise(() => { + /* never resolves */ + }); + const createStub = stub(aiProvider.languageModel, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(aiProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledOnce; + }); + it('clears state when download completes', async () => { + const aiProvider = { + languageModel: { + capabilities: () => + Promise.resolve({ + available: 'after-download' + }), + create: () => {} + } + } as AI; + let resolveDownload; + const downloadPromise = new Promise(resolveCallback => { + resolveDownload = resolveCallback; + }); + const createStub = stub(aiProvider.languageModel, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(aiProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + resolveDownload!(); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledTwice; + }); + it('returns false when model is never available', async () => { + const aiProvider = { + languageModel: { + capabilities: () => + Promise.resolve({ + available: 'no' + }) + } + } as AI; + const adapter = new ChromeAdapter(aiProvider, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + }); + }); + describe('generateContentOnDevice', () => { + it('Extracts and concats initial prompts', async () => { + const aiProvider = { + languageModel: { + create: () => Promise.resolve({}) + } + } as AI; + const factoryStub = stub(aiProvider.languageModel, 'create').resolves({ + prompt: s => Promise.resolve(s) + } as AILanguageModel); + const text = ['first', 'second', 'third']; + const onDeviceParams = { + initialPrompts: [{ role: 'user', content: text[0] }] + } as AILanguageModelCreateOptionsWithSystemPrompt; + const adapter = new ChromeAdapter( + aiProvider, + 'prefer_on_device', + onDeviceParams + ); + const response = await adapter.generateContentOnDevice({ + contents: [ + { role: 'model', parts: [{ text: text[1] }] }, + { role: 'user', parts: [{ text: text[2] }] } + ] + }); + expect(factoryStub).to.have.been.calledOnceWith({ + initialPrompts: [ + { role: 'user', content: text[0] }, + // Asserts tail is passed as initial prompts, and + // role is normalized from model to assistant. + { role: 'assistant', content: text[1] } + ] + }); + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: text[2] }] + } + } + ] + }); + }); + it('Extracts system prompt', async () => { + const aiProvider = { + languageModel: { + create: () => Promise.resolve({}) + } + } as AI; + const factoryStub = stub(aiProvider.languageModel, 'create').resolves({ + prompt: s => Promise.resolve(s) + } as AILanguageModel); + const onDeviceParams = { + systemPrompt: 'be yourself' + } as AILanguageModelCreateOptionsWithSystemPrompt; + const adapter = new ChromeAdapter( + aiProvider, + 'prefer_on_device', + onDeviceParams + ); + const text = 'hi'; + const response = await adapter.generateContentOnDevice({ + contents: [{ role: 'user', parts: [{ text }] }] + }); + expect(factoryStub).to.have.been.calledOnceWith({ + initialPrompts: [], + systemPrompt: onDeviceParams.systemPrompt + }); + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text }] + } + } + ] + }); + }); + }); +}); diff --git a/packages/vertexai/src/methods/chrome-adapter.ts b/packages/vertexai/src/methods/chrome-adapter.ts index f66e02f6711..ebf6b67dbb0 100644 --- a/packages/vertexai/src/methods/chrome-adapter.ts +++ b/packages/vertexai/src/methods/chrome-adapter.ts @@ -15,33 +15,146 @@ * limitations under the License. */ -import { - EnhancedGenerateContentResponse, - GenerateContentRequest, - InferenceMode -} from '../types'; +import { isChrome } from '@firebase/util'; +import { Content, GenerateContentRequest, InferenceMode, Role } from '../types'; /** * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. */ export class ChromeAdapter { + private isDownloading = false; + private downloadPromise: Promise | undefined; + private oldSession: AILanguageModel | undefined; constructor( private aiProvider?: AI, + // TODO: mode can be required now. private mode?: InferenceMode, private onDeviceParams?: AILanguageModelCreateOptionsWithSystemPrompt ) {} - // eslint-disable-next-line @typescript-eslint/no-unused-vars + /** + * Convenience method to check if a given request can be made on-device. + * Encapsulates a few concerns: 1) the mode, 2) API existence, 3) prompt formatting, and + * 4) model availability, including triggering download if necessary. + * Pros: caller needn't be concerned with details of on-device availability. Cons: this method + * spans a few concerns and splits request validation from usage. If instance variables weren't + * already part of the API, we could consider a better separation of concerns. + */ async isAvailable(request: GenerateContentRequest): Promise { - return false; + // Returns false if we should only use in-cloud inference. + if (this.mode === 'only_in_cloud') { + return false; + } + // Returns false if the on-device inference API is undefined. + const isLanguageModelAvailable = + isChrome() && this.aiProvider && this.aiProvider.languageModel; + if (!isLanguageModelAvailable) { + return false; + } + // Returns false if the request can't be run on-device. + if (!ChromeAdapter.isOnDeviceRequest(request)) { + return false; + } + switch (await this.availability()) { + case 'readily': + // Returns true only if a model is immediately available. + return true; + case 'after-download': + // Triggers async model download. + this.download(); + case 'no': + default: + return false; + } } async generateContentOnDevice( - // eslint-disable-next-line @typescript-eslint/no-unused-vars request: GenerateContentRequest - ): Promise { + ): Promise { + const createOptions = this.onDeviceParams || {}; + createOptions.initialPrompts ??= []; + const extractedInitialPrompts = ChromeAdapter.toInitialPrompts( + request.contents + ); + // Assumes validation asserted there is at least one initial prompt. + const prompt = extractedInitialPrompts.pop()!; + createOptions.initialPrompts.push(...extractedInitialPrompts); + const session = await this.session(createOptions); + const result = await session.prompt(prompt.content); + return ChromeAdapter.toResponse(result); + } + private static toResponse(text: string): Response { return { - text: () => '', - functionCalls: () => undefined - }; + json: async () => ({ + candidates: [ + { + content: { + parts: [{ text }] + } + } + ] + }) + } as Response; + } + private static isOnDeviceRequest(request: GenerateContentRequest): boolean { + // Returns false if the prompt is empty. + if (request.contents.length === 0) { + return false; + } + + // Applies the same checks as above, but for each content item. + for (const content of request.contents) { + if (content.role === 'function') { + return false; + } + + if (content.parts.length > 1) { + return false; + } + + if (!content.parts[0].text) { + return false; + } + } + + return true; + } + private async availability(): Promise { + return this.aiProvider?.languageModel + .capabilities() + .then((c: AILanguageModelCapabilities) => c.available); + } + private download(): void { + if (this.isDownloading) { + return; + } + this.isDownloading = true; + this.downloadPromise = this.aiProvider?.languageModel + .create(this.onDeviceParams) + .then(() => { + this.isDownloading = false; + }); + } + private static toOnDeviceRole(role: Role): AILanguageModelPromptRole { + return role === 'model' ? 'assistant' : 'user'; + } + private static toInitialPrompts( + contents: Content[] + ): AILanguageModelPrompt[] { + return contents.map(c => ({ + role: ChromeAdapter.toOnDeviceRole(c.role), + // Assumes contents have been verified to contain only a single TextPart. + content: c.parts[0].text! + })); + } + private async session( + options: AILanguageModelCreateOptionsWithSystemPrompt + ): Promise { + const newSession = await this.aiProvider!.languageModel.create(options); + if (this.oldSession) { + this.oldSession.destroy(); + } + // Holds session reference, so model isn't unloaded from memory. + this.oldSession = newSession; + return newSession; } } diff --git a/packages/vertexai/src/methods/generate-content.test.ts b/packages/vertexai/src/methods/generate-content.test.ts index ec825421709..3a6cffb9f9d 100644 --- a/packages/vertexai/src/methods/generate-content.test.ts +++ b/packages/vertexai/src/methods/generate-content.test.ts @@ -291,24 +291,23 @@ describe('generateContent()', () => { expect(mockFetch).to.be.called; }); it('on-device', async () => { - const expectedText = 'hi'; const chromeAdapter = new ChromeAdapter(); const mockIsAvailable = stub(chromeAdapter, 'isAvailable').resolves(true); - const mockGenerateContent = stub( + const mockResponse = getMockResponse( + 'unary-success-basic-reply-short.json' + ); + const makeRequestStub = stub( chromeAdapter, 'generateContentOnDevice' - ).resolves({ - text: () => expectedText, - functionCalls: () => undefined - }); + ).resolves(mockResponse as Response); const result = await generateContent( fakeApiSettings, 'model', fakeRequestParams, chromeAdapter ); - expect(result.response.text()).to.equal(expectedText); + expect(result.response.text()).to.include('Mountain View, California'); expect(mockIsAvailable).to.be.called; - expect(mockGenerateContent).to.be.calledWith(fakeRequestParams); + expect(makeRequestStub).to.be.calledWith(fakeRequestParams); }); }); diff --git a/packages/vertexai/src/methods/generate-content.ts b/packages/vertexai/src/methods/generate-content.ts index 63745c47fae..ba7a162aa9c 100644 --- a/packages/vertexai/src/methods/generate-content.ts +++ b/packages/vertexai/src/methods/generate-content.ts @@ -16,7 +16,6 @@ */ import { - EnhancedGenerateContentResponse, GenerateContentRequest, GenerateContentResponse, GenerateContentResult, @@ -51,8 +50,8 @@ async function generateContentOnCloud( model: string, params: GenerateContentRequest, requestOptions?: RequestOptions -): Promise { - const response = await makeRequest( +): Promise { + return makeRequest( model, Task.GENERATE_CONTENT, apiSettings, @@ -60,9 +59,6 @@ async function generateContentOnCloud( JSON.stringify(params), requestOptions ); - const responseJson: GenerateContentResponse = await response.json(); - const enhancedResponse = createEnhancedContentResponse(responseJson); - return enhancedResponse; } export async function generateContent( @@ -72,17 +68,19 @@ export async function generateContent( chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions ): Promise { - let enhancedResponse; + let response; if (await chromeAdapter.isAvailable(params)) { - enhancedResponse = await chromeAdapter.generateContentOnDevice(params); + response = await chromeAdapter.generateContentOnDevice(params); } else { - enhancedResponse = await generateContentOnCloud( + response = await generateContentOnCloud( apiSettings, model, params, requestOptions ); } + const responseJson: GenerateContentResponse = await response.json(); + const enhancedResponse = createEnhancedContentResponse(responseJson); return { response: enhancedResponse };