From 6d6e521b56f4074246896481267bfb3d8b334163 Mon Sep 17 00:00:00 2001 From: sharon wang Date: Mon, 17 Nov 2025 11:21:02 -0500 Subject: [PATCH 1/6] Assistant: add model configuration setting - add positron.assistant.configuredModels setting to allow users to statically configure model settings per provider, which overrides dynamic model resolution - update and add tests around model resolution --- extensions/positron-assistant/package.json | 56 +++ .../positron-assistant/package.nls.json | 3 +- .../positron-assistant/src/anthropic.ts | 164 ++++--- .../positron-assistant/src/constants.ts | 3 + .../src/modelDefinitions.ts | 80 ++++ .../src/modelResolutionHelpers.ts | 230 ++++++++++ extensions/positron-assistant/src/models.ts | 427 +++++++++--------- extensions/positron-assistant/src/posit.ts | 101 +++-- .../src/test/anthropic.test.ts | 230 ++++++++++ .../src/test/modelDefinitions.test.ts | 54 +++ .../src/test/modelResolutionHelpers.test.ts | 415 +++++++++++++++++ .../src/test/models.test.ts | 104 ----- .../src/test/openai.test.ts | 319 +++++++++++++ .../contrib/chat/common/languageModels.ts | 55 ++- 14 files changed, 1763 insertions(+), 478 deletions(-) create mode 100644 extensions/positron-assistant/src/modelDefinitions.ts create mode 100644 extensions/positron-assistant/src/modelResolutionHelpers.ts create mode 100644 extensions/positron-assistant/src/test/modelDefinitions.test.ts create mode 100644 extensions/positron-assistant/src/test/modelResolutionHelpers.test.ts delete mode 100644 extensions/positron-assistant/src/test/models.test.ts create mode 100644 extensions/positron-assistant/src/test/openai.test.ts diff --git a/extensions/positron-assistant/package.json b/extensions/positron-assistant/package.json index bb02931d3ee3..2fa99f83188e 100644 --- a/extensions/positron-assistant/package.json +++ b/extensions/positron-assistant/package.json @@ -389,6 +389,62 @@ "experimental" ] }, + "positron.assistant.configuredModels": { + "type": "object", + "default": {}, + "markdownDescription": "%configuration.configuredModels.description%", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Display name for the model" + }, + "identifier": { + "type": "string", + "description": "Model identifier for API calls" + }, + "maxInputTokens": { + "type": "number", + "minimum": 1, + "description": "Maximum input tokens for this model" + }, + "maxOutputTokens": { + "type": "number", + "minimum": 1, + "description": "Maximum output tokens for this model" + } + }, + "required": [ + "name", + "identifier" + ] + } + }, + "examples": [ + { + "openai-compatible": [ + { + "name": "Claude Sonnet 4.5", + "identifier": "claude-sonnet-4-5", + "maxInputTokens": 200000, + "maxOutputTokens": 64000 + }, + { + "name": "GPT-4 Turbo", + "identifier": "gpt-4-turbo", + "maxInputTokens": 8192, + "maxOutputTokens": 2048 + } + ] + } + ], + "tags": [ + "experimental" + ] + }, "positron.assistant.notebookSuggestions.model": { "type": "array", "default": [ diff --git a/extensions/positron-assistant/package.nls.json b/extensions/positron-assistant/package.nls.json index f69a0ec6da7d..740b99989d8b 100644 --- a/extensions/positron-assistant/package.nls.json +++ b/extensions/positron-assistant/package.nls.json @@ -46,5 +46,6 @@ "configuration.preferredModel.description": "A preferred model ID or name (partial match supported) to use if available in Positron Assistant for the current provider.\n\nTakes precedence over `#positron.assistant.defaultModels#`.\n\nRequires a restart to take effect.\n\nExamples:\n- `Claude Sonnet 4.5` prefers the model named 'Claude Sonnet 4.5'\n- `GPT-5` prefers the model named 'GPT-5'", "configuration.defaultModels.description": "A mapping of provider IDs to default model IDs or names (partial match supported) to use for that provider in Positron Assistant.\n\n`#positron.assistant.preferredModel#` takes precedence over this setting.\n\nRequires a restart to take effect.\n\nExample: Item `anthropic-api` and Value `Claude Sonnet 4.5` sets the default model for Anthropic to 'Claude Sonnet 4.5'", "configuration.notebookSuggestions.model.description": "An ordered array of model patterns to try when generating AI suggestions for Positron Notebooks. Patterns are tried in order until a match is found.\n\nEach pattern supports partial matching on model ID or name (case-insensitive). Default is `[\"haiku\", \"mini\"]` which tries to find a model with \"haiku\" in the name first, then tries \"mini\" if not found. If no patterns match or the array is empty, falls back to the current chat session model, then the current provider's model, then the first available model.\n\nExamples:\n- `[\"haiku\", \"mini\"]` (default) tries to find a model with \"haiku\" in the name first, then tries \"mini\" if not found\n- `[\"Claude Sonnet 4.5\", \"GPT-5\"]` tries \"Claude Sonnet 4.5\" first, then \"GPT-5\"\n- `[]` disables pattern matching and uses the default fallback behavior", - "configuration.providerVariables.bedrock.description": "Variables used to configure advanced settings for Bedrock in Positron Assistant.\n\nRequires a restart to take effect.\n\nExample: to set the AWS region and profile for Amazon Bedrock, add items with keys `AWS_REGION` and `AWS_PROFILE`." + "configuration.providerVariables.bedrock.description": "Variables used to configure advanced settings for Bedrock in Positron Assistant.\n\nRequires a restart to take effect.\n\nExample: to set the AWS region and profile for Amazon Bedrock, add items with keys `AWS_REGION` and `AWS_PROFILE`.", + "configuration.configuredModels.description": "Configure custom language models for each provider. These models will be used instead of retrieving the model listing from the provider.\n\nUse provider IDs as keys, with arrays of model configurations as values.\n\nExample:\n```json\n{\n \"anthropic-api\": [\n {\n \"name\": \"Claude Sonnet 4.5\", // required: display name\n \"identifier\": \"claude-sonnet-4-5\", // required: API model ID\n \"maxInputTokens\": 200000, // optional: input token limit\n \"maxOutputTokens\": 8192 // optional: output token limit\n }\n ]\n}\n```" } diff --git a/extensions/positron-assistant/src/anthropic.ts b/extensions/positron-assistant/src/anthropic.ts index e08524c47813..86f7ff9c2071 100644 --- a/extensions/positron-assistant/src/anthropic.ts +++ b/extensions/positron-assistant/src/anthropic.ts @@ -11,9 +11,10 @@ import { isChatImagePart, isCacheBreakpointPart, parseCacheBreakpoint, processMe import { DEFAULT_MAX_TOKEN_INPUT, DEFAULT_MAX_TOKEN_OUTPUT } from './constants.js'; import { log, recordTokenUsage, recordRequestTokenUsage } from './extension.js'; import { TokenUsage } from './tokens.js'; -import { availableModels } from './models.js'; +import { getAllModelDefinitions } from './modelDefinitions.js'; import { LanguageModelDataPartMimeType } from './types.js'; import { applyModelFilters } from './modelFilters.js'; +import { createModelInfo, markDefaultModel } from './modelResolutionHelpers.js'; export const DEFAULT_ANTHROPIC_MODEL_NAME = 'Claude Sonnet 4'; export const DEFAULT_ANTHROPIC_MODEL_MATCH = 'claude-sonnet-4'; @@ -90,10 +91,10 @@ export class AnthropicLanguageModel implements positron.ai.LanguageModelChatProv } async provideLanguageModelChatInformation(_options: { silent: boolean }, token: vscode.CancellationToken): Promise { - log.trace(`[${this.providerName}] Preparing language model chat information...`); + log.debug(`[${this.providerName}] Preparing language model chat information...`); const models = await this.resolveModels(token) ?? []; - log.trace(`[${this.providerName}] Resolved ${models.length} models.`); + log.debug(`[${this.providerName}] Resolved ${models.length} models.`); return this.filterModels(models); } @@ -234,15 +235,74 @@ export class AnthropicLanguageModel implements positron.ai.LanguageModelChatProv return applyModelFilters(models, this.provider, this.providerName); } - private isDefaultUserModel(id: string, name?: string): boolean { - const config = vscode.workspace.getConfiguration('positron.assistant'); - const defaultModels = config.get>('defaultModels') || {}; - if ('anthropic-api' in defaultModels) { - if (id.includes(defaultModels['anthropic-api']) || name?.includes(defaultModels['anthropic-api'])) { - return true; + private retrieveModelsFromConfig(): vscode.LanguageModelChatInformation[] | undefined { + const configuredModels = getAllModelDefinitions(this.provider); + if (configuredModels.length === 0) { + return undefined; + } + + log.info(`[${this.providerName}] Using ${configuredModels.length} configured models.`); + + const modelListing = configuredModels.map((modelDef) => + createModelInfo({ + id: modelDef.identifier, + name: modelDef.name, + family: this.provider, + version: '', + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: modelDef.maxInputTokens, + defaultMaxOutput: modelDef.maxOutputTokens + }) + ); + + return markDefaultModel(modelListing, this.provider, DEFAULT_ANTHROPIC_MODEL_MATCH); + } + + private async retrieveModelsFromApi(): Promise { + try { + const modelListing: vscode.LanguageModelChatInformation[] = []; + const knownAnthropicModels = getAllModelDefinitions(this.provider); + let hasMore = true; + let nextPageToken: string | undefined; + + log.trace(`[${this.providerName}] Fetching models from Anthropic API...`); + + while (hasMore) { + const modelsPage = nextPageToken + ? await this._client.models.list({ after_id: nextPageToken }) + : await this._client.models.list(); + + modelsPage.data.forEach(model => { + const knownModel = knownAnthropicModels?.find(m => model.id.startsWith(m.identifier)); + + modelListing.push( + createModelInfo({ + id: model.id, + name: model.display_name, + family: this.provider, + version: model.created_at, + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: knownModel?.maxInputTokens, + defaultMaxOutput: knownModel?.maxOutputTokens + }) + ); + }); + + hasMore = modelsPage.has_more; + if (hasMore && modelsPage.data.length > 0) { + nextPageToken = modelsPage.data[modelsPage.data.length - 1].id; + } } + + return markDefaultModel(modelListing, this.provider, DEFAULT_ANTHROPIC_MODEL_MATCH); + } catch (error) { + log.warn(`[${this.providerName}] Failed to fetch models from Anthropic API: ${error}`); + return undefined; } - return id.includes(DEFAULT_ANTHROPIC_MODEL_MATCH); } private onContentBlock(block: Anthropic.ContentBlock, progress: vscode.Progress): void { @@ -301,85 +361,21 @@ export class AnthropicLanguageModel implements positron.ai.LanguageModelChatProv } async resolveModels(token: vscode.CancellationToken): Promise { - const modelListing: vscode.LanguageModelChatInformation[] = []; - const knownAnthropicModels = availableModels.get(this.provider); - const userSetMaxInputTokens: Record = vscode.workspace.getConfiguration('positron.assistant').get('maxInputTokens', {}); - const userSetMaxOutputTokens: Record = vscode.workspace.getConfiguration('positron.assistant').get('maxOutputTokens', {}); - let hasMore = true; - let nextPageToken: string | undefined; - - log.trace(`[${this.providerName}] Fetching models from Anthropic API...`); - - while (hasMore) { - const modelsPage = nextPageToken - ? await this._client.models.list({ after_id: nextPageToken }) - : await this._client.models.list(); - - modelsPage.data.forEach(model => { - const knownModel = knownAnthropicModels?.find(m => model.id.startsWith(m.identifier)); - - const knownModelMaxInputTokens = knownModel?.maxInputTokens; - const maxInputTokens = userSetMaxInputTokens[model.id] ?? knownModelMaxInputTokens ?? DEFAULT_MAX_TOKEN_INPUT; - - const knownModelMaxOutputTokens = knownModel?.maxOutputTokens; - const maxOutputTokens = userSetMaxOutputTokens[model.id] ?? knownModelMaxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT; - - modelListing.push({ - id: model.id, - name: model.display_name, - family: this.provider, - version: model.created_at, - maxInputTokens: maxInputTokens, - maxOutputTokens: maxOutputTokens, - capabilities: this.capabilities, - isDefault: this.isDefaultUserModel(model.id, model.display_name), - isUserSelectable: true, - }); - }); + log.debug(`[${this.providerName}] Resolving models...`); - hasMore = modelsPage.has_more; - if (hasMore && modelsPage.data.length > 0) { - nextPageToken = modelsPage.data[modelsPage.data.length - 1].id; - } + const configuredModels = this.retrieveModelsFromConfig(); + if (configuredModels) { + this.modelListing = configuredModels; + return configuredModels; } - if (modelListing.length === 0) { - modelListing.push({ - id: this.id, - name: this.name, - family: this.provider, - version: this._context?.extension.packageJSON.version ?? '', - maxInputTokens: this.maxInputTokens, - maxOutputTokens: this.maxOutputTokens, - capabilities: this.capabilities, - isDefault: true, - isUserSelectable: true, - }); + const apiModels = await this.retrieveModelsFromApi(); + if (apiModels) { + this.modelListing = apiModels; + return apiModels; } - // Mark models as default, ensuring only one default per provider - let hasDefault = false; - for (let i = 0; i < modelListing.length; i++) { - const model = modelListing[i]; - if (!hasDefault && this.isDefaultUserModel(model.id, model.name)) { - modelListing[i] = { ...model, isDefault: true }; - hasDefault = true; - } else { - modelListing[i] = { ...model, isDefault: false }; - } - } - - // If no models match the default ID, make the first model the default. - if (modelListing.length > 0 && !hasDefault) { - modelListing[0] = { - ...modelListing[0], - isDefault: true, - }; - } - - this.modelListing = modelListing; - - return modelListing; + return undefined; } } diff --git a/extensions/positron-assistant/src/constants.ts b/extensions/positron-assistant/src/constants.ts index af5eb59c2081..70febc2be7ea 100644 --- a/extensions/positron-assistant/src/constants.ts +++ b/extensions/positron-assistant/src/constants.ts @@ -22,6 +22,9 @@ export const DEFAULT_MAX_TOKEN_INPUT = 100_000; /** The default max token output if a model's maximum is unknown */ export const DEFAULT_MAX_TOKEN_OUTPUT = 4_096; +/** The minimum allowed token limit for input/output tokens */ +export const MIN_TOKEN_LIMIT = 512; + /** Tag used by tools to indicate a workspace must be open in order to use the tool */ export const TOOL_TAG_REQUIRES_WORKSPACE = 'requires-workspace'; diff --git a/extensions/positron-assistant/src/modelDefinitions.ts b/extensions/positron-assistant/src/modelDefinitions.ts new file mode 100644 index 000000000000..f6f9a2d187db --- /dev/null +++ b/extensions/positron-assistant/src/modelDefinitions.ts @@ -0,0 +1,80 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as vscode from 'vscode'; + +export interface ModelDefinition { + name: string; + identifier: string; + maxInputTokens?: number; + maxOutputTokens?: number; +} + +/** + * Get user-configured models from VS Code settings for a specific provider. + */ +export function getConfiguredModels(providerId: string): ModelDefinition[] { + const config = vscode.workspace.getConfiguration('positron.assistant'); + const configuredModels = config.get>('configuredModels', {}); + return configuredModels[providerId] || []; +} + +/** + * Built-in model definitions that serve as fallback defaults when no user configuration + * is provided and dynamic model querying is not available or fails. + */ +const builtInModelDefinitions = new Map([ + ['posit-ai', [ + { + name: 'Claude Sonnet 4.5', + identifier: 'claude-sonnet-4-5', + maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + maxOutputTokens: 64_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + }, + { + name: 'Claude Opus 4.1', + identifier: 'claude-opus-4-1', + maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + maxOutputTokens: 32_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + }, + { + name: 'Claude Haiku 4.5', + identifier: 'claude-haiku-4-5', + maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + maxOutputTokens: 64_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table + }, + ]], + ['google', [ + { + name: 'Gemini 2.5 Flash', + identifier: 'gemini-2.5-pro-exp-03-25', + maxOutputTokens: 65_536, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview + }, + { + name: 'Gemini 2.0 Flash', + identifier: 'gemini-2.0-flash-exp', + maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-2.0-flash + }, + { + name: 'Gemini 1.5 Flash 002', + identifier: 'gemini-1.5-flash-002', + maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-1.5-flash + }, + ]] +]); + +/** + * Get all available model definitions for a provider, with intelligent fallback hierarchy: + * 1. User-configured models (from settings) - highest priority + * 2. Built-in model definitions - fallback when no user config + */ +export function getAllModelDefinitions(providerId: string): ModelDefinition[] { + const configured = getConfiguredModels(providerId); + if (configured.length > 0) { + return configured; + } + return builtInModelDefinitions.get(providerId) || []; +} + diff --git a/extensions/positron-assistant/src/modelResolutionHelpers.ts b/extensions/positron-assistant/src/modelResolutionHelpers.ts new file mode 100644 index 000000000000..fab3a000d0fc --- /dev/null +++ b/extensions/positron-assistant/src/modelResolutionHelpers.ts @@ -0,0 +1,230 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as vscode from 'vscode'; +import { getAllModelDefinitions } from './modelDefinitions.js'; +import { DEFAULT_MAX_TOKEN_INPUT, DEFAULT_MAX_TOKEN_OUTPUT, MIN_TOKEN_LIMIT } from './constants.js'; +import { log } from './extension.js'; + +/** + * Type definition for token limits configuration from user settings. + */ +export interface TokenLimits { + maxInput: Record; + maxOutput: Record; +} + +/** + * Retrieves user-configured token limits from workspace settings. + * These settings allow users to override default token limits for specific models. + * + * @returns Object containing maxInput and maxOutput token limits by model ID + */ +export function getUserTokenLimits(): TokenLimits { + const config = vscode.workspace.getConfiguration('positron.assistant'); + return { + maxInput: config.get('maxInputTokens', {}), + maxOutput: config.get('maxOutputTokens', {}) + }; +} + +/** + * Determines if a model should be marked as the default for a given provider. + * + * This function checks: + * 1. User-configured default models for the provider + * 2. Falls back to provider-specific default patterns + * + * @param provider The provider ID (e.g., 'anthropic-api', 'openai-compatible) + * @param id The model ID to check + * @param name Optional model display name to check against + * @param defaultMatch Optional fallback pattern to match against (provider-specific) + * @returns true if this model should be the default + */ +export function isDefaultUserModel( + provider: string, + id: string, + name?: string, + defaultMatch?: string +): boolean { + const config = vscode.workspace.getConfiguration('positron.assistant'); + const defaultModels = config.get>('defaultModels') || {}; + + // Check user-configured default for this provider + if (provider in defaultModels) { + const userDefault = defaultModels[provider]; + if (id.includes(userDefault) || name?.includes(userDefault)) { + return true; + } + } + + // Fall back to provider-specific default pattern if provided + if (defaultMatch) { + return id.includes(defaultMatch); + } + + return false; +} + +/** + * Resolves the maximum token count for a model with proper fallback hierarchy. + * + * Priority order: + * 1. User override from workspace settings (maxInputTokens/maxOutputTokens) + * 2. Model definition limits from getAllModelDefinitions() + * 3. Provider-specific defaults + * 4. Global defaults + * + * Includes validation to ensure minimum token limit with helpful warnings. + * + * @param id The model ID to resolve tokens for + * @param type Whether to resolve 'input' or 'output' tokens + * @param provider The provider ID (e.g., 'anthropic-api', 'posit-ai') + * @param providerDefault Optional provider-specific default (overrides global default) + * @param providerName Optional provider display name for logging + * @returns The resolved token limit + */ +export function getMaxTokens( + id: string, + type: 'input' | 'output', + provider: string, + providerDefault?: number, + providerName?: string +): number { + const globalDefault = type === 'input' ? DEFAULT_MAX_TOKEN_INPUT : DEFAULT_MAX_TOKEN_OUTPUT; + const defaultTokens = providerDefault ?? globalDefault; + + // Get model-specific limits from definitions + const configuredModels = getAllModelDefinitions(provider); + const fixedValue = type === 'input' + ? configuredModels?.find(m => m.identifier === id)?.maxInputTokens + : configuredModels?.find(m => m.identifier === id)?.maxOutputTokens; + let maxTokens = fixedValue ?? defaultTokens; + + // Apply user overrides from workspace settings + const configKey = type === 'input' ? 'maxInputTokens' : 'maxOutputTokens'; + const tokensConfig: Record = vscode.workspace.getConfiguration('positron.assistant').get(configKey, {}); + for (const [key, value] of Object.entries(tokensConfig)) { + if (id.indexOf(key) !== -1 && value) { + if (typeof value !== 'number') { + log.warn(`[${providerName ?? provider}] Invalid ${configKey} '${value}' for ${key} (${id}); ignoring`); + continue; + } + if (value < MIN_TOKEN_LIMIT) { + log.warn(`[${providerName ?? provider}] Specified ${configKey} '${value}' for ${key} (${id}) is too low; using ${MIN_TOKEN_LIMIT} instead`); + maxTokens = MIN_TOKEN_LIMIT; + } else { + maxTokens = value; + } + break; + } + } + + log.trace(`[${providerName ?? provider}] Setting ${configKey} for (${id}) to ${maxTokens}`); + return maxTokens; +} + +/** + * Parameters for creating a language model chat information object. + */ +export interface CreateModelInfoParams { + /** The unique model ID */ + id: string; + /** The display name of the model */ + name: string; + /** The provider/family of the model */ + family: string; + /** The version identifier of the model */ + version: string; + /** The provider ID for token resolution */ + provider: string; + /** The provider display name for logging */ + providerName: string; + /** Model capabilities */ + capabilities?: vscode.LanguageModelChatInformation['capabilities']; + /** Optional default max input tokens (overrides model definition defaults) */ + defaultMaxInput?: number; + /** Optional default max output tokens (overrides model definition defaults) */ + defaultMaxOutput?: number; +} + +/** + * Creates a standardized LanguageModelChatInformation object. + * + * This shared utility ensures consistent model information creation across all providers + * while applying the proper token resolution. Default model selection is handled separately + * by the markDefaultModel function to avoid duplication. + * + * @param params The parameters for creating the model information + * @returns A complete LanguageModelChatInformation object with isDefault set to false + */ +export function createModelInfo(params: CreateModelInfoParams): vscode.LanguageModelChatInformation { + const { + id, + name, + family, + version, + provider, + providerName, + capabilities = { vision: true, toolCalling: true, agentMode: true }, + defaultMaxInput, + defaultMaxOutput + } = params; + + return { + id, + name, + family, + version, + maxInputTokens: getMaxTokens(id, 'input', provider, defaultMaxInput, providerName), + maxOutputTokens: getMaxTokens(id, 'output', provider, defaultMaxOutput, providerName), + capabilities, + isDefault: false, + isUserSelectable: true, + }; +} + +/** + * Marks models as default, ensuring only one default per provider. + * + * This utility function standardizes default model selection across all providers. + * It uses the isDefaultUserModel logic to determine which model should be default, + * and ensures exactly one model is marked as default per provider. + * + * @param models Array of models to process + * @param provider The provider ID (used for default model detection) + * @param defaultMatch Optional fallback pattern to match against for default selection + * @returns Array of models with exactly one marked as default + */ +export function markDefaultModel( + models: vscode.LanguageModelChatInformation[], + provider: string, + defaultMatch?: string +): vscode.LanguageModelChatInformation[] { + if (models.length === 0) { + return models; + } + + // Mark models as default, ensuring only one default per provider + let hasDefault = false; + const updatedModels = models.map((model) => { + if (!hasDefault && isDefaultUserModel(provider, model.id, model.name, defaultMatch)) { + hasDefault = true; + return { ...model, isDefault: true }; + } else { + return { ...model, isDefault: false }; + } + }); + + // If no models match the default criteria, make the first model the default + if (updatedModels.length > 0 && !hasDefault) { + updatedModels[0] = { + ...updatedModels[0], + isDefault: true, + }; + } + + return updatedModels; +} diff --git a/extensions/positron-assistant/src/models.ts b/extensions/positron-assistant/src/models.ts index 048327f7c2da..8b7e58cec4d8 100644 --- a/extensions/positron-assistant/src/models.ts +++ b/extensions/positron-assistant/src/models.ts @@ -26,6 +26,8 @@ import { BedrockClient, FoundationModelSummary, InferenceProfileSummary, ListFou import { PositLanguageModel } from './posit.js'; import { applyModelFilters } from './modelFilters'; import { autoconfigureWithManagedCredentials, AWS_MANAGED_CREDENTIALS } from './pwb'; +import { getAllModelDefinitions } from './modelDefinitions'; +import { createModelInfo, getMaxTokens, markDefaultModel } from './modelResolutionHelpers.js'; /** * Models used by chat participants and for vscode.lm.* API functionality. @@ -132,6 +134,8 @@ class EchoLanguageModel implements positron.ai.LanguageModelChatProvider { async provideLanguageModelChatInformation(options: { silent: boolean }, token: vscode.CancellationToken): Promise { log.debug(`[${this.providerName}] Preparing language model chat information...`); const models = this.modelListing ?? await this.resolveModels(token) ?? []; + + log.debug(`[${this.providerName}] Resolved ${models.length} models.`); return this.filterModels(models); } @@ -303,41 +307,7 @@ abstract class AILanguageModel implements positron.ai.LanguageModelChatProvider this.provider = _config.provider; } - get providerName(): string { - return this.providerName; - } - - protected getMaxTokens(id: string, type: 'input' | 'output'): number { - const defaultTokens = type === 'input' - ? (this._config.maxInputTokens ?? DEFAULT_MAX_TOKEN_INPUT) - : (this._config.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT); - - const fixedModels = availableModels.get(this._config.provider); - const fixedValue = type === 'input' - ? fixedModels?.find(m => m.identifier === id)?.maxInputTokens - : fixedModels?.find(m => m.identifier === id)?.maxOutputTokens; - let maxTokens = fixedValue ?? defaultTokens; - - const configKey = type === 'input' ? 'maxInputTokens' : 'maxOutputTokens'; - const tokensConfig: Record = vscode.workspace.getConfiguration('positron.assistant').get(configKey, {}); - for (const [key, value] of Object.entries(tokensConfig)) { - if (id.indexOf(key) !== -1 && value) { - if (typeof value !== 'number') { - log.warn(`[${this.providerName}] Invalid ${configKey} '${value}' for ${key} (${id}); ignoring`); - continue; - } - if (value < 512) { - log.warn(`[${this.providerName}] Specified ${configKey} '${value}' for ${key} (${id}) is too low; using 512 instead`); - maxTokens = 512; - } - maxTokens = value; - break; - } - } - - log.trace(`[${this.providerName}] Setting ${configKey} for (${id}) to ${maxTokens}`); - return maxTokens; - } + abstract get providerName(): string; protected filterModels(models: vscode.LanguageModelChatInformation[]): vscode.LanguageModelChatInformation[] { return applyModelFilters(models, this.provider, this.providerName); @@ -499,7 +469,7 @@ abstract class AILanguageModel implements positron.ai.LanguageModelChatProvider maxSteps: modelOptions.maxSteps ?? 50, tools: modelTools, abortSignal: signal, - maxTokens: this.getMaxTokens(aiModel.modelId, 'output'), + maxTokens: getMaxTokens(aiModel.modelId, 'output', this._config.provider, this._config.maxOutputTokens, this.providerName), }); let accumulatedTextDeltas: string[] = []; @@ -540,7 +510,7 @@ abstract class AILanguageModel implements positron.ai.LanguageModelChatProvider log.warn(`${messagePrefix} RECV error: ${JSON.stringify(part.error, null, 2)}`); const errorMsg = this.parseProviderError(part.error) || (typeof part.error === 'string' ? part.error : JSON.stringify(part.error, null, 2)); - return new Error(`${messagePrefix} Error sending test message: ${errorMsg}`); + throw new Error(`${messagePrefix} Error in chat response: ${errorMsg}`); } } @@ -631,58 +601,58 @@ abstract class AILanguageModel implements positron.ai.LanguageModelChatProvider async resolveModels(token: vscode.CancellationToken): Promise { log.debug(`[${this.providerName}] Resolving models...`); - const availableModelsForProvider = availableModels.get(this._config.provider); - if (!availableModelsForProvider || availableModelsForProvider.length === 0) { - log.info(`[${this.providerName}] No models available; returning default model information.`); - const aiModel = this.aiProvider(this._config.model, this.aiOptions); - return [ - { - id: aiModel.modelId, - name: this.name, - family: aiModel.provider, - version: aiModel.specificationVersion, - maxInputTokens: this.getMaxTokens(aiModel.modelId, 'input'), - maxOutputTokens: this.getMaxTokens(aiModel.modelId, 'output'), - capabilities: this.capabilities, - isDefault: true, - isUserSelectable: true, - } satisfies vscode.LanguageModelChatInformation - ]; + const configuredModels = this.retrieveModelsFromConfig(); + if (configuredModels) { + this.modelListing = configuredModels; + return configuredModels; } - const models: vscode.LanguageModelChatInformation[] = availableModelsForProvider.map(model => ({ - id: model.identifier, - name: model.name, - family: this.provider, - version: this.aiProvider(model.identifier).specificationVersion, - maxInputTokens: 0, - maxOutputTokens: model.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT, - capabilities: this.capabilities, - isDefault: this.isDefaultUserModel(model.identifier, model.name), - isUserSelectable: true, - } satisfies vscode.LanguageModelChatInformation)); - - // If no models match the default ID, make the first model the default. - if (models.length > 0 && !models.some(m => m.isDefault)) { - models[0] = { - ...models[0], - isDefault: true, - }; + // Fallback to default model if no configured models available + const defaultModel = this.createDefaultModel(); + this.modelListing = defaultModel; + return defaultModel; + } + + protected retrieveModelsFromConfig(): vscode.LanguageModelChatInformation[] | undefined { + const configuredModels = getAllModelDefinitions(this.provider); + if (configuredModels.length === 0) { + return undefined; } - this.modelListing = models; - return models; + log.info(`[${this.providerName}] Using ${configuredModels.length} configured models.`); + + const models: vscode.LanguageModelChatInformation[] = configuredModels.map(model => + createModelInfo({ + id: model.identifier, + name: model.name, + family: this.provider, + version: this.aiProvider(model.identifier).specificationVersion, + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: model.maxInputTokens ?? 0, + defaultMaxOutput: model.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT + }) + ); + + return markDefaultModel(models, this.provider, this._config.model); } - protected isDefaultUserModel(id: string, name?: string): boolean { - const config = vscode.workspace.getConfiguration('positron.assistant'); - const defaultModels = config.get>('defaultModels') || {}; - if (this.provider in defaultModels) { - if (id.includes(defaultModels[this.provider]) || name?.includes(defaultModels[this.provider])) { - return true; - } - } - return this._config.model === id; + protected createDefaultModel(): vscode.LanguageModelChatInformation[] { + log.info(`[${this.providerName}] No models available; returning default model information.`); + const aiModel = this.aiProvider(this._config.model, this.aiOptions); + const modelInfo = createModelInfo({ + id: aiModel.modelId, + name: this.name, + family: aiModel.provider, + version: aiModel.specificationVersion, + provider: this._config.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: this._config.maxInputTokens, + defaultMaxOutput: this._config.maxOutputTokens + }); + return [{ ...modelInfo, isDefault: true }]; } /** @@ -775,34 +745,83 @@ export class OpenAILanguageModel extends AILanguageModel implements positron.ai. async provideLanguageModelChatInformation(options: { silent: boolean }, token: vscode.CancellationToken): Promise { log.debug(`[${this.providerName}] Preparing language model chat information...`); - const models = this.modelListing ?? await this.resolveModels(token) ?? []; + const models = await this.resolveModels(token) ?? []; + + log.debug(`[${this.providerName}] Resolved ${models.length} models.`); return this.filterModels(models); } async resolveModels(token: vscode.CancellationToken): Promise { log.debug(`[${this.providerName}] Resolving models...`); - const data = await this.fetchModelsFromAPI(); - if (!data?.data || !Array.isArray(data.data)) { - log.info(`[${this.providerName}] Request was successful, but no models were returned.`); + const configuredModels = this.retrieveModelsFromConfig(); + if (configuredModels) { + this.modelListing = configuredModels; + return configuredModels; + } + + const apiModels = await this.retrieveModelsFromApi(); + if (apiModels) { + this.modelListing = apiModels; + return apiModels; + } + + return undefined; + } + + protected retrieveModelsFromConfig(): vscode.LanguageModelChatInformation[] | undefined { + const configuredModels = getAllModelDefinitions(this.provider); + if (configuredModels.length === 0) { return undefined; } - log.info(`[${this.providerName}] Successfully fetched ${data.data.length} models.`); - const models = data.data.map((model: any) => ({ - id: model.id, - name: model.id, - family: this.provider, - version: model.id, - maxInputTokens: 0, - maxOutputTokens: model.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT, - capabilities: this.capabilities, - isDefault: this.isDefaultUserModel(model.id, model.name), - isUserSelectable: true, - } satisfies vscode.LanguageModelChatInformation)); + log.info(`[${this.providerName}] Using ${configuredModels.length} configured models.`); - this.modelListing = models; - return this.modelListing; + const modelListing = configuredModels.map((modelDef) => + createModelInfo({ + id: modelDef.identifier, + name: modelDef.name, + family: this.provider, + version: modelDef.identifier, + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: modelDef.maxInputTokens ?? 0, + defaultMaxOutput: modelDef.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT + }) + ); + + return markDefaultModel(modelListing, this.provider, this._config.model); + } + + private async retrieveModelsFromApi(): Promise { + try { + const data = await this.fetchModelsFromAPI(); + if (!data?.data || !Array.isArray(data.data)) { + log.info(`[${this.providerName}] Request was successful, but no models were returned.`); + return undefined; + } + log.info(`[${this.providerName}] Successfully fetched ${data.data.length} models.`); + + const models = data.data.map((model: any) => + createModelInfo({ + id: model.id, + name: model.id, + family: this.provider, + version: model.id, + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: 0, + defaultMaxOutput: model.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT + }) + ); + + return models; + } catch (error) { + log.warn(`[${this.providerName}] Failed to fetch models from API: ${error}`); + return undefined; + } } filterModels(models: vscode.LanguageModelChatInformation[]): vscode.LanguageModelChatInformation[] { @@ -1069,12 +1088,6 @@ export class AWSLanguageModel extends AILanguageModel implements positron.ai.Lan inferenceProfiles: InferenceProfileSummary[] = []; constructor(_config: ModelConfig, _context?: vscode.ExtensionContext) { - // Update a stale model configuration to the latest defaults - const models = availableModels.get('amazon-bedrock')?.map(m => m.identifier) || []; - if (!(_config.model in models)) { - _config.name = AWSLanguageModel.source.defaults.name; - _config.model = AWSLanguageModel.source.defaults.model; - } super(_config, _context); const environmentSettings = vscode.workspace.getConfiguration('positron.assistant.providerVariables').get('bedrock', {}); @@ -1145,60 +1158,107 @@ export class AWSLanguageModel extends AILanguageModel implements positron.ai.Lan } async resolveModels(token: vscode.CancellationToken): Promise { - const command = new ListFoundationModelsCommand(); - - log.info(`[${this.providerName}] Fetching available Amazon Bedrock models for these providers: ` + AWSLanguageModel.SUPPORTED_BEDROCK_PROVIDERS.join(', ')); + log.debug(`[${this.providerName}] Resolving models...`); - const response = await this.bedrockClient.send(command); - const modelSummaries = response.modelSummaries; + const configuredModels = this.retrieveModelsFromConfig(); + if (configuredModels) { + this.modelListing = configuredModels; + return configuredModels; + } - if (!modelSummaries || modelSummaries.length === 0) { - log.error(`[${this.providerName}] No Amazon Bedrock models available`); - return []; + const apiModels = await this.retrieveModelsFromApi(); + if (apiModels) { + this.modelListing = apiModels; + return apiModels; } - log.info(`[${this.providerName}] Found ${modelSummaries.length} available models.`); - log.debug(`[${this.providerName}] Fetching available Amazon Bedrock inference profiles...`); - const inferenceResponse = await this.bedrockClient.send(new ListInferenceProfilesCommand()); - this.inferenceProfiles = inferenceResponse.inferenceProfileSummaries ?? []; + return undefined; + } - if (this.inferenceProfiles.length === 0) { - log.error(`[${this.providerName}] No Amazon Bedrock inference profiles available`); - return []; + protected retrieveModelsFromConfig(): vscode.LanguageModelChatInformation[] | undefined { + const configuredModels = getAllModelDefinitions(this.provider); + if (configuredModels.length === 0) { + return undefined; } - log.debug(`[${this.providerName}] Total inference profiles available: ${this.inferenceProfiles.length}`); - - // Filter for basic eligibility before creating model objects - const filteredModelSummaries = this.filterModelSummaries(modelSummaries); - log.debug(`[${this.providerName}] ${filteredModelSummaries.length} models available (from ${modelSummaries.length} total) after removing ineligible models.`); - - // Convert eligible model summaries to LanguageModelChatInformation objects - const models = filteredModelSummaries.map(m => { - const modelId = this.findInferenceProfileForModel(m.modelArn, this.inferenceProfiles); - const modelInfo: vscode.LanguageModelChatInformation = { - id: modelId, - name: m.modelName ?? modelId, + + log.info(`[${this.providerName}] Using ${configuredModels.length} configured models.`); + + const modelListing = configuredModels.map((modelDef) => + createModelInfo({ + id: modelDef.identifier, + name: modelDef.name, family: 'Amazon Bedrock', version: '', - maxInputTokens: AWSLanguageModel.DEFAULT_MAX_TOKENS_INPUT, - maxOutputTokens: AWSLanguageModel.DEFAULT_MAX_TOKENS_OUTPUT, + provider: this.provider, + providerName: this.providerName, capabilities: this.capabilities, - isDefault: this.isDefaultUserModel(modelId, m.modelName), - isUserSelectable: true, - }; - return modelInfo; - }).filter(m => { - if (!m.id) { - log.debug(`[${this.providerName}] Filtering out model without inference profile ARN: ${m.name}`); - return false; + defaultMaxInput: modelDef.maxInputTokens ?? AWSLanguageModel.DEFAULT_MAX_TOKENS_INPUT, + defaultMaxOutput: modelDef.maxOutputTokens ?? AWSLanguageModel.DEFAULT_MAX_TOKENS_OUTPUT + }) + ); + + return modelListing; + } + + private async retrieveModelsFromApi(): Promise { + try { + const command = new ListFoundationModelsCommand(); + + log.info(`[${this.providerName}] Fetching available Amazon Bedrock models for these providers: ` + AWSLanguageModel.SUPPORTED_BEDROCK_PROVIDERS.join(', ')); + + const response = await this.bedrockClient.send(command); + const modelSummaries = response.modelSummaries; + + if (!modelSummaries || modelSummaries.length === 0) { + log.error(`[${this.providerName}] No Amazon Bedrock models available`); + return []; } - return true; - }); + log.info(`[${this.providerName}] Found ${modelSummaries.length} available models.`); - log.debug(`[${this.providerName}] Available models after processing: ${models.map(m => m.name).join(', ')}`); + log.debug(`[${this.providerName}] Fetching available Amazon Bedrock inference profiles...`); + const inferenceResponse = await this.bedrockClient.send(new ListInferenceProfilesCommand()); + this.inferenceProfiles = inferenceResponse.inferenceProfileSummaries ?? []; - this.modelListing = models; - return models; + if (this.inferenceProfiles.length === 0) { + log.error(`[${this.providerName}] No Amazon Bedrock inference profiles available`); + return []; + } + log.debug(`[${this.providerName}] Total inference profiles available: ${this.inferenceProfiles.length}`); + + // Filter for basic eligibility before creating model objects + const filteredModelSummaries = this.filterModelSummaries(modelSummaries); + log.debug(`[${this.providerName}] ${filteredModelSummaries.length} models available (from ${modelSummaries.length} total) after removing ineligible models.`); + + // Convert eligible model summaries to LanguageModelChatInformation objects + const models = filteredModelSummaries.map(m => { + const modelId = this.findInferenceProfileForModel(m.modelArn, this.inferenceProfiles); + const modelInfo = createModelInfo({ + id: modelId, + name: m.modelName ?? modelId, + family: 'Amazon Bedrock', + version: '', + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: AWSLanguageModel.DEFAULT_MAX_TOKENS_INPUT, + defaultMaxOutput: AWSLanguageModel.DEFAULT_MAX_TOKENS_OUTPUT + }); + return modelInfo; + }).filter(m => { + if (!m.id) { + log.debug(`[${this.providerName}] Filtering out model without inference profile ARN: ${m.name}`); + return false; + } + return true; + }); + + log.debug(`[${this.providerName}] Available models after processing: ${models.map(m => m.name).join(', ')}`); + + return models; + } catch (error) { + log.warn(`[${this.providerName}] Failed to fetch models from Bedrock API: ${error}`); + return undefined; + } } filterModels(models: vscode.LanguageModelChatInformation[]): vscode.LanguageModelChatInformation[] { @@ -1414,72 +1474,3 @@ class GoogleLanguageModel extends AILanguageModel implements positron.ai.Languag return GoogleLanguageModel.source.provider.displayName; } } - -interface ModelDefinition { - name: string; - identifier: string; - maxInputTokens?: number; - maxOutputTokens?: number; -} - -// Note: we don't query for available models using any provider API since it may return ones that are not -// suitable for chat and we don't want the selection to be too large -export const availableModels = new Map( - [ - // - ['anthropic-api', [ - { - name: 'Claude Sonnet 4.5', - identifier: 'claude-sonnet-4-5', - maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - maxOutputTokens: 64_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - }, - { - name: 'Claude Opus 4.1', - identifier: 'claude-opus-4-1', - maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - maxOutputTokens: 32_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - }, - { - name: 'Claude Haiku 4.5', - identifier: 'claude-haiku-4-5', - maxInputTokens: 200_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - maxOutputTokens: 64_000, // reference: https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table - }, - ]], - ['google', [ - { - name: 'Gemini 2.5 Flash', - identifier: 'gemini-2.5-pro-exp-03-25', - maxOutputTokens: 65_536, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview - }, - { - name: 'Gemini 2.0 Flash', - identifier: 'gemini-2.0-flash-exp', - maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-2.0-flash - }, - { - name: 'Gemini 1.5 Flash 002', - identifier: 'gemini-1.5-flash-002', - maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-1.5-flash - }, - ]], - ['amazon-bedrock', [ - { - name: 'Claude 4 Sonnet Bedrock', - identifier: 'us.anthropic.claude-sonnet-4-20250514-v1:0', - maxOutputTokens: 8_192, // use more conservative value for Bedrock (up to 64K tokens available) - }, - { - name: 'Claude 4 Opus Bedrock', - identifier: 'us.anthropic.claude-opus-4-20250514-v1:0', - maxOutputTokens: 8_192, // use more conservative value for Bedrock (up to 32K tokens available) - }, - { - name: 'Claude 3.7 Sonnet v1 Bedrock', - identifier: 'us.anthropic.claude-3-7-sonnet-20250219-v1:0', - maxOutputTokens: 8_192, // use more conservative value for Bedrock (up to 64K tokens available) - }, - ]] - ] -); diff --git a/extensions/positron-assistant/src/posit.ts b/extensions/positron-assistant/src/posit.ts index 1338cf12b407..47533954295f 100644 --- a/extensions/positron-assistant/src/posit.ts +++ b/extensions/positron-assistant/src/posit.ts @@ -11,7 +11,9 @@ import { deleteConfiguration, ModelConfig, SecretStorage } from './config'; import { DEFAULT_MAX_TOKEN_INPUT, DEFAULT_MAX_TOKEN_OUTPUT } from './constants.js'; import { log, recordRequestTokenUsage, recordTokenUsage } from './extension.js'; import { isCacheControlOptions, toAnthropicMessages, toAnthropicSystem, toAnthropicToolChoice, toAnthropicTools, toTokenUsage } from './anthropic.js'; -import { availableModels } from './models.js'; +import { getAllModelDefinitions } from './modelDefinitions.js'; +import { createModelInfo, markDefaultModel } from './modelResolutionHelpers.js'; +import { applyModelFilters } from './modelFilters.js'; export const DEFAULT_POSITAI_MODEL_NAME = 'Claude Sonnet 4.5'; export const DEFAULT_POSITAI_MODEL_MATCH = 'claude-sonnet-4-5'; @@ -425,25 +427,11 @@ export class PositLanguageModel implements positron.ai.LanguageModelChatProvider } async provideLanguageModelChatInformation(_options: { silent: boolean }, token: vscode.CancellationToken): Promise { - log.trace('[Posit AI] Preparing language models'); + log.debug(`[${this.providerName}] Preparing language model chat information...`); + const models = await this.resolveModels(token) ?? []; - await this.resolveModels(token); - - if (this.modelListing.length > 0) { - return this.modelListing; - } else { - return [{ - id: PositLanguageModel.source.defaults.model, - name: PositLanguageModel.source.defaults.name, - family: this.provider, - version: this._context?.extension.packageJSON.version ?? '', - maxInputTokens: this.maxInputTokens, - maxOutputTokens: this.maxOutputTokens, - capabilities: this.capabilities, - isDefault: true, - isUserSelectable: true, - }]; - } + log.debug(`[${this.providerName}] Resolved ${models.length} models.`); + return this.filterModels(models); } async provideTokenCount(model: vscode.LanguageModelChatInformation, text: string | vscode.LanguageModelChatMessage | vscode.LanguageModelChatMessage2, token: vscode.CancellationToken): Promise { @@ -459,6 +447,10 @@ export class PositLanguageModel implements positron.ai.LanguageModelChatProvider return; } + protected filterModels(models: vscode.LanguageModelChatInformation[]): vscode.LanguageModelChatInformation[] { + return applyModelFilters(models, this.provider, this.providerName); + } + private onContentBlock(block: Anthropic.ContentBlock, progress: vscode.Progress): void { switch (block.type) { case 'tool_use': @@ -474,38 +466,53 @@ export class PositLanguageModel implements positron.ai.LanguageModelChatProvider progress.report(new vscode.LanguageModelTextPart(textDelta)); } - private isDefaultUserModel(id: string, name?: string): boolean { - const config = vscode.workspace.getConfiguration('positron.assistant'); - const defaultModels = config.get>('defaultModels') || {}; - if ('posit-ai' in defaultModels) { - if (id.includes(defaultModels['posit-ai']) || name?.includes(defaultModels['posit-ai'])) { - return true; - } + private retrieveModelsFromConfig(): vscode.LanguageModelChatInformation[] | undefined { + // Check for configured models (user or built-in) + const configuredModels = getAllModelDefinitions(this.provider); + if (configuredModels.length === 0) { + return undefined; } - return id.includes(DEFAULT_POSITAI_MODEL_MATCH); + + log.info(`[${this.provider}] Using ${configuredModels.length} configured models.`); + + const modelListing = configuredModels.map((modelDef) => + createModelInfo({ + id: modelDef.identifier, + name: modelDef.name, + family: this.provider, + version: '', + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: modelDef.maxInputTokens, + defaultMaxOutput: modelDef.maxOutputTokens + }) + ); + + return markDefaultModel(modelListing, this.provider, DEFAULT_POSITAI_MODEL_MATCH); } async resolveModels(token: vscode.CancellationToken): Promise { - log.trace(`Resolving models for provider ${this._config.provider}`); - return new Promise((resolve) => { - // Use Anthropic model spec for Posit AI provider - const models = availableModels.get('anthropic-api'); - if (models) { - this.modelListing = models.map(model => ({ - id: model.identifier, - name: model.name, - family: this.provider, - version: this._context?.extension.packageJSON.version ?? '', - maxInputTokens: model.maxInputTokens ?? DEFAULT_MAX_TOKEN_INPUT, - maxOutputTokens: model.maxOutputTokens ?? DEFAULT_MAX_TOKEN_OUTPUT, - capabilities: this.capabilities, - isDefault: this.isDefaultUserModel(model.identifier, model.name), - isUserSelectable: true, - } satisfies vscode.LanguageModelChatInformation)); - resolve(this.modelListing); - } else { - resolve(undefined); - } + log.debug(`[${this.provider}] Resolving models...`); + + const configuredModels = this.retrieveModelsFromConfig(); + if (configuredModels) { + this.modelListing = configuredModels; + return configuredModels; + } + + log.warn(`[${this.provider}] No models available. Using fallback model.`); + const fallbackModel = createModelInfo({ + id: PositLanguageModel.source.defaults.model, + name: PositLanguageModel.source.defaults.name, + family: this.provider, + version: this._context?.extension.packageJSON.version ?? '', + provider: this.provider, + providerName: this.providerName, + capabilities: this.capabilities, + defaultMaxInput: this.maxInputTokens, + defaultMaxOutput: this.maxOutputTokens }); + return [{ ...fallbackModel, isDefault: true }]; } } diff --git a/extensions/positron-assistant/src/test/anthropic.test.ts b/extensions/positron-assistant/src/test/anthropic.test.ts index a45b59eb0c61..8cab4f7e4039 100644 --- a/extensions/positron-assistant/src/test/anthropic.test.ts +++ b/extensions/positron-assistant/src/test/anthropic.test.ts @@ -13,6 +13,8 @@ import { EMPTY_TOOL_RESULT_PLACEHOLDER, languageModelCacheBreakpointPart } from import Anthropic from '@anthropic-ai/sdk'; import { MessageStream } from '@anthropic-ai/sdk/lib/MessageStream.js'; import { mock } from './utils.js'; +import * as modelDefinitionsModule from '../modelDefinitions.js'; +import * as helpersModule from '../modelResolutionHelpers.js'; class MockAnthropicClient { messages = { @@ -586,4 +588,232 @@ suite('AnthropicLanguageModel', () => { ] satisfies Anthropic.MessageCreateParams['messages'], 'Unexpected user messages in request body'); }); }); + + suite('Model Resolution', () => { + let mockWorkspaceConfig: sinon.SinonStub; + let mockModelDefinitions: sinon.SinonStub; + let mockHelpers: { createModelInfo: sinon.SinonStub; isDefaultUserModel: sinon.SinonStub; markDefaultModel: sinon.SinonStub }; + let getConfigurationStub: sinon.SinonStub; + + setup(() => { + // Mock vscode.workspace.getConfiguration for model resolution tests + mockWorkspaceConfig = sinon.stub(); + getConfigurationStub = sinon.stub(vscode.workspace, 'getConfiguration').returns({ + get: mockWorkspaceConfig + } as any); + + // Mock getAllModelDefinitions + mockModelDefinitions = sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions'); + + // Mock helper functions + mockHelpers = { + createModelInfo: sinon.stub(helpersModule, 'createModelInfo'), + isDefaultUserModel: sinon.stub(helpersModule, 'isDefaultUserModel'), + markDefaultModel: sinon.stub(helpersModule, 'markDefaultModel') + }; + }); + + teardown(() => { + // Restore specific stubs for this suite + getConfigurationStub.restore(); + mockModelDefinitions.restore(); + mockHelpers.createModelInfo.restore(); + mockHelpers.isDefaultUserModel.restore(); + mockHelpers.markDefaultModel.restore(); + }); + + suite('retrieveModelsFromConfig', () => { + test('returns undefined when no configured models', () => { + mockModelDefinitions.returns([]); + const result = (model as any).retrieveModelsFromConfig(); + assert.strictEqual(result, undefined); + }); + + test('returns configured models when built-in models exist', () => { + const builtInModels = [ + { + name: 'Claude Sonnet 4.5', + identifier: 'claude-sonnet-4-5', + maxInputTokens: 200_000, + maxOutputTokens: 64_000 + } + ]; + mockModelDefinitions.returns(builtInModels); + + const mockModelInfo = { + id: 'claude-sonnet-4-5', + name: 'Claude Sonnet 4.5', + family: 'anthropic-api', + version: '', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + const result = (model as any).retrieveModelsFromConfig(); + + assert.ok(result, 'Should return built-in models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'claude-sonnet-4-5'); + }); + + test('marks one model as default', () => { + const configuredModels = [ + { name: 'Claude A', identifier: 'claude-a' }, + { name: 'Claude B', identifier: 'claude-b' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfoA = { id: 'claude-a', name: 'Claude A', isDefault: false, isUserSelectable: true }; + const mockModelInfoB = { id: 'claude-b', name: 'Claude B', isDefault: false, isUserSelectable: true }; + mockHelpers.createModelInfo.onFirstCall().returns(mockModelInfoA); + mockHelpers.createModelInfo.onSecondCall().returns(mockModelInfoB); + + // Mock markDefaultModel to return the expected result with claude-b as default + const expectedResult = [ + { id: 'claude-a', name: 'Claude A', isDefault: false, isUserSelectable: true }, + { id: 'claude-b', name: 'Claude B', isDefault: true, isUserSelectable: true } + ]; + mockHelpers.markDefaultModel.returns(expectedResult); + + const result = (model as any).retrieveModelsFromConfig(); + + assert.ok(result, 'Should return models'); + assert.strictEqual(result.length, 2); + // Only one should be marked as default + const defaultModels = result.filter((m: any) => m.isDefault); + assert.strictEqual(defaultModels.length, 1); + assert.strictEqual(defaultModels[0].id, 'claude-b'); + }); + + test('falls back to DEFAULT_ANTHROPIC_MODEL_MATCH for default', () => { + const configuredModels = [ + { name: 'Claude Sonnet 4.5', identifier: 'claude-sonnet-4-5' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5', isDefault: false, isUserSelectable: true }; + mockHelpers.createModelInfo.returns(mockModelInfo); + + // Mock markDefaultModel to return the expected result with the model as default + const expectedResult = [ + { id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5', isDefault: true, isUserSelectable: true } + ]; + mockHelpers.markDefaultModel.returns(expectedResult); + + const result = (model as any).retrieveModelsFromConfig(); + + const defaultModel = result.find((m: any) => m.isDefault); + assert.ok(defaultModel, 'Should have a default model'); + assert.strictEqual(defaultModel.id, 'claude-sonnet-4-5'); + + // Verify markDefaultModel was called with the correct parameters + sinon.assert.calledWith(mockHelpers.markDefaultModel, [mockModelInfo], 'anthropic-api', 'claude-sonnet-4'); + }); + }); + + suite('resolveModels integration', () => { + let cancellationToken: vscode.CancellationToken; + + setup(() => { + const cancellationTokenSource = new vscode.CancellationTokenSource(); + cancellationToken = cancellationTokenSource.token; + }); + + test('prioritizes configured models over API', async () => { + const configuredModels = [ + { name: 'User Claude', identifier: 'user-claude' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { + id: 'user-claude', + name: 'User Claude', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + const result = await model.resolveModels(cancellationToken); + + assert.ok(result, 'Should return configured models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'user-claude'); + assert.strictEqual(model.modelListing, result); + }); + + test('falls back to API when no configured models', async () => { + mockModelDefinitions.returns([]); // No configured models + + const mockAnthropicClient = { + models: { + list: sinon.stub().resolves({ + data: [ + { id: 'claude-api-model', display_name: 'Claude API Model', created_at: '2025-01-01T00:00:00Z' } + ], + has_more: false + }) + } + }; + (model as any)._client = mockAnthropicClient; + + const mockModelInfo = { + id: 'claude-api-model', + name: 'Claude API Model', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + const result = await model.resolveModels(cancellationToken); + + assert.ok(result, 'Should return API models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'claude-api-model'); + assert.strictEqual(model.modelListing, result); + }); + + test('returns undefined when both configured and API fail', async () => { + mockModelDefinitions.returns([]); // No configured models + + const mockAnthropicClient = { + models: { + list: sinon.stub().rejects(new Error('API Error')) + } + }; + (model as any)._client = mockAnthropicClient; + + const result = await model.resolveModels(cancellationToken); + + assert.strictEqual(result, undefined); + }); + + test('caches resolved models in modelListing', async () => { + const configuredModels = [ + { name: 'Cached Claude', identifier: 'cached-claude' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { + id: 'cached-claude', + name: 'Cached Claude', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + await model.resolveModels(cancellationToken); + + assert.strictEqual(model.modelListing.length, 1); + assert.strictEqual(model.modelListing[0].id, 'cached-claude'); + }); + }); + }); }); diff --git a/extensions/positron-assistant/src/test/modelDefinitions.test.ts b/extensions/positron-assistant/src/test/modelDefinitions.test.ts new file mode 100644 index 000000000000..1b2d9f87f2a6 --- /dev/null +++ b/extensions/positron-assistant/src/test/modelDefinitions.test.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as vscode from 'vscode'; +import * as sinon from 'sinon'; +import { getAllModelDefinitions } from '../modelDefinitions.js'; + +suite('Model Definitions', () => { + let mockWorkspaceConfig: sinon.SinonStub; + + setup(() => { + // Mock vscode.workspace.getConfiguration + mockWorkspaceConfig = sinon.stub(); + sinon.stub(vscode.workspace, 'getConfiguration').returns({ + get: mockWorkspaceConfig + } as any); + }); + + teardown(() => { + sinon.restore(); + }); + + suite('getAllModelDefinitions', () => { + test('prioritizes user-configured models over built-in', () => { + const userModels = [ + { + name: 'User Claude', + identifier: 'user-claude' + } + ]; + mockWorkspaceConfig.withArgs('configuredModels', {}).returns({ + 'anthropic-api': userModels + }); + + const result = getAllModelDefinitions('anthropic-api'); + + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].name, 'User Claude'); + assert.strictEqual(result[0].identifier, 'user-claude'); + }); + + test('returns empty array for unknown provider', () => { + mockWorkspaceConfig.withArgs('configuredModels', {}).returns({}); + + const result = getAllModelDefinitions('unknown-provider'); + + assert.deepStrictEqual(result, []); + }); + + }); +}); diff --git a/extensions/positron-assistant/src/test/modelResolutionHelpers.test.ts b/extensions/positron-assistant/src/test/modelResolutionHelpers.test.ts new file mode 100644 index 000000000000..390064fad179 --- /dev/null +++ b/extensions/positron-assistant/src/test/modelResolutionHelpers.test.ts @@ -0,0 +1,415 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as vscode from 'vscode'; +import * as sinon from 'sinon'; +import * as extensionModule from '../extension.js'; +import * as modelDefinitionsModule from '../modelDefinitions.js'; +import { + isDefaultUserModel, + getMaxTokens, + markDefaultModel +} from '../modelResolutionHelpers.js'; + +suite('Model Resolution Helpers', () => { + let mockGetConfiguration: sinon.SinonStub; + let mockLog: { + warn: sinon.SinonStub; + trace: sinon.SinonStub; + info: sinon.SinonStub; + error: sinon.SinonStub; + }; + + setup(() => { + // Mock vscode.workspace.getConfiguration + mockGetConfiguration = sinon.stub(); + sinon.stub(vscode.workspace, 'getConfiguration').returns({ + get: mockGetConfiguration + } as any); + + // Mock the log module + mockLog = { + warn: sinon.stub(), + trace: sinon.stub(), + info: sinon.stub(), + error: sinon.stub() + }; + + // Mock the extension module + sinon.stub(extensionModule, 'log').value(mockLog); + }); + + teardown(() => { + sinon.restore(); + }); + + suite('isDefaultUserModel', () => { + test('returns true when model ID matches user-configured default', () => { + const defaultModels = { 'anthropic-api': 'sonnet-4' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = isDefaultUserModel('anthropic-api', 'claude-sonnet-4-5', 'Claude Sonnet 4.5'); + + assert.strictEqual(result, true); + }); + + test('returns true when model name matches user-configured default', () => { + const defaultModels = { 'anthropic-api': 'Sonnet' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = isDefaultUserModel('anthropic-api', 'claude-sonnet-4-5', 'Claude Sonnet 4.5'); + + assert.strictEqual(result, true); + }); + + test('returns false when no match and no defaultMatch provided', () => { + const defaultModels = { 'anthropic-api': 'opus' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = isDefaultUserModel('anthropic-api', 'claude-sonnet-4-5', 'Claude Sonnet 4.5'); + + assert.strictEqual(result, false); + }); + + test('returns true when ID matches defaultMatch pattern', () => { + mockGetConfiguration.withArgs('defaultModels').returns({}); + + const result = isDefaultUserModel('anthropic-api', 'claude-sonnet-4-5', 'Claude Sonnet 4.5', 'sonnet-4'); + + assert.strictEqual(result, true); + }); + + test('prioritizes user config over defaultMatch', () => { + const defaultModels = { 'anthropic-api': 'opus' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + // User wants opus, but defaultMatch suggests sonnet - user config should win + const result = isDefaultUserModel('anthropic-api', 'claude-opus-4-1', 'Claude Opus 4.1', 'sonnet-4'); + + assert.strictEqual(result, true); + }); + + test('handles provider not in defaultModels config', () => { + const defaultModels = { 'anthropic-api': 'sonnet' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = isDefaultUserModel('openai-api', 'gpt-4', 'GPT-4'); + + assert.strictEqual(result, false); + }); + }); + + suite('getMaxTokens', () => { + test('returns global default when no overrides', () => { + mockGetConfiguration.withArgs('maxInputTokens', {}).returns({}); + mockGetConfiguration.withArgs('maxOutputTokens', {}).returns({}); + + // Mock getAllModelDefinitions to return empty array + sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions').returns([]); + + const resultInput = getMaxTokens('test-model', 'input', 'test-provider'); + const resultOutput = getMaxTokens('test-model', 'output', 'test-provider'); + + // Should use DEFAULT_MAX_TOKEN_INPUT (100_000) and DEFAULT_MAX_TOKEN_OUTPUT (4_096) + assert.strictEqual(resultInput, 100_000); + assert.strictEqual(resultOutput, 4_096); + }); + + test('uses provider-specific default over global default', () => { + mockGetConfiguration.withArgs('maxInputTokens', {}).returns({}); + mockGetConfiguration.withArgs('maxOutputTokens', {}).returns({}); + + // Mock getAllModelDefinitions to return empty array + sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions').returns([]); + + const resultInput = getMaxTokens('test-model', 'input', 'test-provider', 200_000, 'Test Provider'); + const resultOutput = getMaxTokens('test-model', 'output', 'test-provider', 8_192, 'Test Provider'); + + assert.strictEqual(resultInput, 200_000); + assert.strictEqual(resultOutput, 8_192); + }); + + test('uses model definition limit over defaults', () => { + mockGetConfiguration.withArgs('maxInputTokens', {}).returns({}); + mockGetConfiguration.withArgs('maxOutputTokens', {}).returns({}); + + // Mock getAllModelDefinitions to return model with specific token limits + sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions').returns([ + { + identifier: 'test-model', + name: 'Test Model', + maxInputTokens: 175_000, + maxOutputTokens: 32_000 + } + ]); + + const resultInput = getMaxTokens('test-model', 'input', 'test-provider', 200_000); + const resultOutput = getMaxTokens('test-model', 'output', 'test-provider', 8_192); + + assert.strictEqual(resultInput, 175_000); + assert.strictEqual(resultOutput, 32_000); + }); + + test('uses user workspace setting over model definition', () => { + mockGetConfiguration.withArgs('maxInputTokens', {}).returns({ 'test-model': 250_000 }); + mockGetConfiguration.withArgs('maxOutputTokens', {}).returns({ 'test-model': 64_000 }); + + // Mock getAllModelDefinitions to return model with different limits + sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions').returns([ + { + identifier: 'test-model', + name: 'Test Model', + maxInputTokens: 175_000, + maxOutputTokens: 32_000 + } + ]); + + const resultInput = getMaxTokens('test-model', 'input', 'test-provider'); + const resultOutput = getMaxTokens('test-model', 'output', 'test-provider'); + + assert.strictEqual(resultInput, 250_000); + assert.strictEqual(resultOutput, 64_000); + }); + }); + + suite('markDefaultModel', () => { + test('marks user-preferred model as default', () => { + const models = [ + { + id: 'claude-opus-4-1', + name: 'Claude Opus 4.1', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 8_192, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-sonnet-4-5', + name: 'Claude Sonnet 4.5', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + } + ]; + + // Mock workspace configuration to simulate user preference for opus + const defaultModels = { 'anthropic-api': 'opus' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = markDefaultModel(models as any, 'anthropic-api', 'claude-sonnet-4'); + + const defaultModel = result.find((m: any) => m.isDefault); + assert.strictEqual(defaultModel.id, 'claude-opus-4-1'); + const nonDefaultModels = result.filter((m: any) => !m.isDefault); + assert.strictEqual(nonDefaultModels.length, 1); + assert.strictEqual(nonDefaultModels[0].id, 'claude-sonnet-4-5'); + }); + + test('marks first model as default when no user preference matches', () => { + const models = [ + { + id: 'claude-a', + name: 'Claude A', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 8_192, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-b', + name: 'Claude B', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + } + ]; + + // Mock workspace configuration with no matching preference + const defaultModels = { 'anthropic-api': 'nonexistent' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = markDefaultModel(models as any, 'anthropic-api', 'claude-sonnet-4'); + + assert.strictEqual(result[0].isDefault, true); + assert.strictEqual(result[1].isDefault, false); + }); + + test('uses defaultMatch when no user preference is set', () => { + const models = [ + { + id: 'claude-opus-4-1', + name: 'Claude Opus 4.1', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 8_192, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-sonnet-4-5', + name: 'Claude Sonnet 4.5', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + } + ]; + + // Mock workspace configuration with no default models + mockGetConfiguration.withArgs('defaultModels').returns({}); + + const result = markDefaultModel(models as any, 'anthropic-api', 'sonnet-4'); + + const defaultModel = result.find((m: any) => m.isDefault); + assert.strictEqual(defaultModel.id, 'claude-sonnet-4-5'); + const nonDefaultModels = result.filter((m: any) => !m.isDefault); + assert.strictEqual(nonDefaultModels.length, 1); + assert.strictEqual(nonDefaultModels[0].id, 'claude-opus-4-1'); + }); + + test('ensures only one model is marked as default', () => { + const models = [ + { + id: 'claude-a', + name: 'Claude A', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 8_192, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-b', + name: 'Claude B', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-c', + name: 'Claude C', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 32_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + } + ]; + + // Mock workspace configuration that would match multiple models + const defaultModels = { 'anthropic-api': 'claude' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = markDefaultModel(models as any, 'anthropic-api', 'claude-sonnet-4'); + + const defaultModels_result = result.filter((m: any) => m.isDefault); + assert.strictEqual(defaultModels_result.length, 1); + assert.strictEqual(defaultModels_result[0].id, 'claude-a'); // First match wins + }); + + test('handles empty model list', () => { + const result = markDefaultModel([], 'anthropic-api', 'claude-sonnet-4'); + + assert.deepStrictEqual(result, []); + }); + + test('preserves model properties while updating isDefault', () => { + const models = [ + { + id: 'test-model', + name: 'Test Model', + family: 'test-family', + version: '1.0', + maxInputTokens: 100_000, + maxOutputTokens: 4_096, + capabilities: { vision: false, toolCalling: true, agentMode: false }, + isDefault: false, + isUserSelectable: true + } + ]; + + // Mock workspace configuration with no default models + mockGetConfiguration.withArgs('defaultModels').returns({}); + + const result = markDefaultModel(models as any, 'test-provider'); + + assert.strictEqual(result.length, 1); + const model = result[0]; + assert.strictEqual(model.id, 'test-model'); + assert.strictEqual(model.name, 'Test Model'); + assert.strictEqual(model.family, 'test-family'); + assert.strictEqual(model.version, '1.0'); + assert.strictEqual(model.maxInputTokens, 100_000); + assert.strictEqual(model.maxOutputTokens, 4_096); + assert.deepStrictEqual(model.capabilities, { vision: false, toolCalling: true, agentMode: false }); + assert.strictEqual(model.isDefault, true); // Should be marked as default (first/only model) + assert.strictEqual(model.isUserSelectable, true); + }); + + test('prioritizes user config over defaultMatch', () => { + const models = [ + { + id: 'claude-opus-4-1', + name: 'Claude Opus 4.1', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 8_192, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + }, + { + id: 'claude-sonnet-4-5', + name: 'Claude Sonnet 4.5', + family: 'anthropic-api', + version: '1.0', + maxInputTokens: 200_000, + maxOutputTokens: 64_000, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: false, + isUserSelectable: true + } + ]; + + // User wants opus, but defaultMatch suggests sonnet - user config should win + const defaultModels = { 'anthropic-api': 'opus' }; + mockGetConfiguration.withArgs('defaultModels').returns(defaultModels); + + const result = markDefaultModel(models as any, 'anthropic-api', 'sonnet-4'); + + const defaultModel = result.find((m: any) => m.isDefault); + assert.strictEqual(defaultModel.id, 'claude-opus-4-1'); + }); + }); +}); diff --git a/extensions/positron-assistant/src/test/models.test.ts b/extensions/positron-assistant/src/test/models.test.ts deleted file mode 100644 index 179d632d6b5c..000000000000 --- a/extensions/positron-assistant/src/test/models.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (C) 2025 Posit Software, PBC. All rights reserved. - * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as assert from 'assert'; -import * as vscode from 'vscode'; -import * as positron from 'positron'; -import * as sinon from 'sinon'; -import { OpenAILanguageModel } from '../models.js'; -import { ModelConfig } from '../config.js'; - -suite('Models', () => { - let mockWorkspaceConfig: sinon.SinonStub; - - setup(() => { - // Mock vscode.workspace.getConfiguration - mockWorkspaceConfig = sinon.stub(); - sinon.stub(vscode.workspace, 'getConfiguration').returns({ - get: mockWorkspaceConfig - } as any); - }); - - teardown(() => { - sinon.restore(); - }); - - suite('OpenAILanguageModel', () => { - let mockConfig: ModelConfig; - let openAIModel: OpenAILanguageModel; - - setup(() => { - mockConfig = { - id: 'openai-test', - provider: 'openai-api', - type: positron.PositronLanguageModelType.Chat, - name: 'OpenAI Test', - model: 'gpt-4', - apiKey: 'test-api-key', // pragma: allowlist secret - baseUrl: 'https://api.openai.com/v1' - }; - - // Mock the applyModelFilters import - mockWorkspaceConfig.withArgs('unfilteredProviders', []).returns([]); - mockWorkspaceConfig.withArgs('filterModels', []).returns([]); - - openAIModel = new OpenAILanguageModel(mockConfig); - }); - - test('filterModels filters out models based on FILTERED_MODEL_PATTERNS', async () => { - // Create mock models that include patterns to be filtered - const allModels = [ - // Models that should be kept - { id: 'gpt-4', name: 'gpt-4', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, - { id: 'gpt-3.5-turbo', name: 'gpt-3.5-turbo', family: 'openai-api', version: '1.0', maxInputTokens: 4096, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, - - // Models that should be filtered out based on FILTERED_MODEL_PATTERNS - { id: 'gpt-5-search-api', name: 'gpt-5-search-api', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'search' - { id: 'gpt-4o-mini-audio-preview', name: 'gpt-4o-mini-audio-preview', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'audio' - { id: 'gpt-4o-realtime-preview', name: 'gpt-4o-realtime-preview', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'realtime' - { id: 'gpt-4o-transcribe', name: 'gpt-4o-transcribe', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'transcribe' - - // Case insensitive test - should be filtered out - { id: 'GPT-4-AUDIO-MODEL', name: 'GPT-4-AUDIO-MODEL', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'audio' (case insensitive) - { id: 'Text-Search-Model', name: 'Text-Search-Model', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'search' (case insensitive) - { id: 'GPT-4O-REALTIME-TEST', name: 'GPT-4O-REALTIME-TEST', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'realtime' (case insensitive) - - // Edge cases - should NOT be filtered out (word boundary protection) - { id: 'gpt-4-research-model', name: 'gpt-4-research-model', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'research' but not 'search' as whole word - { id: 'claude-multiaudio-beta', name: 'claude-multiaudio-beta', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'multiaudio' but not 'audio' as whole word - ] satisfies vscode.LanguageModelChatInformation[]; - - const result = openAIModel.filterModels(allModels); - - // Should include 4 models: 2 original + 2 edge case models that shouldn't be filtered - assert.strictEqual(result.length, 4, 'Should filter out audio, search, realtime, and transcribe models while preserving edge cases'); - - const modelIds = result.map(m => m.id); - - // Should include these models (don't contain any filtered patterns) - assert.ok(modelIds.includes('gpt-4'), 'Should include gpt-4'); - assert.ok(modelIds.includes('gpt-3.5-turbo'), 'Should include gpt-3.5-turbo'); - - // Should include edge case models (word boundary protection) - assert.ok(modelIds.includes('gpt-4-research-model'), 'Should include gpt-4-research-model (research != search as whole word)'); - assert.ok(modelIds.includes('claude-multiaudio-beta'), 'Should include claude-multiaudio-beta (multiaudio != audio as whole word)'); - - // Should exclude models with 'search' pattern - assert.ok(!modelIds.includes('gpt-5-search-api'), 'Should exclude gpt-5-search-api (contains search)'); - assert.ok(!modelIds.includes('Text-Search-Model'), 'Should exclude Text-Search-Model (contains search, case insensitive)'); - - // Should exclude models with 'audio' pattern - assert.ok(!modelIds.includes('gpt-4o-mini-audio-preview'), 'Should exclude gpt-4o-mini-audio-preview (contains audio)'); - assert.ok(!modelIds.includes('GPT-4-AUDIO-MODEL'), 'Should exclude GPT-4-AUDIO-MODEL (contains audio, case insensitive)'); - - // Should exclude models with 'realtime' pattern - assert.ok(!modelIds.includes('gpt-4o-realtime-preview'), 'Should exclude gpt-4o-realtime-preview (contains realtime)'); - assert.ok(!modelIds.includes('GPT-4O-REALTIME-TEST'), 'Should exclude GPT-4O-REALTIME-TEST (contains realtime, case insensitive)'); - - // Should exclude models with 'transcribe' pattern - assert.ok(!modelIds.includes('gpt-4o-transcribe'), 'Should exclude gpt-4o-transcribe (contains transcribe)'); - }); - }); -}); diff --git a/extensions/positron-assistant/src/test/openai.test.ts b/extensions/positron-assistant/src/test/openai.test.ts new file mode 100644 index 000000000000..82311d78d040 --- /dev/null +++ b/extensions/positron-assistant/src/test/openai.test.ts @@ -0,0 +1,319 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2025 Posit Software, PBC. All rights reserved. + * Licensed under the Elastic License 2.0. See LICENSE.txt for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as vscode from 'vscode'; +import * as positron from 'positron'; +import * as sinon from 'sinon'; +import { OpenAILanguageModel } from '../models.js'; +import { ModelConfig } from '../config.js'; +import * as modelDefinitionsModule from '../modelDefinitions.js'; +import * as helpersModule from '../modelResolutionHelpers.js'; + +suite('OpenAILanguageModel', () => { + let mockWorkspaceConfig: sinon.SinonStub; + let mockConfig: ModelConfig; + let openAIModel: OpenAILanguageModel; + + setup(() => { + // Mock vscode.workspace.getConfiguration + mockWorkspaceConfig = sinon.stub(); + sinon.stub(vscode.workspace, 'getConfiguration').returns({ + get: mockWorkspaceConfig + } as any); + + mockConfig = { + id: 'openai-test', + provider: 'openai-api', + type: positron.PositronLanguageModelType.Chat, + name: 'OpenAI Test', + model: 'gpt-4', + apiKey: 'test-api-key', // pragma: allowlist secret + baseUrl: 'https://api.openai.com/v1', + maxInputTokens: 8192, + maxOutputTokens: 4096 + }; + + // Mock the applyModelFilters import + mockWorkspaceConfig.withArgs('unfilteredProviders', []).returns([]); + mockWorkspaceConfig.withArgs('filterModels', []).returns([]); + + openAIModel = new OpenAILanguageModel(mockConfig); + }); + + teardown(() => { + sinon.restore(); + }); + + test('filterModels filters out models based on FILTERED_MODEL_PATTERNS', async () => { + // Create mock models that include patterns to be filtered + const allModels = [ + // Models that should be kept + { id: 'gpt-4', name: 'gpt-4', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, + { id: 'gpt-3.5-turbo', name: 'gpt-3.5-turbo', family: 'openai-api', version: '1.0', maxInputTokens: 4096, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, + + // Models that should be filtered out based on FILTERED_MODEL_PATTERNS + { id: 'gpt-5-search-api', name: 'gpt-5-search-api', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'search' + { id: 'gpt-4o-mini-audio-preview', name: 'gpt-4o-mini-audio-preview', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'audio' + { id: 'gpt-4o-realtime-preview', name: 'gpt-4o-realtime-preview', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'realtime' + { id: 'gpt-4o-transcribe', name: 'gpt-4o-transcribe', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'transcribe' + + // Case insensitive test - should be filtered out + { id: 'GPT-4-AUDIO-MODEL', name: 'GPT-4-AUDIO-MODEL', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'audio' (case insensitive) + { id: 'Text-Search-Model', name: 'Text-Search-Model', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'search' (case insensitive) + { id: 'GPT-4O-REALTIME-TEST', name: 'GPT-4O-REALTIME-TEST', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'realtime' (case insensitive) + + // Edge cases - should NOT be filtered out (word boundary protection) + { id: 'gpt-4-research-model', name: 'gpt-4-research-model', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'research' but not 'search' as whole word + { id: 'claude-multiaudio-beta', name: 'claude-multiaudio-beta', family: 'openai-api', version: '1.0', maxInputTokens: 8192, maxOutputTokens: 4096, capabilities: {}, isDefault: false, isUserSelectable: true }, // contains 'multiaudio' but not 'audio' as whole word + ] satisfies vscode.LanguageModelChatInformation[]; + + const result = openAIModel.filterModels(allModels); + + // Should include 4 models: 2 original + 2 edge case models that shouldn't be filtered + assert.strictEqual(result.length, 4, 'Should filter out audio, search, realtime, and transcribe models while preserving edge cases'); + + const modelIds = result.map(m => m.id); + + // Should include these models (don't contain any filtered patterns) + assert.ok(modelIds.includes('gpt-4'), 'Should include gpt-4'); + assert.ok(modelIds.includes('gpt-3.5-turbo'), 'Should include gpt-3.5-turbo'); + + // Should include edge case models (word boundary protection) + assert.ok(modelIds.includes('gpt-4-research-model'), 'Should include gpt-4-research-model (research != search as whole word)'); + assert.ok(modelIds.includes('claude-multiaudio-beta'), 'Should include claude-multiaudio-beta (multiaudio != audio as whole word)'); + + // Should exclude models with 'search' pattern + assert.ok(!modelIds.includes('gpt-5-search-api'), 'Should exclude gpt-5-search-api (contains search)'); + assert.ok(!modelIds.includes('Text-Search-Model'), 'Should exclude Text-Search-Model (contains search, case insensitive)'); + + // Should exclude models with 'audio' pattern + assert.ok(!modelIds.includes('gpt-4o-mini-audio-preview'), 'Should exclude gpt-4o-mini-audio-preview (contains audio)'); + assert.ok(!modelIds.includes('GPT-4-AUDIO-MODEL'), 'Should exclude GPT-4-AUDIO-MODEL (contains audio, case insensitive)'); + + // Should exclude models with 'realtime' pattern + assert.ok(!modelIds.includes('gpt-4o-realtime-preview'), 'Should exclude gpt-4o-realtime-preview (contains realtime)'); + assert.ok(!modelIds.includes('GPT-4O-REALTIME-TEST'), 'Should exclude GPT-4O-REALTIME-TEST (contains realtime, case insensitive)'); + + // Should exclude models with 'transcribe' pattern + assert.ok(!modelIds.includes('gpt-4o-transcribe'), 'Should exclude gpt-4o-transcribe (contains transcribe)'); + }); + + suite('Model Resolution', () => { + let mockModelDefinitions: sinon.SinonStub; + let mockHelpers: { createModelInfo: sinon.SinonStub; isDefaultUserModel: sinon.SinonStub; markDefaultModel: sinon.SinonStub }; + + setup(() => { + // Mock getAllModelDefinitions + mockModelDefinitions = sinon.stub(modelDefinitionsModule, 'getAllModelDefinitions'); + + // Mock helper functions + mockHelpers = { + createModelInfo: sinon.stub(helpersModule, 'createModelInfo'), + isDefaultUserModel: sinon.stub(helpersModule, 'isDefaultUserModel'), + markDefaultModel: sinon.stub(helpersModule, 'markDefaultModel') + }; + }); + + teardown(() => { + // Restore specific stubs for this suite + mockModelDefinitions.restore(); + mockHelpers.createModelInfo.restore(); + mockHelpers.isDefaultUserModel.restore(); + mockHelpers.markDefaultModel.restore(); + }); + + suite('retrieveModelsFromConfig', () => { + test('returns undefined when no configured models', () => { + mockModelDefinitions.returns([]); + const result = (openAIModel as any).retrieveModelsFromConfig(); + assert.strictEqual(result, undefined); + }); + + test('returns configured models when built-in models exist', () => { + const builtInModels = [ + { + name: 'GPT-4', + identifier: 'gpt-4', + maxInputTokens: 8192, + maxOutputTokens: 4096 + } + ]; + mockModelDefinitions.returns(builtInModels); + + const mockModelInfo = { + id: 'gpt-4', + name: 'GPT-4', + family: 'openai-api', + version: 'gpt-4', + maxInputTokens: 8192, + maxOutputTokens: 4096, + capabilities: { vision: true, toolCalling: true, agentMode: true }, + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + const result = (openAIModel as any).retrieveModelsFromConfig(); + + assert.ok(result, 'Should return built-in models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'gpt-4'); + }); + + test('marks one model as default', () => { + const configuredModels = [ + { name: 'GPT-4', identifier: 'gpt-4' }, + { name: 'GPT-3.5 Turbo', identifier: 'gpt-3.5-turbo' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfoA = { id: 'gpt-4', name: 'GPT-4', isDefault: false, isUserSelectable: true }; + const mockModelInfoB = { id: 'gpt-3.5-turbo', name: 'GPT-3.5 Turbo', isDefault: false, isUserSelectable: true }; + mockHelpers.createModelInfo.onFirstCall().returns(mockModelInfoA); + mockHelpers.createModelInfo.onSecondCall().returns(mockModelInfoB); + + // Mock markDefaultModel to return the expected result with gpt-3.5-turbo as default + const expectedResult = [ + { id: 'gpt-4', name: 'GPT-4', isDefault: false, isUserSelectable: true }, + { id: 'gpt-3.5-turbo', name: 'GPT-3.5 Turbo', isDefault: true, isUserSelectable: true } + ]; + mockHelpers.markDefaultModel.returns(expectedResult); + + const result = (openAIModel as any).retrieveModelsFromConfig(); + + assert.ok(result, 'Should return models'); + assert.strictEqual(result.length, 2); + // Only one should be marked as default + const defaultModels = result.filter((m: any) => m.isDefault); + assert.strictEqual(defaultModels.length, 1); + assert.strictEqual(defaultModels[0].id, 'gpt-3.5-turbo'); + }); + + test('falls back to default model match for default', () => { + const configuredModels = [ + { name: 'GPT-4', identifier: 'gpt-4' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { id: 'gpt-4', name: 'GPT-4', isDefault: false, isUserSelectable: true }; + mockHelpers.createModelInfo.returns(mockModelInfo); + + // Mock markDefaultModel to return the expected result with the model as default + const expectedResult = [ + { id: 'gpt-4', name: 'GPT-4', isDefault: true, isUserSelectable: true } + ]; + mockHelpers.markDefaultModel.returns(expectedResult); + + const result = (openAIModel as any).retrieveModelsFromConfig(); + + const defaultModel = result.find((m: any) => m.isDefault); + assert.ok(defaultModel, 'Should have a default model'); + assert.strictEqual(defaultModel.id, 'gpt-4'); + + // Verify markDefaultModel was called with the correct parameters + sinon.assert.calledWith(mockHelpers.markDefaultModel, [mockModelInfo], 'openai-api', 'gpt-4'); + }); + }); + + suite('resolveModels integration', () => { + let cancellationToken: vscode.CancellationToken; + let fetchStub: sinon.SinonStub; + + setup(() => { + const cancellationTokenSource = new vscode.CancellationTokenSource(); + cancellationToken = cancellationTokenSource.token; + fetchStub = sinon.stub(global, 'fetch'); + }); + + teardown(() => { + fetchStub.restore(); + }); + + test('prioritizes configured models over API', async () => { + const configuredModels = [ + { name: 'User GPT', identifier: 'user-gpt' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { + id: 'user-gpt', + name: 'User GPT', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + const result = await openAIModel.resolveModels(cancellationToken); + + assert.ok(result, 'Should return configured models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'user-gpt'); + assert.strictEqual((openAIModel as any).modelListing, result); + }); + + test('falls back to API when no configured models', async () => { + mockModelDefinitions.returns([]); // No configured models + + const apiResponse = { + data: [ + { id: 'gpt-api-model', object: 'model', created: 1640995200, owned_by: 'openai' } + ] + }; + fetchStub.resolves({ + ok: true, + json: () => Promise.resolve(apiResponse) + }); + + const mockModelInfo = { + id: 'gpt-api-model', + name: 'gpt-api-model', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + + const result = await openAIModel.resolveModels(cancellationToken); + + assert.ok(result, 'Should return API models'); + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].id, 'gpt-api-model'); + assert.strictEqual((openAIModel as any).modelListing, result); + }); + + test('returns undefined when both configured and API fail', async () => { + mockModelDefinitions.returns([]); // No configured models + fetchStub.rejects(new Error('API Error')); + + const result = await openAIModel.resolveModels(cancellationToken); + + assert.strictEqual(result, undefined); + }); + + test('caches resolved models in modelListing', async () => { + const configuredModels = [ + { name: 'Cached GPT', identifier: 'cached-gpt' } + ]; + mockModelDefinitions.returns(configuredModels); + + const mockModelInfo = { + id: 'cached-gpt', + name: 'Cached GPT', + isDefault: true, + isUserSelectable: true + }; + mockHelpers.createModelInfo.returns(mockModelInfo); + mockHelpers.markDefaultModel.returns([mockModelInfo]); + + await openAIModel.resolveModels(cancellationToken); + + assert.strictEqual((openAIModel as any).modelListing.length, 1); + assert.strictEqual((openAIModel as any).modelListing[0].id, 'cached-gpt'); + }); + }); + }); +}); diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index c478c7fdc077..1cef0948c115 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -528,34 +528,15 @@ export class LanguageModelsService implements ILanguageModelsService { this._isInitialSetup = false; } - // Listen for changes to the filterModels configuration. The initial filtering + // Listen for changes to model configuration. The initial filtering and configuration // is done in the Positron Assistant extension when models are resolved. this._store.add(this._configurationService.onDidChangeConfiguration(e => { if (e.affectsConfiguration('positron.assistant.filterModels')) { this._logService.trace('[LM] Filter models configuration changed, re-resolving language models'); - // Re-resolve all registered providers to apply new filters - const allVendors = Array.from(this._vendors.keys()); - if (allVendors.length === 0) { - return; - } - // Re-resolve all vendors - const vendorPromises = allVendors.map( - vendor => this._resolveLanguageModels(vendor, true)); - - // After all are resolved, check if the current provider is still valid - Promise.all(vendorPromises).then(() => { - // if the current provider is now filtered out, switch to another available provider - const currentProvider = this._currentProvider; - const availableProviders = this.getLanguageModelProviders(); - if (currentProvider && !availableProviders.some(p => p.id === currentProvider.id)) { - this._logService.trace('[LM] Current provider was filtered out, switching to next available', currentProvider.id); - if (availableProviders.length > 0) { - this.currentProvider = availableProviders[0]; - } else { - this.currentProvider = undefined; - } - } - }); + this._reResolveLanguageModels(); + } else if (e.affectsConfiguration('positron.assistant.configuredModels')) { + this._logService.trace('[LM] Configured models configuration changed, re-resolving language models'); + this._reResolveLanguageModels(); } })); // --- End Positron --- @@ -598,6 +579,32 @@ export class LanguageModelsService implements ILanguageModelsService { return Array.from(this._modelCache.keys()); } // --- Start Positron --- + private _reResolveLanguageModels(): void { + // Re-resolve all registered providers to apply new configuration + const allVendors = Array.from(this._vendors.keys()); + if (allVendors.length === 0) { + return; + } + // Re-resolve all vendors + const vendorPromises = allVendors.map( + vendor => this._resolveLanguageModels(vendor, true)); + + // After all are resolved, check if the current provider is still valid + Promise.all(vendorPromises).then(() => { + // if the current provider is no longer available, switch to another available provider + const currentProvider = this._currentProvider; + const availableProviders = this.getLanguageModelProviders(); + if (currentProvider && !availableProviders.some(p => p.id === currentProvider.id)) { + this._logService.trace('[LM] Current provider is no longer available, switching to next available', currentProvider.id); + if (availableProviders.length > 0) { + this.currentProvider = availableProviders[0]; + } else { + this.currentProvider = undefined; + } + } + }); + } + private getSelectedProviderStorageKey(): string { return `chat.currentLanguageProvider`; } From 82f1b301c89015007c25a8dd446284232a9f2cdd Mon Sep 17 00:00:00 2001 From: sharon Date: Wed, 19 Nov 2025 11:45:04 -0500 Subject: [PATCH 2/6] Remove deprecated Gemini 1.5 Flash 002 Co-authored-by: Melissa Barca <5323711+melissa-barca@users.noreply.github.com> Signed-off-by: sharon --- extensions/positron-assistant/src/modelDefinitions.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/extensions/positron-assistant/src/modelDefinitions.ts b/extensions/positron-assistant/src/modelDefinitions.ts index f6f9a2d187db..466d8b4a3a3d 100644 --- a/extensions/positron-assistant/src/modelDefinitions.ts +++ b/extensions/positron-assistant/src/modelDefinitions.ts @@ -57,11 +57,6 @@ const builtInModelDefinitions = new Map([ identifier: 'gemini-2.0-flash-exp', maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-2.0-flash }, - { - name: 'Gemini 1.5 Flash 002', - identifier: 'gemini-1.5-flash-002', - maxOutputTokens: 8_192, // reference: https://ai.google.dev/gemini-api/docs/models#gemini-1.5-flash - }, ]] ]); From 14f0c2c4131bb69f0e7fbee2a1a8316cb0fd3d17 Mon Sep 17 00:00:00 2001 From: sharon Date: Wed, 19 Nov 2025 11:48:17 -0500 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Melissa Barca <5323711+melissa-barca@users.noreply.github.com> Signed-off-by: sharon --- extensions/positron-assistant/src/modelResolutionHelpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-assistant/src/modelResolutionHelpers.ts b/extensions/positron-assistant/src/modelResolutionHelpers.ts index fab3a000d0fc..94d236b7cb10 100644 --- a/extensions/positron-assistant/src/modelResolutionHelpers.ts +++ b/extensions/positron-assistant/src/modelResolutionHelpers.ts @@ -37,7 +37,7 @@ export function getUserTokenLimits(): TokenLimits { * 1. User-configured default models for the provider * 2. Falls back to provider-specific default patterns * - * @param provider The provider ID (e.g., 'anthropic-api', 'openai-compatible) + * @param provider The provider ID (e.g., 'anthropic-api', 'openai-compatible') * @param id The model ID to check * @param name Optional model display name to check against * @param defaultMatch Optional fallback pattern to match against (provider-specific) From eabe6ef8a34f3aa44d91222e4bef570dc245472e Mon Sep 17 00:00:00 2001 From: sharon wang Date: Wed, 19 Nov 2025 11:47:11 -0500 Subject: [PATCH 4/6] update configuredModels minimum input and output tokens to 512 --- extensions/positron-assistant/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/positron-assistant/package.json b/extensions/positron-assistant/package.json index 2fa99f83188e..18351d72fc02 100644 --- a/extensions/positron-assistant/package.json +++ b/extensions/positron-assistant/package.json @@ -408,12 +408,12 @@ }, "maxInputTokens": { "type": "number", - "minimum": 1, + "minimum": 512, "description": "Maximum input tokens for this model" }, "maxOutputTokens": { "type": "number", - "minimum": 1, + "minimum": 512, "description": "Maximum output tokens for this model" } }, From 13de2680b662059e4210bfa925043039741de59b Mon Sep 17 00:00:00 2001 From: sharon Date: Wed, 19 Nov 2025 11:49:15 -0500 Subject: [PATCH 5/6] Remove confusing comment Co-authored-by: Melissa Barca <5323711+melissa-barca@users.noreply.github.com> Signed-off-by: sharon --- src/vs/workbench/contrib/chat/common/languageModels.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index 1cef0948c115..19fc8d5a439c 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -585,7 +585,6 @@ export class LanguageModelsService implements ILanguageModelsService { if (allVendors.length === 0) { return; } - // Re-resolve all vendors const vendorPromises = allVendors.map( vendor => this._resolveLanguageModels(vendor, true)); From 1dbf111319da5f11a91c06b0126a3f49c2547cb7 Mon Sep 17 00:00:00 2001 From: sharon wang Date: Thu, 20 Nov 2025 18:14:12 -0500 Subject: [PATCH 6/6] Implement provider verification for configured models and add corresponding tests --- .../positron-assistant/src/extension.ts | 4 +- .../src/modelDefinitions.ts | 24 +++++++++++ .../src/test/modelDefinitions.test.ts | 40 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/extensions/positron-assistant/src/extension.ts b/extensions/positron-assistant/src/extension.ts index 8b2b144566f3..513ea942035e 100644 --- a/extensions/positron-assistant/src/extension.ts +++ b/extensions/positron-assistant/src/extension.ts @@ -27,6 +27,7 @@ import { registerPromptManagement } from './promptRender.js'; import { collectDiagnostics } from './diagnostics.js'; import { BufferedLogOutputChannel } from './logBuffer.js'; import { resetAssistantState } from './reset.js'; +import { verifyProvidersInConfiguredModels } from './modelDefinitions.js'; const hasChatModelsContextKey = 'positron-assistant.hasChatModels'; @@ -437,7 +438,7 @@ export function getRequestTokenUsage(requestId: string): { tokens: TokenUsage; p return requestTokenUsage.get(requestId); } -export function activate(context: vscode.ExtensionContext) { +export async function activate(context: vscode.ExtensionContext) { // Create the log output channel. context.subscriptions.push(log); @@ -458,6 +459,7 @@ export function activate(context: vscode.ExtensionContext) { positron.ai.addLanguageModelConfig(expandConfigToSource(stored)); }); } + await verifyProvidersInConfiguredModels(); } catch (error) { const msg = error instanceof Error ? error.message : JSON.stringify(error); vscode.window.showErrorMessage( diff --git a/extensions/positron-assistant/src/modelDefinitions.ts b/extensions/positron-assistant/src/modelDefinitions.ts index 466d8b4a3a3d..f35f037ddfdd 100644 --- a/extensions/positron-assistant/src/modelDefinitions.ts +++ b/extensions/positron-assistant/src/modelDefinitions.ts @@ -4,6 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import * as vscode from 'vscode'; +import { getEnabledProviders } from './config.js'; +import { log } from './extension.js'; export interface ModelDefinition { name: string; @@ -21,6 +23,28 @@ export function getConfiguredModels(providerId: string): ModelDefinition[] { return configuredModels[providerId] || []; } +/** + * Check whether the provider IDs in the configured models are valid providers. + */ +export async function verifyProvidersInConfiguredModels() { + const config = vscode.workspace.getConfiguration('positron.assistant'); + const configuredModels = config.get>('configuredModels', {}); + const enabledProviders = await getEnabledProviders(); + + const invalidProviders = Object.keys(configuredModels).filter(providerId => !enabledProviders.includes(providerId)); + if (invalidProviders.length === 0) { + return; + } + + const message = vscode.l10n.t('Configured models contain unsupported providers: {0}. Please review your configuration for \'positron.assistant.configuredModels\'', invalidProviders.map(p => `'${p}'`).join(', ')); + log.warn(message); + const settingsAction = vscode.l10n.t('Open Settings'); + const selectedAction = await vscode.window.showWarningMessage(message, settingsAction); + if (selectedAction === settingsAction) { + await vscode.commands.executeCommand('workbench.action.openSettings', 'positron.assistant.configuredModels'); + } +} + /** * Built-in model definitions that serve as fallback defaults when no user configuration * is provided and dynamic model querying is not available or fails. diff --git a/extensions/positron-assistant/src/test/modelDefinitions.test.ts b/extensions/positron-assistant/src/test/modelDefinitions.test.ts index 1b2d9f87f2a6..b5b7bf4d7a68 100644 --- a/extensions/positron-assistant/src/test/modelDefinitions.test.ts +++ b/extensions/positron-assistant/src/test/modelDefinitions.test.ts @@ -50,5 +50,45 @@ suite('Model Definitions', () => { assert.deepStrictEqual(result, []); }); + test('show a warning if configured models include unsupported providers', async () => { + const userModels = { + 'unsupported-provider': [ + { + name: 'Some Model', + identifier: 'some-model' + } + ] + }; + mockWorkspaceConfig.withArgs('configuredModels', {}).returns(userModels); + + const showWarningMessageStub = sinon.stub(vscode.window, 'showWarningMessage').resolves(); + + // Call the function that verifies providers + const { verifyProvidersInConfiguredModels } = await import('../modelDefinitions.js'); + await verifyProvidersInConfiguredModels(); + + assert.strictEqual(showWarningMessageStub.calledOnce, true); + const warningMessage = showWarningMessageStub.getCall(0).args[0]; + assert.ok(warningMessage.includes('unsupported-provider')); + }); + + test('does not show a warning if all configured providers are supported', async () => { + const userModels = { + 'anthropic-api': [ + { + name: 'Claude Sonnet 4.5', + identifier: 'claude-sonnet-4-5' + } + ] + }; + mockWorkspaceConfig.withArgs('configuredModels', {}).returns(userModels); + const showWarningMessageStub = sinon.stub(vscode.window, 'showWarningMessage').resolves(); + + // Call the function that verifies providers + const { verifyProvidersInConfiguredModels } = await import('../modelDefinitions.js'); + await verifyProvidersInConfiguredModels(); + + assert.strictEqual(showWarningMessageStub.notCalled, true); + }); }); });