Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/types/src/global-settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ export const SECRET_STATE_KEYS = [
"awsSessionToken",
"openAiApiKey",
"ollamaApiKey",
"lmStudioApiKey",
"geminiApiKey",
"openAiNativeApiKey",
"deepSeekApiKey",
Expand Down
1 change: 1 addition & 0 deletions packages/types/src/provider-settings/lm-studio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export const lmStudioProviderDefinition = createProviderDefinition({
...baseProviderSettingsShape,
[LM_STUDIO_MODEL_ID_FIELD]: z.string().optional(),
lmStudioBaseUrl: z.string().optional(),
lmStudioApiKey: z.string().optional(),
lmStudioDraftModelId: z.string().optional(),
lmStudioSpeculativeDecodingEnabled: z.boolean().optional(),
},
Expand Down
86 changes: 62 additions & 24 deletions src/api/providers/fetchers/__tests__/lmstudio.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ describe("LMStudio Fetcher", () => {

await forceFullModelDetailsLoad(baseUrl, modelId)

expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: "wss://securehost:4321" })
expect(mockLoadModel).toHaveBeenCalledWith(modelId)
expect(mockFlushModels).toHaveBeenCalledWith({ provider: providerIdentifiers.lmstudio, baseUrl }, true)
Expand Down Expand Up @@ -148,7 +148,7 @@ describe("LMStudio Fetcher", () => {
const result = await getLMStudioModels(baseUrl)

expect(mockedAxios.get).toHaveBeenCalledTimes(1)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1)
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl })
expect(mockListDownloadedModels).toHaveBeenCalledTimes(1)
Expand All @@ -168,7 +168,7 @@ describe("LMStudio Fetcher", () => {
const result = await getLMStudioModels(baseUrl)

expect(mockedAxios.get).toHaveBeenCalledTimes(1)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1)
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl })
expect(mockListDownloadedModels).toHaveBeenCalledTimes(1)
Expand Down Expand Up @@ -408,7 +408,7 @@ describe("LMStudio Fetcher", () => {

await getLMStudioModels("")

expect(mockedAxios.get).toHaveBeenCalledWith(`${defaultBaseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${defaultBaseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: defaultLmsUrl })
})

Expand All @@ -420,7 +420,7 @@ describe("LMStudio Fetcher", () => {

await getLMStudioModels(httpsBaseUrl)

expect(mockedAxios.get).toHaveBeenCalledWith(`${httpsBaseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${httpsBaseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: wssLmsUrl })
})

Expand All @@ -434,51 +434,58 @@ describe("LMStudio Fetcher", () => {
expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled()
})

it("should return an empty object and log error if axios.get fails with a generic error", async () => {
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(function () {})
it("should throw (not silently return empty) if axios.get fails with a generic error", async () => {
const networkError = new Error("Network connection failed")
mockedAxios.get.mockRejectedValueOnce(networkError)

const result = await getLMStudioModels(baseUrl)
await expect(getLMStudioModels(baseUrl)).rejects.toThrow("Network connection failed")

expect(mockedAxios.get).toHaveBeenCalledTimes(1)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled()
expect(mockListLoaded).not.toHaveBeenCalled()
expect(consoleErrorSpy).toHaveBeenCalledWith(
`Error fetching LMStudio models: ${JSON.stringify(networkError, Object.getOwnPropertyNames(networkError), 2)}`,
)
expect(result).toEqual({})
consoleErrorSpy.mockRestore()
})

it("should return an empty object and log info if axios.get fails with ECONNREFUSED", async () => {
const consoleInfoSpy = vi.spyOn(console, "warn").mockImplementation(function () {})
const econnrefusedError = new Error("Connection refused")
;(econnrefusedError as any).code = "ECONNREFUSED"
it("should throw a connection-specific error if axios.get fails with ECONNREFUSED", async () => {
const econnrefusedError = Object.assign(new Error("Connection refused"), { code: "ECONNREFUSED" })
mockedAxios.get.mockRejectedValueOnce(econnrefusedError)

const result = await getLMStudioModels(baseUrl)
await expect(getLMStudioModels(baseUrl)).rejects.toThrow(
`Unable to connect to LM Studio at ${baseUrl}. Is LM Studio's local server running?`,
)

expect(mockedAxios.get).toHaveBeenCalledTimes(1)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`)
expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { headers: {} })
expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled()
expect(mockListLoaded).not.toHaveBeenCalled()
expect(consoleInfoSpy).toHaveBeenCalledWith(`Error connecting to LMStudio at ${baseUrl}`)
expect(result).toEqual({})
consoleInfoSpy.mockRestore()
})

it("should throw an auth-specific error if axios.get fails with a 401", async () => {
const unauthorizedError = Object.assign(new Error("Request failed with status code 401"), {
response: { status: 401 },
})
mockedAxios.get.mockRejectedValueOnce(unauthorizedError)

await expect(getLMStudioModels(baseUrl, "wrong-key")).rejects.toThrow(
"LM Studio rejected the request. Check that the API key is correct.",
)

expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled()
})

it("should return an empty object and log error if listDownloadedModels fails", async () => {
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(function () {})
const consoleWarnSpy = vi.spyOn(console, "warn").mockImplementation(function () {})
const listError = new Error("LMStudio SDK internal error")

mockedAxios.get.mockResolvedValueOnce({ data: {} })
mockListLoaded.mockRejectedValueOnce(listError)
// The REST fallback also has nothing to offer here.
mockedAxios.get.mockResolvedValueOnce({ data: { data: [] } })

const result = await getLMStudioModels(baseUrl)

expect(mockedAxios.get).toHaveBeenCalledTimes(1)
expect(mockedAxios.get).toHaveBeenCalledTimes(2)
expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1)
expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl })
expect(mockListLoaded).toHaveBeenCalledTimes(1)
Expand All @@ -487,6 +494,37 @@ describe("LMStudio Fetcher", () => {
)
expect(result).toEqual({})
consoleErrorSpy.mockRestore()
consoleWarnSpy.mockRestore()
})

it("should fall back to the REST API when the websocket SDK client can't authenticate", async () => {
// The initial connectivity check (with the API key) succeeds...
mockedAxios.get.mockResolvedValueOnce({ data: { data: [] } })
// ...but the websocket SDK client has no way to send the API key, so LM Studio
// rejects it and every SDK call comes back empty/failing.
mockListDownloadedModels.mockRejectedValueOnce(new Error("Unauthorized"))
mockListLoaded.mockRejectedValueOnce(new Error("Unauthorized"))
// The REST fallback (/api/v0/models), which does receive the API key, succeeds.
mockedAxios.get.mockResolvedValueOnce({
data: {
data: [
{ id: "qwen/qwen3-35b-a3b", type: "vlm", max_context_length: 262144, state: "loaded" },
{ id: "google/gemma-4-12b-qat", type: "vlm", max_context_length: 262144 },
{ id: "text-embedding-nomic-embed-text-v1.5", type: "embeddings", max_context_length: 2048 },
],
},
})

const result = await getLMStudioModels(baseUrl, "sk-lm-test-key")

expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/api/v0/models`, {
headers: { Authorization: "Bearer sk-lm-test-key" },
})
// Embedding models are excluded; only the two chat-capable models remain.
expect(Object.keys(result)).toEqual(["qwen/qwen3-35b-a3b", "google/gemma-4-12b-qat"])
expect(result["qwen/qwen3-35b-a3b"].contextWindow).toBe(262144)
expect(result["qwen/qwen3-35b-a3b"].supportsImages).toBe(true)
expect(hasLoadedFullDetails("qwen/qwen3-35b-a3b")).toBe(true)
})
})
})
108 changes: 98 additions & 10 deletions src/api/providers/fetchers/lmstudio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,18 @@ const modelsWithLoadedDetails = new Set<string>()

export const hasLoadedFullDetails = (modelId: string): boolean => modelsWithLoadedDetails.has(modelId)

export const forceFullModelDetailsLoad = async (baseUrl: string, modelId: string): Promise<void> => {
export const forceFullModelDetailsLoad = async (baseUrl: string, modelId: string, apiKey?: string): Promise<void> => {
try {
// Test the connection to LM Studio first
// Crrors will be caught further down.
await axios.get(`${baseUrl}/v1/models`)
const headers: Record<string, string> = apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
await axios.get(`${baseUrl}/v1/models`, { headers })
const lmsUrl = baseUrl.replace(/^http:\/\//, "ws://").replace(/^https:\/\//, "wss://")

const client = new LMStudioClient({ baseUrl: lmsUrl })
await client.llm.model(modelId)
// Flush and refresh cache to get updated model details
await flushModels({ provider: providerIdentifiers.lmstudio, baseUrl }, true)
await flushModels({ provider: providerIdentifiers.lmstudio, baseUrl, apiKey }, true)

// Mark this model as having full details loaded.
modelsWithLoadedDetails.add(modelId)
Expand Down Expand Up @@ -49,7 +50,72 @@ export const parseLMStudioModel = (rawModel: LLMInstanceInfo | LLMInfo): ModelIn
return modelInfo
}

export async function getLMStudioModels(baseUrl = "http://localhost:1234"): Promise<Record<string, ModelInfo>> {
// Shape of entries returned by LM Studio's REST API (GET /api/v0/models). Unlike the
// @lmstudio/sdk websocket client used below, this plain HTTP endpoint honors the
// Authorization header, so it works against remote/tunneled servers that require an API key.
interface LMStudioRestModel {
id: string
type?: "llm" | "vlm" | "embeddings"
publisher?: string
arch?: string
quantization?: string
state?: "loaded" | "not-loaded"
max_context_length?: number
loaded_context_length?: number
}

const parseLMStudioRestModel = (rawModel: LMStudioRestModel): ModelInfo => {
const contextLength = rawModel.loaded_context_length ?? rawModel.max_context_length

return Object.assign({}, lMStudioDefaultModelInfo, {
description:
[rawModel.publisher, rawModel.arch, rawModel.quantization].filter(Boolean).join(" - ") || rawModel.id,
contextWindow: contextLength ?? lMStudioDefaultModelInfo.contextWindow,
supportsPromptCache: true,
supportsImages: rawModel.type === "vlm",
maxTokens: contextLength ?? lMStudioDefaultModelInfo.maxTokens,
})
}

// Fetch models via LM Studio's plain REST API as a fallback for when the websocket SDK
// client returns nothing -- most commonly because it has no way to send the API key a
// remote/tunneled server requires (see getLMStudioModels below).
async function getModelsViaRestApi(
baseUrl: string,
headers: Record<string, string>,
): Promise<Record<string, ModelInfo>> {
const models: Record<string, ModelInfo> = {}

try {
const response = await axios.get(`${baseUrl}/api/v0/models`, { headers })
const data = response.data?.data

if (Array.isArray(data)) {
for (const rawModel of data as LMStudioRestModel[]) {
if (!rawModel?.id || rawModel.type === "embeddings") {
continue
}

models[rawModel.id] = parseLMStudioRestModel(rawModel)

if (rawModel.state === "loaded") {
modelsWithLoadedDetails.add(rawModel.id)
}
}
}
} catch (error) {
console.warn(
`[LMStudio] REST API fallback (/api/v0/models) failed: ${error instanceof Error ? error.message : String(error)}`,
)
}

return models
}

export async function getLMStudioModels(
baseUrl = "http://localhost:1234",
apiKey?: string,
): Promise<Record<string, ModelInfo>> {
// clear the set of models that have full details loaded
modelsWithLoadedDetails.clear()
// clearing the input can leave an empty string; use the default in that case
Expand All @@ -59,15 +125,28 @@ export async function getLMStudioModels(baseUrl = "http://localhost:1234"): Prom
// ws is required to connect using the LMStudio library
const lmsUrl = baseUrl.replace(/^http:\/\//, "ws://").replace(/^https:\/\//, "wss://")

if (!URL.canParse(lmsUrl)) {
return models
}

// Test the connection to LM Studio first. Unlike the best-effort model-detail
// lookups below, a failure here (wrong URL, server not running, bad API key) must
// propagate so callers can surface a real error instead of silently reporting an
// empty model list as if the refresh had succeeded.
const headers: Record<string, string> = apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
try {
if (!URL.canParse(lmsUrl)) {
return models
await axios.get(`${baseUrl}/v1/models`, { headers })
} catch (error) {
if (error.code === "ECONNREFUSED") {
throw new Error(`Unable to connect to LM Studio at ${baseUrl}. Is LM Studio's local server running?`)
}
if (error.response?.status === 401 || error.response?.status === 403) {
throw new Error(`LM Studio rejected the request. Check that the API key is correct.`)
}
throw error instanceof Error ? error : new Error(String(error))
}

// test the connection to LM Studio first
// errors will be caught further down
await axios.get(`${baseUrl}/v1/models`)

try {
const client = new LMStudioClient({ baseUrl: lmsUrl })

// First, try to get all downloaded models
Expand Down Expand Up @@ -125,5 +204,14 @@ export async function getLMStudioModels(baseUrl = "http://localhost:1234"): Prom
}
}

// The websocket SDK client above has no way to authenticate with an API key, so it
// silently returns nothing against remote/tunneled servers that require one even though
// the initial REST connectivity check (and thus the user's key) succeeded. Fall back to
// LM Studio's REST API, which honors the same Authorization header, before giving up.
if (Object.keys(models).length === 0) {
const restModels = await getModelsViaRestApi(baseUrl, headers)
Object.assign(models, restModels)
}

return models
}
4 changes: 2 additions & 2 deletions src/api/providers/fetchers/modelCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,9 @@ const KEY_SCOPED_PROVIDERS: ReadonlySet<RouterName> = new Set([
providerIdentifiers.zooGateway, // Per-session-token account identity
providerIdentifiers.kimiCode, // Per-session-token account identity
providerIdentifiers.nanogpt, // Public catalog can still vary by API-key allowlist
providerIdentifiers.lmstudio, // Remote/tunneled servers may gate models behind an API key
])

// Providers whose model lists are scoped to the signed-in user (e.g. per-account
// allowlists or org policies). For these we MUST NOT cache results on disk or
// in memory: a sign-in/out cycle could otherwise serve a previous user's model
// list to the next user, and stale data could mask backend allowlist updates.
Expand Down Expand Up @@ -244,7 +244,7 @@ async function fetchModelsFromProvider(options: GetModelsOptions): Promise<Model
models = await getOllamaModels(options.baseUrl, options.apiKey)
break
case providerIdentifiers.lmstudio:
models = await getLMStudioModels(options.baseUrl)
models = await getLMStudioModels(options.baseUrl, options.apiKey)
break
case providerIdentifiers.vercelAiGateway:
models = await getVercelAiGatewayModels()
Expand Down
5 changes: 3 additions & 2 deletions src/api/providers/lm-studio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
super()
this.options = options

// LM Studio uses "noop" as a placeholder API key
const apiKey = "noop"
// LM Studio uses "noop" as a placeholder API key when none is configured.
const apiKey = this.options.lmStudioApiKey || "noop"

this.client = new OpenAI({
baseURL: (this.options.lmStudioBaseUrl || "http://localhost:1234") + "/v1",
Expand Down Expand Up @@ -191,6 +191,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
const models = getModelsFromCache({
provider: providerIdentifiers.lmstudio,
baseUrl: this.options.lmStudioBaseUrl,
apiKey: this.options.lmStudioApiKey,
})
if (models && this.options.lmStudioModelId && models[this.options.lmStudioModelId]) {
return {
Expand Down
1 change: 1 addition & 0 deletions src/core/webview/ClineProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,7 @@ export class ClineProvider
await forceFullModelDetailsLoad(
cline.apiConfiguration.lmStudioBaseUrl ?? "http://localhost:1234",
cline.apiConfiguration.lmStudioModelId!,
cline.apiConfiguration.lmStudioApiKey,
)
}
} catch (error) {
Expand Down
2 changes: 1 addition & 1 deletion src/core/webview/__tests__/ClineProvider.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -574,7 +574,7 @@ describe("ClineProvider", () => {
},
} as Task)

expect(forceFullModelDetailsLoad).toHaveBeenCalledWith("http://localhost:1234", "test-model")
expect(forceFullModelDetailsLoad).toHaveBeenCalledWith("http://localhost:1234", "test-model", undefined)
})

test("does not reload full model details when the LM Studio model is already loaded", async () => {
Expand Down
Loading
Loading