Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ db_mode=json
########################################
# LLM Core Settings
########################################
LLM_PROVIDER=gemini # gemini | openai | claude | grok | ollama | openrouter | requesty | minimax | apiroute
LLM_PROVIDER=gemini # gemini | openai | claude | grok | ollama | openrouter | requesty | cheaperinference | minimax | apiroute
EMB_PROVIDER=openai # openai | gemini | ollama etc.
LLM_TEMP=1
LLM_MAXTOK=16384
Expand Down Expand Up @@ -65,6 +65,15 @@ REQUESTY_EMBED_MODEL=openai/text-embedding-3-large
# EU region: https://router.eu.requesty.ai/v1
REQUESTY_BASE_URL="https://router.requesty.ai/v1"

########################################
# Cheaper Inference (https://cheaperinference.com)
########################################
# Get a key at https://cheaperinference.com/signup
# Embeddings use the OpenAI settings above
CHEAPER_INFERENCE_API_KEY=
CHEAPER_INFERENCE_MODEL=gpt-5.4-mini
CHEAPER_INFERENCE_BASE_URL="https://api.cheaperinference.com/v1"

########################################
# Grok (xAI)
########################################
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ The platform provides a modern interface for students, educators, and researcher

### Supported AI Models

- Google Gemini β€’ OpenAI GPT β€’ Anthropic Claude β€’ xAI Grok β€’ [MiniMax](https://www.minimax.io/) β€’ [API Route](https://www.api-route.com) β€’ Ollama (local) β€’ OpenRouter β€’ [Requesty](https://requesty.ai)
- Google Gemini β€’ OpenAI GPT β€’ Anthropic Claude β€’ xAI Grok β€’ [MiniMax](https://www.minimax.io/) β€’ [API Route](https://www.api-route.com) β€’ Ollama (local) β€’ OpenRouter β€’ [Requesty](https://requesty.ai) β€’ [Cheaper Inference](https://cheaperinference.com)

### Embedding Providers

Expand Down
3 changes: 3 additions & 0 deletions backend/src/config/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ export const config = {
requesty_model: process.env.REQUESTY_MODEL || 'openai/gpt-4o-mini',
requesty_embed_model: process.env.REQUESTY_EMBED_MODEL || 'openai/text-embedding-3-large',
requesty_base: process.env.REQUESTY_BASE_URL || 'https://router.requesty.ai/v1',
cheaperinference: process.env.CHEAPER_INFERENCE_API_KEY || '',
cheaperinference_model: process.env.CHEAPER_INFERENCE_MODEL || 'gpt-5.4-mini',
cheaperinference_base: process.env.CHEAPER_INFERENCE_BASE_URL || 'https://api.cheaperinference.com/v1',
gemini: process.env.gemini || process.env.GOOGLE_API_KEY || '',
gemini_model: process.env.gemini_model || 'gemini-1.5-pro',
gemini_embed_model: process.env.gemini_embed_model || 'text-embedding-004',
Expand Down
63 changes: 63 additions & 0 deletions backend/src/utils/llm/models/__tests__/cheaperinference.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'

vi.mock('@langchain/openai', () => {
const MockChatOpenAI = vi.fn(function (this: any, opts: any) {
this.invoke = vi.fn().mockResolvedValue({ content: 'mock response' })
this._opts = opts
})
const MockOpenAIEmbeddings = vi.fn(function (this: any, opts: any) {
this.embedDocuments = vi.fn().mockResolvedValue([[0.1, 0.2]])
this.embedQuery = vi.fn().mockResolvedValue([0.1, 0.2])
this._opts = opts
})
return { ChatOpenAI: MockChatOpenAI, OpenAIEmbeddings: MockOpenAIEmbeddings }
})

import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'
import { makeLLM, makeEmbeddings } from '../cheaperinference'

describe('Cheaper Inference LLM provider', () => {
beforeEach(() => {
vi.clearAllMocks()
})

it('should create a ChatOpenAI instance with Cheaper Inference defaults', () => {
const llm = makeLLM({ cheaperinference: 'test-key' })

expect(ChatOpenAI).toHaveBeenCalledWith(
expect.objectContaining({
model: 'gpt-5.4-mini',
apiKey: 'test-key',
configuration: { baseURL: 'https://api.cheaperinference.com/v1' },
}),
)
expect(typeof llm.invoke).toBe('function')
expect(typeof llm.call).toBe('function')
})

it('should use configured model and base URL', () => {
makeLLM({
cheaperinference: 'key',
cheaperinference_model: 'claude-sonnet-5',
cheaperinference_base: 'https://gateway.example.test/v1',
})

expect(ChatOpenAI).toHaveBeenCalledWith(
expect.objectContaining({
model: 'claude-sonnet-5',
configuration: { baseURL: 'https://gateway.example.test/v1' },
}),
)
})

it('should create embeddings with the OpenAI settings', () => {
makeEmbeddings({ openai: 'openai-key' })

expect(OpenAIEmbeddings).toHaveBeenCalledWith(
expect.objectContaining({
model: 'text-embedding-3-large',
apiKey: 'openai-key',
}),
)
})
})
21 changes: 21 additions & 0 deletions backend/src/utils/llm/models/cheaperinference.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'
import { wrapChat } from './util'
import type { MkLLM, MkEmb, EmbeddingsLike } from './types'

export const makeLLM: MkLLM = (cfg: any) => {
const m = new ChatOpenAI({
model: cfg.cheaperinference_model || 'gpt-5.4-mini',
apiKey: cfg.cheaperinference || '',
configuration: { baseURL: cfg.cheaperinference_base || 'https://api.cheaperinference.com/v1' },
temperature: cfg.temp ?? 0.7,
maxTokens: cfg.max_tokens,
})
return wrapChat(m)
}

export const makeEmbeddings: MkEmb = (cfg: any): EmbeddingsLike => {
return new OpenAIEmbeddings({
model: cfg.openai_embed_model || 'text-embedding-3-large',
apiKey: cfg.openai || process.env.OPENAI_API_KEY,
})
}
2 changes: 2 additions & 0 deletions backend/src/utils/llm/models/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import * as grok from './grok'
import * as claude from './claude'
import * as openrouter from './openrouter'
import * as requesty from './requesty'
import * as cheaperinference from './cheaperinference'
import * as minimax from './minimax'
import * as apiroute from './apiroute'
import { config } from '../../../config/env'
Expand All @@ -21,6 +22,7 @@ function pick(p: string) {
case 'claude': return claude
case 'openrouter': return openrouter
case 'requesty': return requesty
case 'cheaperinference': return cheaperinference
case 'minimax': return minimax
case 'apiroute':
case 'api_route':
Expand Down