Skip to content

Commit 0b3ce2f

Browse files
raptorsunraptorsun
committed
feat: llama stack inference provider config file section
Signed-off-by: Haoyu Sun <[email protected]>
1 parent b779177 commit 0b3ce2f

File tree

4 files changed

+274
-0
lines changed

4 files changed

+274
-0
lines changed

‎internal/controller/constants.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,24 @@ const (
112112
LLMProviderHashStateCacheKey = "llmprovider-hash"
113113
// AzureOpenAIType is the name of the Azure OpenAI provider type
114114
AzureOpenAIType = "azure_openai"
115+
// OpenAIType is the name of the OpenAI provider type
116+
OpenAIType = "openai"
117+
// WatsonXType is the name of the Watson X provider type
118+
WatsonXType = "watsonx"
119+
// LlamaStackOpenAIType is the name of the OpenAI provider type for the llama stack
120+
LlamaStackOpenAIType = "remote::openai"
121+
// LlamaStackAzureOpenAIType is the name of the Azure OpenAI provider type for the llama stack
122+
LlamaStackAzureOpenAIType = "remote::azure_openai"
123+
// LlamaStackWatsonXType is the name of the Watson X provider type for the llama stack
124+
LlamaStackWatsonXType = "remote::watsonx"
115125
// AdditionalCAHashStateCacheKey is the key of the hash value of the additional CA certificates in the state cache
116126
AdditionalCAHashStateCacheKey = "additionalca-hash"
117127
// DeploymentInProgress message
118128
DeploymentInProgress = "In Progress"
129+
// LlamaStackConfigCmName is the name of the llama stack configuration configmap
130+
LlamaStackConfigCmName = "llama-stack-config"
131+
// LlamaStackConfigFilename is the name of the llama stack configuration file
132+
LlamaStackConfigFilename = "llama-stack-config.yaml"
119133

120134
/*** console UI plugin ***/
121135
// ConsoleUIConfigMapName is the name of the console UI nginx configmap
@@ -289,4 +303,14 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
289303
MCPSECRETDATAPATH = "header"
290304
// LSCAppServerActivatorCmName is the name of the LSC app server activator configmap
291305
LSCAppServerActivatorCmName = "lsc-app-server-activator"
306+
307+
/*** llama stack configuration ***/
308+
// LlamaStackConfigVersion is the version of the llama stack configuration file, currently only version 2 is supported
309+
LlamaStackConfigVersion = "2"
310+
// OpenAIAPIKeyEnvVar is the environment variable for the OpenAI API key
311+
OpenAIAPIKeyEnvVar = "OPENAI_API_KEY"
312+
// AzureOpenAIAPIKeyEnvVar is the environment variable for the Azure OpenAI API key
313+
AzureOpenAIAPIKeyEnvVar = "AZURE_OPENAI_API_KEY"
314+
// WatsonXAPIKeyEnvVar is the environment variable for the Watson X API key
315+
WatsonXAPIKeyEnvVar = "WATSONX_API_KEY"
292316
)

‎internal/controller/llama_stack_types.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package controller
2+
3+
/*** Llama stack configuration ***/
4+
5+
// LlamaStackConfig is the root structure of the llama stack configuration file
6+
type LlamaStackConfig struct {
7+
// Version is the version of the llama stack configuration file, currently only version 2 is supported
8+
Version string `json:"version"`
9+
// Providers is the list of providers for the llama stack, it contains providers for each of the llama stack components
10+
// such as inference, datasetio, etc.
11+
Providers ProviderList `json:"providers"`
12+
}
13+
14+
// ProviderList is the list of providers for the llama stack, it contains providers for each of the llama stack components
15+
// such as inference, datasetio, etc.
16+
// here is an example of the providers section of the llama stack configuration file:
17+
/*
18+
providers:
19+
agents:
20+
- config:
21+
persistence_store:
22+
db_path: .llama/distributions/ollama/agents_store.db
23+
namespace: null
24+
type: sqlite
25+
responses_store:
26+
db_path: .llama/distributions/ollama/responses_store.db
27+
type: sqlite
28+
provider_id: meta-reference
29+
provider_type: inline::meta-reference
30+
datasetio:
31+
- config:
32+
kvstore:
33+
db_path: .llama/distributions/ollama/huggingface_datasetio.db
34+
namespace: null
35+
type: sqlite
36+
provider_id: huggingface
37+
provider_type: remote::huggingface
38+
- config:
39+
kvstore:
40+
db_path: .llama/distributions/ollama/localfs_datasetio.db
41+
namespace: null
42+
type: sqlite
43+
provider_id: localfs
44+
provider_type: inline::localfs
45+
inference:
46+
- provider_id: openai
47+
provider_type: remote::openai
48+
config:
49+
api_key: ${env.OPENAI_API_KEY}
50+
*/
51+
type ProviderList struct {
52+
Inference []InferenceProviderConfig `json:"inference"`
53+
54+
// todo: add other providers here
55+
}
56+
57+
type InferenceProviderConfig struct {
58+
// Provider ID is the unique identifier in the llama stack configuration
59+
ProviderID string `json:"provider_id"`
60+
// Provider Type is the type of the provider, this determines the underlyingtype of Config field below
61+
ProviderType string `json:"provider_type"`
62+
// Config is the provider specific configuration, can be one of the following:
63+
// - InferenceProviderOpenAI
64+
// - InferenceProviderAzureOpenAI
65+
// - InferenceProviderWatsonX
66+
Config interface{} `json:"config"`
67+
}
68+
69+
/*** Inference provider configuration ***/
70+
71+
// OpenAI inference provider configuration
72+
// https://llamastack.github.io/docs/providers/inference/remote_openai
73+
type InferenceProviderOpenAI struct {
74+
APIKey string `json:"api_key,omitempty"`
75+
BaseURL string `json:"base_url,omitempty"`
76+
}
77+
78+
// Azure OpenAI inference provider configuration
79+
// https://llamastack.github.io/docs/providers/inference/remote_azure_openai
80+
type InferenceProviderAzureOpenAI struct {
81+
APIKey string `json:"api_key,omitempty"`
82+
APIBase string `json:"api_base,omitempty"`
83+
APIVersion string `json:"api_version,omitempty"`
84+
APIType string `json:"api_type,omitempty"`
85+
}
86+
87+
// WatsonX inference provider configuration
88+
// https://llamastack.github.io/docs/providers/inference/remote_watsonx
89+
type InferenceProviderWatsonX struct {
90+
APIKey string `json:"api_key,omitempty"`
91+
URL string `json:"url,omitempty"`
92+
ProjectID string `json:"project_id,omitempty"`
93+
Timeout int `json:"timeout,omitempty"`
94+
}

‎internal/controller/lsc_app_server_assets.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ package controller
22

33
import (
44
"context"
5+
"fmt"
56

67
appsv1 "k8s.io/api/apps/v1"
78
corev1 "k8s.io/api/core/v1"
89
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"sigs.k8s.io/yaml"
911

1012
olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1"
1113
)
@@ -62,3 +64,53 @@ func (r *OLSConfigReconciler) updateLSCDeployment(ctx context.Context, existingD
6264

6365
return nil
6466
}
67+
68+
// todo: implement other fields for the llama stack configuration map
69+
// generateLlamaStackConfigMap generates the llama stack configuration map for the LSC app server
70+
func (r *OLSConfigReconciler) generateLlamaStackConfigFile(ctx context.Context, cr *olsv1alpha1.OLSConfig) (string, error) {
71+
llamaStackConfig := &LlamaStackConfig{
72+
Version: LlamaStackConfigVersion,
73+
}
74+
75+
// inference providers
76+
for _, provider := range cr.Spec.LLMConfig.Providers {
77+
78+
providerConfig := InferenceProviderConfig{
79+
ProviderID: provider.Name,
80+
}
81+
switch provider.Type {
82+
case AzureOpenAIType:
83+
providerConfig.ProviderType = LlamaStackAzureOpenAIType
84+
providerConfig.Config = &InferenceProviderAzureOpenAI{
85+
// todo: add these environment variables to the podspec in generateLSCDeployment
86+
APIKey: fmt.Sprintf("${env.%s}", AzureOpenAIAPIKeyEnvVar),
87+
APIBase: provider.URL,
88+
APIVersion: provider.APIVersion,
89+
APIType: "", // default api type is "azure" for Azure OpenAI
90+
}
91+
// warning: AzureDeploymentName is not supported by Llama stack yet
92+
case OpenAIType:
93+
providerConfig.ProviderType = LlamaStackOpenAIType
94+
providerConfig.Config = &InferenceProviderOpenAI{
95+
// todo: add these environment variables to the podspec in generateLSCDeployment
96+
APIKey: fmt.Sprintf("${env.%s}", OpenAIAPIKeyEnvVar),
97+
BaseURL: provider.URL,
98+
}
99+
case WatsonXType:
100+
providerConfig.ProviderType = LlamaStackWatsonXType
101+
providerConfig.Config = &InferenceProviderWatsonX{
102+
// todo: add these environment variables to the podspec in generateLSCDeployment
103+
APIKey: fmt.Sprintf("${env.%s}", WatsonXAPIKeyEnvVar),
104+
URL: provider.URL,
105+
ProjectID: provider.WatsonProjectID,
106+
}
107+
}
108+
llamaStackConfig.Providers.Inference = append(llamaStackConfig.Providers.Inference, providerConfig)
109+
}
110+
111+
llamaStackConfigBytes, err := yaml.Marshal(llamaStackConfig)
112+
if err != nil {
113+
return "", fmt.Errorf("failed to generate llama stack configuration file %w", err)
114+
}
115+
return string(llamaStackConfigBytes), nil
116+
}

‎internal/controller/lsc_app_server_assets_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,108 @@ var _ = Describe("LSC App server assets", Label("LSCBackend"), Ordered, func() {
7575
})
7676
})
7777

78+
Context("Llama stack config file generation", func() {
79+
addAzureOpenAIProvider := func(cr *olsv1alpha1.OLSConfig) *olsv1alpha1.OLSConfig {
80+
cr.Spec.LLMConfig.Providers = append(cr.Spec.LLMConfig.Providers, olsv1alpha1.ProviderSpec{
81+
Name: "testProviderAzureOpenAI",
82+
URL: testURL,
83+
Type: "azure_openai",
84+
APIVersion: "testAzureVersion",
85+
})
86+
return cr
87+
}
88+
89+
addOpenAIProvider := func(cr *olsv1alpha1.OLSConfig) *olsv1alpha1.OLSConfig {
90+
cr.Spec.LLMConfig.Providers = append(cr.Spec.LLMConfig.Providers, olsv1alpha1.ProviderSpec{
91+
Name: "testProviderOpenAI",
92+
URL: testURL,
93+
Type: "openai",
94+
})
95+
return cr
96+
}
97+
98+
addWatsonXProvider := func(cr *olsv1alpha1.OLSConfig) *olsv1alpha1.OLSConfig {
99+
cr.Spec.LLMConfig.Providers = append(cr.Spec.LLMConfig.Providers, olsv1alpha1.ProviderSpec{
100+
Name: "testProviderWatsonX",
101+
URL: testURL,
102+
Type: "watsonx",
103+
WatsonProjectID: "testProjectID",
104+
})
105+
return cr
106+
}
107+
108+
BeforeEach(func() {
109+
ctx = context.Background()
110+
rOptions = &OLSConfigReconcilerOptions{
111+
OpenShiftMajor: "123",
112+
OpenshiftMinor: "456",
113+
LightspeedServiceImage: "lightspeed-service:latest",
114+
OpenShiftMCPServerImage: "openshift-mcp-server:latest",
115+
Namespace: OLSNamespaceDefault,
116+
}
117+
r = &OLSConfigReconciler{
118+
Options: *rOptions,
119+
logger: logf.Log.WithName("olsconfig.reconciler"),
120+
Client: k8sClient,
121+
Scheme: k8sClient.Scheme(),
122+
stateCache: make(map[string]string),
123+
}
124+
cr = getEmptyOLSConfigCR()
125+
})
126+
127+
Describe("Inference Provider", func() {
128+
It("should generate a valid llama stack config file for OpenAI provider", func() {
129+
cr = addOpenAIProvider(cr)
130+
llamaStackConfigFile, err := r.generateLlamaStackConfigFile(ctx, cr)
131+
Expect(err).NotTo(HaveOccurred())
132+
ExpectedConfigFile := `providers:
133+
inference:
134+
- config:
135+
api_key: ${env.OPENAI_API_KEY}
136+
base_url: https://testURL
137+
provider_id: testProviderOpenAI
138+
provider_type: remote::openai
139+
version: "2"
140+
`
141+
Expect(llamaStackConfigFile).To(Equal(ExpectedConfigFile))
142+
})
143+
144+
It("should generate a valid llama stack config file for Azure OpenAI provider", func() {
145+
cr = addAzureOpenAIProvider(cr)
146+
llamaStackConfigFile, err := r.generateLlamaStackConfigFile(ctx, cr)
147+
Expect(err).NotTo(HaveOccurred())
148+
Expect(llamaStackConfigFile).NotTo(BeEmpty())
149+
ExpectedConfigFile := `providers:
150+
inference:
151+
- config:
152+
api_base: https://testURL
153+
api_key: ${env.AZURE_OPENAI_API_KEY}
154+
api_version: testAzureVersion
155+
provider_id: testProviderAzureOpenAI
156+
provider_type: remote::azure_openai
157+
version: "2"
158+
`
159+
Expect(llamaStackConfigFile).To(Equal(ExpectedConfigFile))
160+
})
161+
162+
It("should generate a valid llama stack config file for Watson X provider", func() {
163+
cr = addWatsonXProvider(cr)
164+
llamaStackConfigFile, err := r.generateLlamaStackConfigFile(ctx, cr)
165+
Expect(err).NotTo(HaveOccurred())
166+
ExpectedConfigFile := `providers:
167+
inference:
168+
- config:
169+
api_key: ${env.WATSONX_API_KEY}
170+
project_id: testProjectID
171+
url: https://testURL
172+
provider_id: testProviderWatsonX
173+
provider_type: remote::watsonx
174+
version: "2"
175+
`
176+
Expect(llamaStackConfigFile).To(Equal(ExpectedConfigFile))
177+
})
178+
})
179+
180+
// TODO: Add more tests cases for once implementation is complete
181+
})
78182
})

0 commit comments

Comments
 (0)