OLS-2102: llama stack config of LM providers

internal/controller/constants.go

@@ -28,6 +28,8 @@ const (
 	/*** application server configuration file ***/
 	// OLSConfigName is the name of the OLSConfig configmap
 	OLSConfigCmName = "olsconfig"
+	// AppServerConfigCmName is the name of the app server configmap
+	AppServerConfigCmName = "olsconfig"
 	// OLSCAConfigMap is the name of the OLS TLS ca certificate configmap
 	OLSCAConfigMap = "openshift-service-ca.crt"
 	// OLSNamespaceDefault is the default namespace for OLS
@@ -110,10 +112,30 @@ const (
 	LLMProviderHashStateCacheKey = "llmprovider-hash"
 	// AzureOpenAIType is the name of the Azure OpenAI provider type
 	AzureOpenAIType = "azure_openai"
+	// OpenAIType is the name of the OpenAI provider type
+	OpenAIType = "openai"
+	// WatsonXType is the name of the Watson X provider type
+	WatsonXType = "watsonx"
+	// RHELAIType is the name of the RHELAI provider type
+	RHELAIType = "rhelai_vllm"
+	// RHOAIType is the name of the RHOAI provider type
+	RHOAIType = "rhoai_vllm"
+	// LlamaStackOpenAIType is the name of the OpenAI provider type for the llama stack
+	LlamaStackOpenAIType = "remote::openai"
+	// LlamaStackAzureOpenAIType is the name of the Azure OpenAI provider type for the llama stack
+	LlamaStackAzureOpenAIType = "remote::azure_openai"
+	// LlamaStackWatsonXType is the name of the Watson X provider type for the llama stack
+	LlamaStackWatsonXType = "remote::watsonx"
+	// LlamaStackVLLMType is the name of the VLLM provider type for the llama stack
+	LlamaStackVLLMType = "remote::vllm"
 	// AdditionalCAHashStateCacheKey is the key of the hash value of the additional CA certificates in the state cache
 	AdditionalCAHashStateCacheKey = "additionalca-hash"
 	// DeploymentInProgress message
 	DeploymentInProgress = "In Progress"
+	// LlamaStackConfigCmName is the name of the llama stack configuration configmap
+	LlamaStackConfigCmName = "llama-stack-config"
+	// LlamaStackConfigFilename is the name of the llama stack configuration file
+	LlamaStackConfigFilename = "llama-stack-config.yaml"
 
 	/*** console UI plugin ***/
 	// ConsoleUIConfigMapName is the name of the console UI nginx configmap
@@ -285,4 +307,20 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
 	MCPHeadersMountRoot = "/etc/mcp/headers"
 	// Header Secret Data Path
 	MCPSECRETDATAPATH = "header"
+	// LSCAppServerActivatorCmName is the name of the LSC app server activator configmap
+	LSCAppServerActivatorCmName = "lsc-app-server-activator"
+
+	/*** llama stack configuration ***/
+	// LlamaStackConfigVersion is the version of the llama stack configuration file, currently only version 2 is supported
+	LlamaStackConfigVersion = "2"
+	// OpenAIAPIKeyEnvVar is the environment variable for the OpenAI API key
+	OpenAIAPIKeyEnvVar = "OPENAI_API_KEY" // #nosec G101
+	// AzureOpenAIAPIKeyEnvVar is the environment variable for the Azure OpenAI API key
+	AzureOpenAIAPIKeyEnvVar = "AZURE_OPENAI_API_KEY" // #nosec G101
+	// WatsonXAPIKeyEnvVar is the environment variable for the Watson X API key
+	WatsonXAPIKeyEnvVar = "WATSONX_API_KEY" // #nosec G101
+	// RHELAIAPITokenEnvVar is the environment variable for the RHELAI API token
+	RHELAIAPITokenEnvVar = "RHELAI_API_TOKEN" // #nosec G101
+	// RHOAIAPITokenEnvVar is the environment variable for the RHOAI API token
+	RHOAIAPITokenEnvVar = "RHOAI_API_TOKEN" // #nosec G101
 )

internal/controller/errors.go

@@ -68,6 +68,7 @@ const (
 	ErrGetConsolePluginDeployment         = "failed to get Console Plugin deployment"
 	ErrGetConsolePluginNetworkPolicy      = "failed to get Console Plugin network policy"
 	ErrGetConsolePluginService            = "failed to get Console Plugin service"
+	ErrGetLSCActivatorConfigmap           = "failed to get LSC backend activator configmap"
 	ErrGetLLMSecret                       = "failed to get LLM provider secret" // #nosec G101
 	ErrGetOperatorNetworkPolicy           = "failed to get operator network policy"
 	ErrGetPostgresNetworkPolicy           = "failed to get OLS Postgres network policy"

internal/controller/llama_stack_types.go

@@ -0,0 +1,104 @@
+package controller
+
+/*** Llama stack configuration ***/
+
+// LlamaStackConfig is the root structure of the llama stack configuration file
+type LlamaStackConfig struct {
+	// Version is the version of the llama stack configuration file, currently only version 2 is supported
+	Version string `json:"version"`
+	// Providers is the list of providers for the llama stack, it contains providers for each of the llama stack components
+	// such as inference, datasetio, etc.
+	Providers ProviderList `json:"providers"`
+}
+
+// ProviderList is the list of providers for the llama stack, it contains providers for each of the llama stack components
+// such as inference, datasetio, etc.
+// here is an example of the providers section of the llama stack configuration file:
+/*
+   providers:
+     agents:
+     - config:
+         persistence_store:
+           db_path: .llama/distributions/ollama/agents_store.db
+           namespace: null
+           type: sqlite
+         responses_store:
+           db_path: .llama/distributions/ollama/responses_store.db
+           type: sqlite
+       provider_id: meta-reference
+       provider_type: inline::meta-reference
+     datasetio:
+     - config:
+         kvstore:
+           db_path: .llama/distributions/ollama/huggingface_datasetio.db
+           namespace: null
+           type: sqlite
+       provider_id: huggingface
+       provider_type: remote::huggingface
+     - config:
+         kvstore:
+           db_path: .llama/distributions/ollama/localfs_datasetio.db
+           namespace: null
+           type: sqlite
+       provider_id: localfs
+       provider_type: inline::localfs
+     inference:
+       - provider_id: openai
+         provider_type: remote::openai
+         config:
+           api_key: ${env.OPENAI_API_KEY}
+*/
+type ProviderList struct {
+	Inference []InferenceProviderConfig `json:"inference"`
+
+	// todo: add other providers here
+}
+
+type InferenceProviderConfig struct {
+	// Provider ID is the unique identifier in the llama stack configuration
+	ProviderID string `json:"provider_id"`
+	// Provider Type is the type of the provider, this determines the underlyingtype of Config field below
+	ProviderType string `json:"provider_type"`
+	// Config is the provider specific configuration, can be one of the following:
+	// - InferenceProviderOpenAI
+	// - InferenceProviderAzureOpenAI
+	// - InferenceProviderWatsonX
+	Config interface{} `json:"config"`
+}
+
+/*** Inference provider configuration ***/
+
+// OpenAI inference provider configuration
+// https://llamastack.github.io/docs/providers/inference/remote_openai
+type InferenceProviderOpenAI struct {
+	APIKey  string `json:"api_key,omitempty"`
+	BaseURL string `json:"base_url,omitempty"`
+}
+
+// Azure OpenAI inference provider configuration
+// https://llamastack.github.io/docs/providers/inference/remote_azure_openai
+type InferenceProviderAzureOpenAI struct {
+	APIKey     string `json:"api_key,omitempty"`
+	APIBase    string `json:"api_base,omitempty"`
+	APIVersion string `json:"api_version,omitempty"`
+	APIType    string `json:"api_type,omitempty"`
+}
+
+// WatsonX inference provider configuration
+// https://llamastack.github.io/docs/providers/inference/remote_watsonx
+type InferenceProviderWatsonX struct {
+	APIKey    string `json:"api_key,omitempty"`
+	URL       string `json:"url,omitempty"`
+	ProjectID string `json:"project_id,omitempty"`
+	Timeout   int    `json:"timeout,omitempty"`
+}
+
+// RHELAI inference provider configuration
+// https://llamastack.github.io/docs/providers/inference/remote_vllm
+type InferenceProviderVLLM struct {
+	APIToken      string `json:"api_token,omitempty"`
+	URL           string `json:"url,omitempty"`
+	MaxTokens     int    `json:"max_tokens,omitempty"`
+	TLSVerify     bool   `json:"tls_verify,omitempty"`
+	RefreshModels bool   `json:"refresh_models,omitempty"`
+}

internal/controller/lsc_app_server_assets.go

@@ -0,0 +1,140 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1"
+)
+
+// todo: implement LSC config map generation
+//
+//nolint:unused // Ignore unused lint error before implementation of reconciliation functions
+func (r *OLSConfigReconciler) generateLSCConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) (*corev1.ConfigMap, error) { //lint:ignore U1000 Ignore unused lint error before implementation of reconciliation functions
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      AppServerConfigCmName,
+			Namespace: r.Options.Namespace,
+		},
+	}
+	return configMap, nil
+}
+
+// todo: implement LSC deployment generation
+//
+//nolint:unused // Ignore unused lint error before implementation of reconciliation functions
+func (r *OLSConfigReconciler) generateLSCDeployment(ctx context.Context, cr *olsv1alpha1.OLSConfig) (*appsv1.Deployment, error) { //lint:ignore U1000 Ignore unused lint error before implementation of reconciliation functions
+
+	// todo: mount LLM token secret as the environment variable for the llama stack configuration file
+	// refer to generateLlamaStackConfigFile for the environment variable names and its corresponding provider type
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      OLSAppServerDeploymentName,
+			Namespace: r.Options.Namespace,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: cr.Spec.OLSConfig.DeploymentConfig.Replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: generateAppServerSelectorLabels(),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: generateAppServerSelectorLabels(),
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "lsc-app-server",
+							Image: r.Options.LightspeedServiceImage,
+						},
+					},
+				},
+			},
+		},
+	}
+	return deployment, nil
+}
+
+// todo: implement LSC deployment update
+//
+//nolint:unused // Ignore unused lint error before implementation of reconciliation functions
+func (r *OLSConfigReconciler) updateLSCDeployment(ctx context.Context, existingDeployment, desiredDeployment *appsv1.Deployment) error {
+
+	return nil
+}
+
+// todo: implement other fields for the llama stack configuration map
+// generateLlamaStackConfigMap generates the llama stack configuration map for the LSC app server
+//
+//nolint:unused // Ignore unused lint error before implementation of reconciliation functions
+func (r *OLSConfigReconciler) generateLlamaStackConfigFile(ctx context.Context, cr *olsv1alpha1.OLSConfig) (string, error) {
+	llamaStackConfig := &LlamaStackConfig{
+		Version: LlamaStackConfigVersion,
+	}
+
+	// inference providers
+	for _, provider := range cr.Spec.LLMConfig.Providers {
+
+		providerConfig := InferenceProviderConfig{
+			ProviderID: provider.Name,
+		}
+		switch provider.Type {
+		case AzureOpenAIType:
+			providerConfig.ProviderType = LlamaStackAzureOpenAIType
+			providerConfig.Config = &InferenceProviderAzureOpenAI{
+				// todo: add these environment variables to the podspec in generateLSCDeployment
+				APIKey:     fmt.Sprintf("${env.%s}", AzureOpenAIAPIKeyEnvVar),
+				APIBase:    provider.URL,
+				APIVersion: provider.APIVersion,
+				APIType:    "", // default api type is "azure" for Azure OpenAI
+			}
+			// warning: AzureDeploymentName is not supported by Llama stack yet
+		case OpenAIType:
+			providerConfig.ProviderType = LlamaStackOpenAIType
+			providerConfig.Config = &InferenceProviderOpenAI{
+				// todo: add these environment variables to the podspec in generateLSCDeployment
+				APIKey:  fmt.Sprintf("${env.%s}", OpenAIAPIKeyEnvVar),
+				BaseURL: provider.URL,
+			}
+		case WatsonXType:
+			providerConfig.ProviderType = LlamaStackWatsonXType
+			providerConfig.Config = &InferenceProviderWatsonX{
+				// todo: add these environment variables to the podspec in generateLSCDeployment
+				APIKey:    fmt.Sprintf("${env.%s}", WatsonXAPIKeyEnvVar),
+				URL:       provider.URL,
+				ProjectID: provider.WatsonProjectID,
+			}
+		case RHELAIType:
+
+			providerConfig.ProviderType = LlamaStackVLLMType
+			providerConfig.Config = &InferenceProviderVLLM{
+				// todo: add these environment variables to the podspec in generateLSCDeployment
+				APIToken: fmt.Sprintf("${env.%s}", RHELAIAPITokenEnvVar),
+				URL:      provider.URL,
+				// to be noted: max_tokens, tls_verify, refresh_models are available for both RHELAI and RHOAI but not specified in OLSConfig CR
+			}
+		case RHOAIType:
+			providerConfig.ProviderType = LlamaStackVLLMType
+			providerConfig.Config = &InferenceProviderVLLM{
+				// todo: add these environment variables to the podspec in generateLSCDeployment
+				APIToken: fmt.Sprintf("${env.%s}", RHOAIAPITokenEnvVar),
+				URL:      provider.URL,
+				// to be noted: max_tokens, tls_verify, refresh_models are available for both RHELAI and RHOAI but not specified in OLSConfig CR
+			}
+		default:
+			return "", fmt.Errorf("unsupported provider type: %s", provider.Type)
+		}
+		llamaStackConfig.Providers.Inference = append(llamaStackConfig.Providers.Inference, providerConfig)
+	}
+
+	llamaStackConfigBytes, err := yaml.Marshal(llamaStackConfig)
+	if err != nil {
+		return "", fmt.Errorf("failed to generate llama stack configuration file %w", err)
+	}
+	return string(llamaStackConfigBytes), nil
+}