From 9bd7d17ff6633b3f2b68a468d4a91d3e1dc33877 Mon Sep 17 00:00:00 2001 From: nold Date: Fri, 11 Sep 2026 18:02:08 +0200 Subject: [PATCH] [model-config] feat: add environment variables support for backends (#10721) * feat: add environment variables support for backends in model configurations - Add field to model configuration to pass environment variables to backend processes - Update backend options and model configuration handling - Add documentation for environment variables configuration with examples including CUDA_VISIBLE_DEVICES Assisted-by: qwen-agentworld-35b-a3b Signed-off-by: nold * fix(test): Test environment variables configuration parsing from YAML Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: nold --------- Signed-off-by: nold Signed-off-by: nold Co-authored-by: nold Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/backend.proto | 3 +++ core/backend/options.go | 12 ++++++++++++ core/config/meta/registry.go | 7 +++++++ core/config/model_config.go | 3 +++ core/config/model_config_test.go | 6 ++++++ core/services/worker/supervisor.go | 2 +- docs/content/advanced/model-configuration.md | 18 ++++++++++++++++++ pkg/model/initializers.go | 2 +- pkg/model/loader_options.go | 9 +++++++++ pkg/model/process.go | 13 ++++++++++--- pkg/model/process_exit_test.go | 2 +- 11 files changed, 71 insertions(+), 6 deletions(-) diff --git a/backend/backend.proto b/backend/backend.proto index 7be3d5a4c..c658fb53c 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -500,6 +500,9 @@ message ModelOptions { // Unknown keys produce an error at LoadModel time. string EngineArgs = 73; + // EnvVars carries environment variables to be passed to the backend process. + map EnvVars = 76; + // Proxy carries the cloud-proxy backend's per-model configuration. // Empty for non-proxy backends. ProxyOptions Proxy = 74; diff --git a/core/backend/options.go b/core/backend/options.go index 93a6eadc0..815328378 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -244,6 +244,10 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo defOpts = append(defOpts, model.WithModelSizeBytes(sizeBytes)) } + if c.Environment != nil && len(c.Environment) > 0 { + defOpts = append(defOpts, model.WithEnvVars(c.Environment)) + } + return append(defOpts, opts...) } @@ -525,6 +529,14 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions { opts.DraftModel = filepath.Join(modelPath, c.DraftModel) } + // Add environment variables from model configuration + if c.Environment != nil && len(c.Environment) > 0 { + opts.EnvVars = make(map[string]string) + for k, v := range c.Environment { + opts.EnvVars[k] = v + } + } + return opts } diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 44447a6d6..c3914994b 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -93,6 +93,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Advanced: true, Order: 9, }, + "env": { + Section: "general", + Label: "Environment Variables", + Description: "Environment variables to be applied to the backend process", + Component: "map-editor", + Order: 10, + }, // --- LLM --- "context_size": { diff --git a/core/config/model_config.go b/core/config/model_config.go index ecc234d8c..ce8a4d1bf 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -160,6 +160,9 @@ type ModelConfig struct { Proxy ProxyConfig `yaml:"proxy,omitempty" json:"proxy,omitempty"` MITM MITMModelConfig `yaml:"mitm,omitempty" json:"mitm,omitempty"` Limits LimitsConfig `yaml:"limits,omitempty" json:"limits,omitempty"` + + // Environment variables to set when starting the backend process + Environment map[string]string `yaml:"env,omitempty" json:"env,omitempty"` } // CompressionConfig controls opt-in compression of chat history before inference. diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index 0abc0c7b6..828160fe1 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -199,6 +199,12 @@ parameters: Expect(valid).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) + // Test environment variables configuration parsing from YAML + envYAML := "name: env-test-model\nbackend: test-backend\nenv:\n TEST_ENV_VAR: test_value\n ANOTHER_VAR: another_value\n" + var envConfig ModelConfig + Expect(yaml.Unmarshal([]byte(envYAML), &envConfig)).To(Succeed()) + Expect(envConfig.Environment).To(HaveKeyWithValue("TEST_ENV_VAR", "test_value")) + Expect(envConfig.Environment).To(HaveKeyWithValue("ANOTHER_VAR", "another_value")) tcAndChat := FLAG_TOKEN_CLASSIFY | FLAG_CHAT tcCombined := ModelConfig{ Name: "ner-and-chat", diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 1d9b52365..7f8228b81 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -465,7 +465,7 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin bindAddr := fmt.Sprintf("0.0.0.0:%d", port) clientAddr := fmt.Sprintf("127.0.0.1:%d", port) - proc, err := s.ml.StartProcess(backendPath, backend, bindAddr) + proc, err := s.ml.StartProcess(backendPath, backend, bindAddr, nil) if err != nil { s.releasePortForKey(backend, port) s.mu.Unlock() diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 44f388bdd..bb82c531c 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -1059,6 +1059,24 @@ Multiple detectors union their detections; overlapping spans resolve to the stro > The earlier regex pattern tier (`pii.patterns`, the global pattern catalogue, `--pii-config`, and the `/api/pii/patterns` admin endpoints) has been removed, along with response/streaming-side redaction. Those keys now no-op with a startup warning; migrate to `pii.detectors` + a detector's `pii_detection` block. +## Environment Variables Configuration + +Model configurations can specify environment variables passed to the backend process: + +```yaml +name: vllm-model +backend: vllm +parameters: + model: my-vllm-model + +env: + VLLM_WORKER_MULTIPROC_METHOD: "spawn" + VLLM_CACHE_DIR: "/tmp/vllm_cache" + CUDA_VISIBLE_DEVICES: "0,1" +``` + +Environment variables are appended to the system environment variables and will override any conflicting system variables with the same name. + ## Complete Example Here's a comprehensive example combining many options: diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 49276abc3..cbfb7840c 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -133,7 +133,7 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID, return nil, fmt.Errorf("failed allocating free ports: %s", err.Error()) } // Make sure the process is executable - process, err := ml.startProcess(uri, modelID, serverAddress) + process, err := ml.StartProcess(uri, modelID, serverAddress, o.envVars) if err != nil { xlog.Error("failed to launch", "error", err, "path", uri) return nil, err diff --git a/pkg/model/loader_options.go b/pkg/model/loader_options.go index b6db0a47d..8c8bfffad 100644 --- a/pkg/model/loader_options.go +++ b/pkg/model/loader_options.go @@ -26,6 +26,9 @@ type Options struct { // by the caller using the vram estimation scaffolding. When non-zero it is // registered with the watchdog so size-aware eviction can rank models. modelSizeBytes int64 + + // envVars contains model-specific environment variables to pass to the backend process + envVars map[string]string } // WithConfigRevision binds a load to the semantic revision of the resolved @@ -112,6 +115,12 @@ func WithModelSizeBytes(bytes int64) Option { } } +func WithEnvVars(envVars map[string]string) Option { + return func(o *Options) { + o.envVars = envVars + } +} + func NewOptions(opts ...Option) *Options { o := &Options{ gRPCOptions: &pb.ModelOptions{}, diff --git a/pkg/model/process.go b/pkg/model/process.go index 9b799e5a5..712f870af 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -229,11 +229,11 @@ func (ml *ModelLoader) GetGRPCPID(id string) (int, error) { // StartProcess starts a gRPC backend process and returns its process handle. // This is the public wrapper for the internal startProcess method, used by // the serve-backend CLI subcommand to start a backend on a specified address. -func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) { - return ml.startProcess(grpcProcess, id, serverAddress, args...) +func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string, envVars map[string]string, args ...string) (*process.Process, error) { + return ml.startProcess(grpcProcess, id, serverAddress, envVars, args...) } -func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) { +func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, envVars map[string]string, args ...string) (*process.Process, error) { // Make sure the process is executable // Check first if it has executable permissions if fi, err := os.Stat(grpcProcess); err == nil { @@ -277,6 +277,13 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string // exit cannot strand request files directly in the host's shared /tmp. stateDir := runtime.dir + // Add model-specific environment variables + if envVars != nil { + for key, value := range envVars { + env = append(env, fmt.Sprintf("%s=%s", key, value)) + } + } + grpcControlProcess := process.New( process.WithStateDir(stateDir), process.WithName(filepath.Base(grpcProcess)), diff --git a/pkg/model/process_exit_test.go b/pkg/model/process_exit_test.go index 2b941bf6e..1f0bd9093 100644 --- a/pkg/model/process_exit_test.go +++ b/pkg/model/process_exit_test.go @@ -28,7 +28,7 @@ var _ = Describe("backend process exit diagnostics", func() { }) loader := NewModelLoader(&system.SystemState{Model: system.Model{ModelsPath: tmpDir}}) - process, err := loader.startProcess(backendPath, "test-model", "127.0.0.1:65535") + process, err := loader.startProcess(backendPath, "test-model", "127.0.0.1:65535", nil) Expect(err).ToNot(HaveOccurred()) Eventually(process.Done()).Should(BeClosed()) backendTemp, err := os.ReadFile(backendPath + ".tmpdir")