diff --git a/src/openllm/__main__.py b/src/openllm/__main__.py index 68f32980..d627a8be 100644 --- a/src/openllm/__main__.py +++ b/src/openllm/__main__.py @@ -120,7 +120,7 @@ def _select_target(bento: BentoInfo, targets: list[DeploymentTarget]) -> Deploym return selected -def _select_action(bento: BentoInfo, score: float) -> None: +def _select_action(bento: BentoInfo, score: float, context: typing.Optional[str] = None) -> None: if score > 0: options: list[typing.Any] = [ questionary.Separator('Available actions'), @@ -180,17 +180,27 @@ def _select_action(bento: BentoInfo, score: float) -> None: output(f' $ openllm serve {bento}', style='orange') elif action == 'deploy': ensure_cloud_context() - targets = get_cloud_machine_spec() + targets = get_cloud_machine_spec(context=context) target = _select_target(bento, targets) try: - cloud_deploy(bento, target) + cloud_deploy(bento, target, context=context) finally: output('\nUse this command to run the action again:', style='green') output(f' $ openllm deploy {bento} --instance-type {target.name}', style='orange') @app.command(help='get started interactively') -def hello(repo: typing.Optional[str] = None) -> None: +def hello( + repo: typing.Optional[str] = None, + env: typing.Optional[list[str]] = typer.Option( + None, + '--env', + help='Environment variables to pass to the deployment command. Format: NAME or NAME=value. Can be specified multiple times.', + ), + context: typing.Optional[str] = typer.Option( + None, '--context', help='BentoCloud context name to pass to the deployment command.' + ), +) -> None: cmd_update() INTERACTIVE.set(True) @@ -211,7 +221,7 @@ def hello(repo: typing.Optional[str] = None) -> None: bento_name, repo = _select_bento_name(models, target) bento, score = _select_bento_version(models, target, bento_name, repo) - _select_action(bento, score) + _select_action(bento, score, context=context) @app.command(help='start an OpenAI API compatible chat server and chat in browser') @@ -278,15 +288,20 @@ def deploy( '--env', help='Environment variables to pass to the deployment command. Format: NAME or NAME=value. Can be specified multiple times.', ), + context: typing.Optional[str] = typer.Option( + None, '--context', help='BentoCloud context name to pass to the deployment command.' + ), ) -> None: cmd_update() if verbose: VERBOSE_LEVEL.set(20) bento = ensure_bento(model, repo_name=repo) if instance_type is not None: - return cloud_deploy(bento, DeploymentTarget(accelerators=[], name=instance_type), cli_envs=env) + return cloud_deploy( + bento, DeploymentTarget(accelerators=[], name=instance_type), cli_envs=env, context=context + ) targets = sorted( - filter(lambda x: can_run(bento, x) > 0, get_cloud_machine_spec()), + filter(lambda x: can_run(bento, x) > 0, get_cloud_machine_spec(context=context)), key=lambda x: can_run(bento, x), reverse=True, ) @@ -295,7 +310,7 @@ def deploy( raise typer.Exit(1) target = targets[0] output(f'Recommended instance type: {target.name}', style='green') - cloud_deploy(bento, target, cli_envs=env) + cloud_deploy(bento, target, cli_envs=env, context=context) @app.callback(invoke_without_command=True) diff --git a/src/openllm/cloud.py b/src/openllm/cloud.py index 503c558b..56bc0dce 100644 --- a/src/openllm/cloud.py +++ b/src/openllm/cloud.py @@ -21,6 +21,7 @@ def _get_deploy_cmd( bento: BentoInfo, target: typing.Optional[DeploymentTarget] = None, cli_envs: typing.Optional[list[str]] = None, + context: typing.Optional[str] = None, ) -> tuple[list[str], EnvVars]: cmd = ['bentoml', 'deploy', bento.bentoml_tag] env = EnvVars({'BENTOML_HOME': f'{bento.repo.path}/bentoml'}) @@ -45,10 +46,15 @@ def _get_deploy_cmd( # Process envs defined in bento.yaml, skipping those overridden by CLI required_envs = bento.bento_yaml.get('envs', []) + + all_required_env_names = [env['name'] for env in required_envs if 'name' in env] required_env_names = [ env['name'] for env in required_envs - if 'name' in env and env['name'] not in explicit_envs and not env.get('value') + if 'name' in env + and env['name'] not in explicit_envs + and not env.get('value') + and env['name'] not in os.environ ] if required_env_names: output( @@ -85,6 +91,11 @@ def _get_deploy_cmd( raise typer.Exit(1) cmd += ['--env', f'{name}={value}'] + # Add any required envs from os.environ that haven't been handled yet + for name in all_required_env_names: + if name in os.environ: + cmd += ['--env', f'{name}={os.environ.get(name)}'] + # Add explicitly provided env vars from CLI for name, value in explicit_envs.items(): cmd += ['--env', f'{name}={value}'] @@ -92,6 +103,9 @@ def _get_deploy_cmd( if target: cmd += ['--instance-type', target.name] + if context: + cmd += ['--context', context] + base_config = resolve_cloud_config() if not base_config.exists(): raise Exception('Cannot find cloud config.') @@ -148,9 +162,11 @@ def ensure_cloud_context() -> None: raise typer.Exit(1) -def get_cloud_machine_spec() -> list[DeploymentTarget]: +def get_cloud_machine_spec(context: typing.Optional[str] = None) -> list[DeploymentTarget]: ensure_cloud_context() cmd = ['bentoml', 'deployment', 'list-instance-types', '-o', 'json'] + if context: + cmd += ['--context', context] try: result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) instance_types = json.loads(result) @@ -174,8 +190,11 @@ def get_cloud_machine_spec() -> list[DeploymentTarget]: def deploy( - bento: BentoInfo, target: DeploymentTarget, cli_envs: typing.Optional[list[str]] = None + bento: BentoInfo, + target: DeploymentTarget, + cli_envs: typing.Optional[list[str]] = None, + context: typing.Optional[str] = None, ) -> None: ensure_cloud_context() - cmd, env = _get_deploy_cmd(bento, target, cli_envs=cli_envs) + cmd, env = _get_deploy_cmd(bento, target, cli_envs=cli_envs, context=context) run_command(cmd, env=env, cwd=None)