mirror/OpenLLM
1
0
Fork 0
mirror of https://github.com/bentoml/OpenLLM.git synced 2026-08-01 18:48:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): `openllm models` for showing available

Browse Source 
Signed-off-by: aarnphm-ec2-dev <29749331+aarnphm@users.noreply.github.com>
This commit is contained in:
aarnphm-ec2-dev
2023-07-24 22:58:14 +00:00
parent 0fe9d83ff3
commit 084786c898
2 changed files with 74 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

311
src/openllm/cli.py
View File

@@ -153,25 +153,14 @@ OPENLLM_FIGLET = """\
_AnyCallable = t.Callable[..., t.Any]
FC = t.TypeVar("FC", bound=t.Union[_AnyCallable, click.Command])
def parse_device_callback(
ctx: click.Context, param: click.Parameter, value: tuple[tuple[str], ...] | None
) -> TupleStr | None:
if value is None:
return value
if not LazyType(TupleStr).isinstance(value):
ctx.fail(f"{param} only accept multiple values, not {type(value)} (value: {value})")
def parse_device_callback(ctx: click.Context, param: click.Parameter, value: tuple[tuple[str], ...] | None) -> TupleStr | None:
if value is None: return value
if not LazyType(TupleStr).isinstance(value): ctx.fail(f"{param} only accept multiple values, not {type(value)} (value: {value})")
el: TupleStr = tuple(i for k in value for i in k)
# NOTE: --device all is a special case
if len(el) == 1 and el[0] == "all":
return tuple(map(str, available_devices()))
if len(el) == 1 and el[0] == "all": return tuple(map(str, available_devices()))
return el
def _echo(text: t.Any, fg: str = "green", _with_style: bool = True, **attrs: t.Any) -> None:
call = click.echo
if _with_style:
@@ -179,7 +168,6 @@ def _echo(text: t.Any, fg: str = "green", _with_style: bool = True, **attrs: t.A
call = click.secho
call(text, **attrs)
def output_option(f: _AnyCallable, *, factory: t.Any = click) -> t.Callable[[FC], FC]:
return factory.option(
"-o",
@@ -193,15 +181,12 @@ def output_option(f: _AnyCallable, *, factory: t.Any = click) -> t.Callable[[FC]
show_envvar=True,
)(f)
def machine_option(factory: t.Any) -> t.Callable[[FC], FC]:
return factory.option("--machine", is_flag=True, default=False, hidden=True)
def model_id_option(factory: t.Any, model_env: EnvVarMixin | None = None) -> t.Callable[[FC], FC]:
envvar = None
if model_env is not None:
envvar = model_env.model_id
if model_env is not None: envvar = model_env.model_id
return factory.option(
"--model-id",
type=click.STRING,
@@ -211,7 +196,6 @@ def model_id_option(factory: t.Any, model_env: EnvVarMixin | None = None) -> t.C
show_envvar=True if envvar is not None else False,
)
def model_version_option(factory: t.Any) -> t.Callable[[FC], FC]:
return factory.option(
"--model-version",
@@ -220,7 +204,6 @@ def model_version_option(factory: t.Any) -> t.Callable[[FC], FC]:
help="Optional model version to save for this model. It will be inferred automatically from model-id.",
)
def workers_per_resource_callback(ctx: click.Context, param: click.Parameter, value: str | None) -> str | None:
if value is None: return value
value = inflection.underscore(value)
@@ -241,8 +224,7 @@ def workers_per_resource_option(factory: t.Any, build: bool = False) -> t.Callab
- ``conserved``: This will determine the number of available GPU resources, and only assign one worker for the LLMRunner. For example, if ther are 4 GPUs available, then ``conserved`` is equivalent to ``--workers-per-resource 0.25``.
"""
if build:
help_str += """\n
if build: help_str += """\n
**Note**: The workers value passed into 'build' will determine how the LLM can
be provisioned in Kubernetes as well as in standalone container. This will
ensure it has the same effect with 'openllm start --workers ...'"""
@@ -251,11 +233,8 @@ def workers_per_resource_option(factory: t.Any, build: bool = False) -> t.Callab
def quantize_option(factory: t.Any, build: bool = False, model_env: EnvVarMixin | None = None) -> t.Callable[[FC], FC]:
envvar = None
if model_env is not None:
envvar = model_env.quantize
help_str = (
"Running this model in quantized mode." if not build else "Set quantization mode for serving in deployment."
)
if model_env is not None: envvar = model_env.quantize
help_str = "Running this model in quantized mode." if not build else "Set quantization mode for serving in deployment."
help_str += "\nNOTE: Quantization is only available for PyTorch models. "
return factory.option(
"--quantise",
@@ -269,12 +248,9 @@ def quantize_option(factory: t.Any, build: bool = False, model_env: EnvVarMixin
)
def bettertransformer_option(
factory: t.Any, build: bool = False, model_env: EnvVarMixin | None = None
) -> t.Callable[[FC], FC]:
def bettertransformer_option(factory: t.Any, build: bool = False, model_env: EnvVarMixin | None = None) -> t.Callable[[FC], FC]:
envvar = None
if model_env is not None:
envvar = model_env.bettertransformer
if model_env is not None: envvar = model_env.bettertransformer
return factory.option(
"--bettertransformer",
is_flag=True,
@@ -300,39 +276,28 @@ def serialisation_option(factory: t.Any) -> t.Callable[[FC], FC]:
envvar="OPENLLM_SERIALIZATION",
)
_adapter_mapping_key = "adapter_map"
def _id_callback(ctx: click.Context, _: click.Parameter, value: tuple[str, ...] | None) -> None:
if not value:
return None
if _adapter_mapping_key not in ctx.params:
ctx.params[_adapter_mapping_key] = {}
if not value: return None
if _adapter_mapping_key not in ctx.params: ctx.params[_adapter_mapping_key] = {}
for v in value:
adapter_id, *adapter_name = v.rsplit(":", maxsplit=1)
try:
# try to resolve the full path if users pass in relative,
# currently only support one level of resolve path.
adapter_id = resolve_user_filepath(adapter_id, os.getcwd())
except FileNotFoundError:
pass
# try to resolve the full path if users pass in relative,
# currently only support one level of resolve path with current directory
try: adapter_id = resolve_user_filepath(adapter_id, os.getcwd())
except FileNotFoundError: pass
ctx.params[_adapter_mapping_key][adapter_id] = adapter_name[0] if len(adapter_name) > 0 else None
return None
@attr.define
class CliContext:
cloud_context: str | None = attr.field(default=None, converter=attr.converters.default_if_none("default"))
def with_options(self, **attrs: t.Any) -> t.Self: return attr.evolve(self, **attrs)
class OpenLLMCommandGroup(BentoMLCommandGroup):
NUMBER_OF_COMMON_PARAMS = 5 # parameters in common_params + 1 faked group option header
@staticmethod
def common_params(f: FC) -> t.Callable[[FC], FC]:
"""This is not supposed to be used with unprocessed click function.
This should be used a the last currying from common_params -> usage_tracking -> exception_handling.
"""
# The following logics is similar to one of BentoMLCommandGroup
from bentoml._internal.configuration import DEBUG_ENV_VAR
@@ -354,20 +319,13 @@ class OpenLLMCommandGroup(BentoMLCommandGroup):
configure_logging()
return f(*args, **attrs)
return wrapper
@staticmethod
def usage_tracking(func: _AnyCallable, group: click.Group, **attrs: t.Any) -> _AnyCallable:
"""This is not supposed to be used with unprocessed click function.
This should be used a the last currying from common_params -> usage_tracking -> exception_handling.
"""
command_name = attrs.get("name", func.__name__)
@functools.wraps(func)
def wrapper(do_not_track: bool, *args: P.args, **attrs: P.kwargs) -> t.Any:
if do_not_track:
with analytics.set_bentoml_tracking(): return func(*args, **attrs)
start_time = time.time_ns()
with analytics.set_bentoml_tracking():
assert group.name is not None, "group.name should not be None"
@@ -385,17 +343,10 @@ class OpenLLMCommandGroup(BentoMLCommandGroup):
event.return_code = 2 if isinstance(e, KeyboardInterrupt) else 1
analytics.track(event)
raise
return wrapper
@staticmethod
def exception_handling(func: _AnyCallable, group: click.Group, **attrs: t.Any) -> ClickFunctionWrapper[..., t.Any]:
"""This is not supposed to be used with unprocessed click function.
This should be used a the last currying from common_params -> usage_tracking -> exception_handling.
"""
command_name = attrs.get("name", func.__name__)
@functools.wraps(func)
def wrapper(*args: P.args, **attrs: P.kwargs) -> t.Any:
try: return func(*args, **attrs)
@@ -403,7 +354,6 @@ class OpenLLMCommandGroup(BentoMLCommandGroup):
raise click.ClickException(click.style(f"[{group.name}] '{command_name}' failed: " + err.message, fg="red")) from err
except KeyboardInterrupt: pass
return t.cast("ClickFunctionWrapper[..., t.Any]", wrapper)
def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
cmd_name = self.resolve_alias(cmd_name)
_mapping = {"start": _cached_http, "start-grpc": _cached_grpc}
@@ -417,18 +367,15 @@ class OpenLLMCommandGroup(BentoMLCommandGroup):
except bentoml.exceptions.NotFound: pass
raise click.BadArgumentUsage(f"{cmd_name} is not a valid model identifier supported by OpenLLM.") from None
return super().get_command(ctx, cmd_name)
def list_commands(self, ctx: click.Context) -> list[str]:
if ctx.command.name in {"start", "start-grpc"}: return list(openllm.CONFIG_MAPPING.keys())
return super().list_commands(ctx)
@override
def command(self, *args: t.Any, **attrs: t.Any):
"""Override the default 'cli.command' with supports for aliases for given command, and it wraps the implementation with common parameters."""
if "context_settings" not in attrs: attrs["context_settings"] = {}
if "max_content_width" not in attrs["context_settings"]: attrs["context_settings"]["max_content_width"] = 120
aliases = attrs.pop("aliases", None)
def wrapper(f: _AnyCallable) -> click.Command:
name = f.__name__.lower()
if name.endswith("_command"): name = name[:-8]
@@ -455,7 +402,6 @@ class OpenLLMCommandGroup(BentoMLCommandGroup):
return cmd
return wrapper
@click.group(cls=OpenLLMCommandGroup, context_settings=_CONTEXT_SETTINGS, name="openllm")
@click.version_option(__version__, "--version", "-v")
def cli() -> None:
@@ -472,7 +418,6 @@ def cli() -> None:
Fine-tune, serve, deploy, and monitor any LLMs with ease.
""" # noqa: D205
@cli.group(cls=OpenLLMCommandGroup, context_settings=_CONTEXT_SETTINGS, name="start", aliases=["start-http"])
def start_command() -> None:
"""Start any LLM as a REST server.
@@ -483,7 +428,6 @@ def start_command() -> None:
```
"""
@cli.group(cls=OpenLLMCommandGroup, context_settings=_CONTEXT_SETTINGS, name="start-grpc")
def start_grpc_command() -> None:
"""Start any LLM as a gRPC server.
@@ -494,13 +438,11 @@ def start_grpc_command() -> None:
```
"""
# NOTE: A list of bentoml option that is not needed for parsing.
# NOTE: User shouldn't set '--working-dir', as OpenLLM will setup this.
# NOTE: production is also deprecated
_IGNORED_OPTIONS = {"working_dir", "production", "protocol_version"}
def parse_serve_args(serve_grpc: bool) -> t.Callable[[t.Callable[..., openllm.LLMConfig]], t.Callable[[FC], FC]]:
"""Parsing `bentoml serve|serve-grpc` click.Option to be parsed via `openllm start`."""
from bentoml_cli.cli import cli
@@ -529,10 +471,7 @@ def parse_serve_args(serve_grpc: bool) -> t.Callable[[t.Callable[..., openllm.LL
return group(f)
return decorator
_http_server_args = parse_serve_args(False)
_grpc_server_args = parse_serve_args(True)
_http_server_args, _grpc_server_args = parse_serve_args(False), parse_serve_args(True)
def start_decorator(llm_config: openllm.LLMConfig, serve_grpc: bool = False) -> t.Callable[[_AnyCallable], t.Callable[[FC], FC]]:
opts = [
@@ -566,7 +505,6 @@ def start_decorator(llm_config: openllm.LLMConfig, serve_grpc: bool = False) ->
The following are currently being worked on:
- DeepSpeed Inference: [link](https://www.deepspeed.ai/inference/)
""",
),
cog.optgroup.option("--device", type=dantic.CUDA, multiple=True, envvar="CUDA_VISIBLE_DEVICES", callback=parse_device_callback, help=f"Assign GPU devices (if available) for {llm_config['model_name']}.", show_envvar=True),
@@ -595,15 +533,11 @@ def start_decorator(llm_config: openllm.LLMConfig, serve_grpc: bool = False) ->
cog.optgroup.option("--adapter-id", default=None, help="Optional name or path for given LoRA adapter" + f" to wrap '{llm_config['model_name']}'", multiple=True, callback=_id_callback, metavar="[PATH | [remote/][adapter_name:]adapter_id][, ...]"),
click.option("--return-process", is_flag=True, default=False, help="Internal use only.", hidden=True),
]
def decorator(f: _AnyCallable) -> _AnyCallable:
for opt in reversed(opts):
f = opt(f)
for opt in reversed(opts): f = opt(f)
return f
return decorator
def parse_config_options(
config: openllm.LLMConfig,
server_timeout: int,
@@ -620,7 +554,6 @@ def parse_config_options(
environ["BENTOML_CONFIG_OPTIONS"] = _bentoml_config_options_env
return environ
_wpr_strategies = {"round_robin", "conserved"}
def start_command_factory(model: str, _context_settings: DictStrAny | None = None, _serve_grpc: bool = False) -> click.Command:
@@ -689,16 +622,11 @@ Available official model_id(s): [default: {llm_config['default_id']}]
return_process: bool,
**attrs: t.Any,
) -> openllm.LLMConfig | subprocess.Popen[bytes]:
if serialisation_format == "safetensors" and quantize is not None:
if os.getenv("OPENLLM_SERIALIZATION_WARNING", str(True)).upper() in ENV_VARS_TRUE_VALUES:
_echo(
f"'--quantize={quantize}' might not work with 'safetensors' serialisation format. Use with caution!. To silence this warning, set \"OPENLLM_SERIALIZATION_WARNING=False\"\nNote: You can always fallback to '--serialisation legacy' when running quantisation.",
fg="yellow",
)
if serialisation_format == "safetensors" and quantize is not None and os.getenv("OPENLLM_SERIALIZATION_WARNING", str(True)).upper() in ENV_VARS_TRUE_VALUES:
_echo(f"'--quantize={quantize}' might not work with 'safetensors' serialisation format. Use with caution!. To silence this warning, set \"OPENLLM_SERIALIZATION_WARNING=False\"\nNote: You can always fallback to '--serialisation legacy' when running quantisation.", fg="yellow")
adapter_map: dict[str, str | None] | None = attrs.pop(_adapter_mapping_key, None)
config, server_attrs = llm_config.model_validate_click(**attrs)
server_timeout = first_not_none(server_timeout, default=config["timeout"])
server_attrs.update({"working_dir": os.path.dirname(__file__), "timeout": server_timeout})
if _serve_grpc: server_attrs["grpc_protocol_version"] = "v1"
# NOTE: currently, theres no development args in bentoml.Server. To be fixed upstream.
@@ -709,8 +637,12 @@ Available official model_id(s): [default: {llm_config['default_id']}]
if isinstance(wpr, str):
if wpr == "round_robin": wpr = 1.0
elif wpr == "conserved":
available_gpu = device if device else available_devices()
wpr = 1.0 if len(available_gpu) == 0 else float(1 / len(available_gpu))
if device and device_count() == 0:
_echo("--device will have no effect as there is no GPUs available", fg="yellow")
wpr = 1.0
else:
available_gpu = len(device) if device else device_count()
wpr = 1.0 if available_gpu == 0 else float(1 / available_gpu)
else: wpr = float(wpr)
# Create a new model env to work with the envvar during CLI invocation
@@ -725,7 +657,7 @@ Available official model_id(s): [default: {llm_config['default_id']}]
start_env.update(
{
"OPENLLM_MODEL": model,
"BENTOML_DEBUG": str(not get_quiet_mode()),
"BENTOML_DEBUG": str(get_debug_mode()),
"BENTOML_HOME": os.getenv("BENTOML_HOME", BentoMLContainer.bentoml_home.get()),
"OPENLLM_ADAPTER_MAP": orjson.dumps(adapter_map).decode(),
"OPENLLM_SERIALIZATION": serialisation_format,
@@ -777,7 +709,6 @@ Available official model_id(s): [default: {llm_config['default_id']}]
return start_cmd
def noop_command(llm_config: openllm.LLMConfig, _serve_grpc: bool, **command_attrs: t.Any) -> click.Command:
context_settings = command_attrs.pop("context_settings", {})
context_settings.update({"ignore_unknown_options": True, "allow_extra_args": True})
@@ -789,78 +720,40 @@ def noop_command(llm_config: openllm.LLMConfig, _serve_grpc: bool, **command_att
_echo("No GPU available, therefore this command is disabled", fg="red")
analytics.track_start_init(llm_config)
return llm_config
return noop
def prerequisite_check(
ctx: click.Context,
llm_config: openllm.LLMConfig,
quantize: t.LiteralString | None,
adapter_map: dict[str, str | None] | None,
num_workers: int,
) -> None:
def prerequisite_check(ctx: click.Context, llm_config: openllm.LLMConfig, quantize: t.LiteralString | None, adapter_map: dict[str, str | None] | None, num_workers: int) -> None:
if quantize:
if device_count() < 1:
_echo("Quantization requires at least 1 GPU (got None)", fg="red")
ctx.exit(1)
if llm_config.default_implementation() == "vllm":
_echo("Quantization is not yet supported with vLLM", fg="red")
ctx.exit(1)
if adapter_map and not is_peft_available():
_echo(
"Using adapter requires 'peft' to be available. Make sure to install with 'pip install \"openllm[fine-tune]\"'",
fg="red",
)
_echo("Using adapter requires 'peft' to be available. Make sure to install with 'pip install \"openllm[fine-tune]\"'", fg="red")
ctx.exit(1)
requirements = llm_config["requirements"]
if requirements is not None and len(requirements) > 0:
missing_requirements = [i for i in requirements if importlib.util.find_spec(inflection.underscore(i)) is None]
if len(missing_requirements) > 0:
_echo(
f"Make sure to have the following dependencies available: {missing_requirements}",
fg="yellow",
)
if num_workers > 1 and device_count() < num_workers:
raise click.BadOptionUsage(
"workers_per_resource",
f"# of workers is infered to {num_workers} GPUs per runner worker, while there are only"
f"'{device_count()}' for inference. (Tip: Try again using '--workers-per-resource={1/device_count()}')",
ctx=ctx,
)
if len(missing_requirements) > 0: _echo(f"Make sure to have the following dependencies available: {missing_requirements}", fg="yellow")
if num_workers > 1 and device_count() < num_workers: raise click.BadOptionUsage("workers_per_resource", f"# of workers is infered to {num_workers} GPUs per runner worker, while there are only '{device_count()}' for inference. (Tip: Try again using '--workers-per-resource={1/device_count()}')", ctx=ctx)
@cli.command(name="import", aliases=["download"])
@click.argument(
"model",
type=click.Choice([inflection.dasherize(name) for name in openllm.CONFIG_MAPPING.keys()]),
)
@click.argument(
"model_id",
type=click.STRING,
default=None,
metavar="Optional[REMOTE_REPO/MODEL_ID | /path/to/local/model]",
required=False,
)
@click.argument("model", type=click.Choice([inflection.dasherize(name) for name in openllm.CONFIG_MAPPING.keys()]))
@click.argument("model_id", type=click.STRING, default=None, metavar="Optional[REMOTE_REPO/MODEL_ID | /path/to/local/model]", required=False)
@click.argument("converter", envvar="CONVERTER", type=click.STRING, default=None, required=False, metavar=None)
@model_version_option(click)
@click.option(
"--runtime",
type=click.Choice(["ggml", "transformers"]),
default="transformers",
help="The runtime to use for the given model. Default is transformers.",
)
@click.option("--runtime", type=click.Choice(["ggml", "transformers"]), default="transformers", help="The runtime to use for the given model. Default is transformers.")
@output_option
@quantize_option(click)
@machine_option(click)
@click.option(
"--implementation",
type=click.Choice(["pt", "tf", "flax", "vllm"]),
default=None,
help="The implementation for saving this LLM.",
)
@click.option("--implementation", type=click.Choice(["pt", "tf", "flax", "vllm"]), default=None, help="The implementation for saving this LLM.")
@serialisation_option(click)
def download_models_command(
def import_command(
model: str,
model_id: str | None,
converter: str | None,
@@ -949,10 +842,9 @@ def download_models_command(
_echo(msg, fg="yellow", nl=True)
_ref = openllm.serialisation.get(llm, auto_import=True)
if impl == "pt" and is_torch_available() and torch.cuda.is_available(): torch.cuda.empty_cache()
if machine:
# NOTE: We will prefix the tag with __tag__ and we can use regex to correctly
# get the tag from 'bentoml.bentos.build|build_bentofile'
_echo(f"__tag__:{_ref.tag}", fg="white")
# NOTE: We will prefix the tag with __tag__ and we can use regex to correctly
# get the tag from 'bentoml.bentos.build|build_bentofile'
if machine: _echo(f"__tag__:{_ref.tag}", fg="white")
elif output == "pretty":
if _previously_saved: _echo(f"{model} with 'model_id={model_id}' is already setup for framework '{impl}': {_ref.tag!s}", nl=True, fg="yellow")
else: _echo(f"Saved model: {_ref.tag}")
@@ -960,11 +852,9 @@ def download_models_command(
else: _echo(_ref.tag)
return _ref
_cached_http = {key: start_command_factory(key, _context_settings=_CONTEXT_SETTINGS) for key in openllm.CONFIG_MAPPING}
_cached_grpc = {key: start_command_factory(key, _context_settings=_CONTEXT_SETTINGS, _serve_grpc=True) for key in openllm.CONFIG_MAPPING}
@overload
def _start(
model_name: str,
@@ -1180,51 +1070,29 @@ def _build(
serialisation_format,
]
if quantize and bettertransformer:
raise OpenLLMException("'quantize' and 'bettertransformer' are currently mutually exclusive.")
if quantize and bettertransformer: raise OpenLLMException("'quantize' and 'bettertransformer' are currently mutually exclusive.")
if quantize: args.extend(["--quantize", quantize])
if bettertransformer: args.append("--bettertransformer")
if containerize and push: raise OpenLLMException("'containerize' and 'push' are currently mutually exclusive.")
if push: args.extend(["--push"])
if containerize: args.extend(["--containerize"])
if model_id: args.extend(["--model-id", model_id])
if build_ctx: args.extend(["--build-ctx", build_ctx])
if enable_features: args.extend([f"--enable-features={f}" for f in enable_features])
if workers_per_resource: args.extend(["--workers-per-resource", str(workers_per_resource)])
if overwrite: args.append("--overwrite")
if adapter_map: args.extend([f"--adapter-id={k}{':'+v if v is not None else ''}" for k, v in adapter_map.items()])
if model_version: args.extend(["--model-version", model_version])
if dockerfile_template: args.extend(["--dockerfile-template", dockerfile_template])
if additional_args: args.extend(additional_args)
if quantize:
args.extend(["--quantize", quantize])
if bettertransformer:
args.append("--bettertransformer")
if containerize and push:
raise OpenLLMException("'containerize' and 'push' are currently mutually exclusive.")
if push:
args.extend(["--push"])
if containerize:
args.extend(["--containerize"])
if model_id:
args.extend(["--model-id", model_id])
if build_ctx:
args.extend(["--build-ctx", build_ctx])
if enable_features:
args.extend([f"--enable-features={f}" for f in enable_features])
if workers_per_resource:
args.extend(["--workers-per-resource", str(workers_per_resource)])
if overwrite:
args.append("--overwrite")
if adapter_map:
args.extend([f"--adapter-id={k}{':'+v if v is not None else ''}" for k, v in adapter_map.items()])
if model_version:
args.extend(["--model-version", model_version])
if dockerfile_template:
args.extend(["--dockerfile-template", dockerfile_template])
if additional_args:
args.extend(additional_args)
try:
output = subprocess.check_output(args, env=os.environ.copy(), cwd=build_ctx or os.getcwd())
try: output = subprocess.check_output(args, env=os.environ.copy(), cwd=build_ctx or os.getcwd())
except subprocess.CalledProcessError as e:
logger.error("Exception caught while building %s", model_name, exc_info=e)
if e.stderr:
raise OpenLLMException(e.stderr.decode("utf-8")) from None
if e.stderr: raise OpenLLMException(e.stderr.decode("utf-8")) from None
raise OpenLLMException(str(e)) from None
return bentoml.get(_tag_parsing(output), _bento_store=bento_store)
def _import_model(
model_name: str,
/,
@@ -1275,36 +1143,21 @@ def _import_model(
"--serialisation",
serialisation_format,
]
if model_id is not None:
args.append(model_id)
if model_version is not None:
args.extend(["--model-version", str(model_version)])
if additional_args is not None:
args.extend(additional_args)
if quantize is not None:
args.extend(["--quantize", quantize])
return download_models_command.main(args=args, standalone_mode=False)
if model_id is not None: args.append(model_id)
if model_version is not None: args.extend(["--model-version", str(model_version)])
if additional_args is not None: args.extend(additional_args)
if quantize is not None: args.extend(["--quantize", quantize])
return import_command.main(args=args, standalone_mode=False)
def _list_models() -> DictStrAny:
"""List all available models within the local store."""
args = ["-o", "json", "--show-available", "--machine"]
return models_command.main(args=args, standalone_mode=False)
start, start_grpc, build, import_model, list_models = (
codegen.gen_sdk(_start, _serve_grpc=False),
codegen.gen_sdk(_start, _serve_grpc=True),
codegen.gen_sdk(_build),
codegen.gen_sdk(_import_model),
codegen.gen_sdk(_list_models),
)
return models_command.main(args=["-o", "json", "--show-available", "--machine"], standalone_mode=False)
start, start_grpc, build, import_model, list_models = codegen.gen_sdk(_start, _serve_grpc=False), codegen.gen_sdk(_start, _serve_grpc=True), codegen.gen_sdk(_build), codegen.gen_sdk(_import_model), codegen.gen_sdk(_list_models)
@cli.command(context_settings={"token_normalize_func": inflection.underscore})
@click.argument(
"model_name", type=click.Choice([inflection.dasherize(name) for name in openllm.CONFIG_MAPPING.keys()])
)
@click.argument("model_name", type=click.Choice([inflection.dasherize(name) for name in openllm.CONFIG_MAPPING.keys()]))
@model_id_option(click)
@output_option
@machine_option(click)
@@ -1321,9 +1174,7 @@ start, start_grpc, build, import_model, list_models = (
)
@click.option(
"--enable-features",
help="Enable additional features for building this LLM Bento. Available: {}".format(
", ".join(openllm.utils.OPTIONAL_DEPENDENCIES)
),
help="Enable additional features for building this LLM Bento. Available: {}".format(", ".join(openllm.utils.OPTIONAL_DEPENDENCIES)),
multiple=True,
nargs=1,
metavar="FEATURE[,FEATURE]",
@@ -1472,8 +1323,7 @@ def build_command(
dockerfile_template=dockerfile_template_path,
runtime=runtime,
)
except Exception:
raise
except Exception: raise
if machine: _echo(f"__tag__:{bento.tag}", fg="white")
elif output == "pretty":
@@ -1559,24 +1409,19 @@ def models_command(ctx: click.Context, output: OutputLiteral, show_available: bo
try: openllm.AutoLLM.for_model(m, llm_config=config)
except Exception as e: failed_initialized.append((m, e))
ids_in_local_store: DictStrAny | None = None
ids_in_local_store = {k: [i for i in bentoml.models.list() if "framework" in i.info.labels and i.info.labels["framework"] == "openllm" and "model_name" in i.info.labels and i.info.labels["model_name"] == k] for k in json_data.keys()}
ids_in_local_store = {k: v for k, v in ids_in_local_store.items() if v}
local_models: DictStrAny | None = None
if show_available:
ids_in_local_store = {k: [i for i in bentoml.models.list() if "framework" in i.info.labels and i.info.labels["framework"] == "openllm" and "model_name" in i.info.labels and i.info.labels["model_name"] == k] for k in json_data.keys()}
ids_in_local_store = {k: v for k, v in ids_in_local_store.items() if v}
local_models = {k: [str(i.tag) for i in val] for k, val in ids_in_local_store.items()}
if machine:
if show_available:
assert ids_in_local_store
assert local_models
json_data["local"] = local_models
if show_available: json_data["local"] = local_models
return json_data
elif output == "pretty":
import tabulate
tabulate.PRESERVE_WHITESPACE = True
# llm, architecture, url, model_id, installation, cpu, gpu, runtime_impl
data: list[str | tuple[str, str, list[str], str, t.LiteralString, t.LiteralString, tuple[LiteralRuntime, ...]] ] = []
for m, v in json_data.items():
@@ -1625,19 +1470,13 @@ def models_command(ctx: click.Context, output: OutputLiteral, show_available: bo
_echo(err, fg="red")
if show_available:
assert ids_in_local_store is not None
assert local_models
if len(ids_in_local_store) == 0:
_echo("No models available locally.")
ctx.exit(0)
_echo("The following are available in local store:", fg="magenta")
_echo(orjson.dumps(local_models, option=orjson.OPT_INDENT_2).decode(), fg="white")
else:
if show_available:
assert ids_in_local_store
assert local_models
json_data["local"] = local_models
if show_available: json_data["local"] = local_models
_echo(orjson.dumps(json_data, option=orjson.OPT_INDENT_2,).decode(), fg="white")
ctx.exit(0)
@@ -1768,12 +1607,10 @@ def query(ctx: click.Context, prompt: str, endpoint: str, timeout: int, server_t
else: _echo(res["responses"], fg="white")
ctx.exit(0)
@cli.group(name="utils")
def utils_command() -> None:
"""Utilities Subcommand group."""
@utils_command.command()
@click.pass_context
def list_bentos(ctx: click.Context):
@@ -1796,8 +1633,6 @@ def dive_bentos(ctx: click.Context, bento: str, _bento_store: BentoStore = Provi
# copy and paste this into a new shell
_echo(f"cd $(python -m bentoml get {bentomodel.tag!s} -o path)", fg="white")
@overload
def get_prompt(model_name: str, prompt: str, format: str | None, output: OutputLiteral, machine: t.Literal[True] = True) -> str: ...
@overload

2
tools/dependencies.py
View File

@@ -174,7 +174,7 @@ _BASE_DEPENDENCIES = [
Dependencies(name="click", lower_constraint="8.1.6"),
Dependencies(name="typing_extensions"),
Dependencies(name="cuda-python", platform=("Darwin", "ne")),
Dependencies(name="bitsandbytes", upper_constraint="0.40"), # Currently only <0.40 works with CUDA 11.8
Dependencies(name="bitsandbytes", upper_constraint="0.42"), # 0.41 works with CUDA 11.8
]
_NIGHTLY_MAPPING: dict[str, Dependencies] = {