mirror/OpenLLM
1
0
Fork 0
mirror of https://github.com/bentoml/OpenLLM.git synced 2026-02-20 07:33:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

perf: unify LLM interface (#518)

Browse Source 
Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com>
Signed-off-by: paperspace <29749331+aarnphm@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Aaron Pham
2023-11-06 20:39:43 -05:00
committed by GitHub
parent f2639879af
commit e2029c934b
136 changed files with 9646 additions and 11244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
openllm-python/src/openllm/testing.py
View File

@@ -42,15 +42,15 @@ def build_container(bento: bentoml.Bento | str | bentoml.Tag, image_tag: str | N
@contextlib.contextmanager
def prepare(model: str,
model_id: str | None = None,
implementation: LiteralBackend = 'pt',
model_id: str,
backend: LiteralBackend = 'pt',
deployment_mode: t.Literal['container', 'local'] = 'local',
clean_context: contextlib.ExitStack | None = None,
cleanup: bool = True) -> t.Iterator[str]:
if clean_context is None:
clean_context = contextlib.ExitStack()
cleanup = True
llm = openllm.infer_auto_class(implementation).for_model(model, model_id=model_id, ensure_available=True)
llm = openllm.LLM[t.Any, t.Any](model_id, backend=backend)
bento_tag = bentoml.Tag.from_taglike(f'{llm.llm_type}-service:{llm.tag.version}')
if not bentoml.list(bento_tag):
bento = clean_context.enter_context(build_bento(model, model_id=model_id, cleanup=cleanup))