From ae35ee811521903a1ea35dc2cbf7e73e13619d77 Mon Sep 17 00:00:00 2001
From: aarnphm-ec2-dev <29749331+aarnphm@users.noreply.github.com>
Date: Tue, 8 Aug 2023 10:17:48 +0000
Subject: [PATCH] fix(build): set legacy serialisation for vllm on Bento

Signed-off-by: aarnphm-ec2-dev <29749331+aarnphm@users.noreply.github.com>
---
 src/openllm/bundle/_package.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/openllm/bundle/_package.py b/src/openllm/bundle/_package.py
index f618bf48..ce8eae14 100644
--- a/src/openllm/bundle/_package.py
+++ b/src/openllm/bundle/_package.py
@@ -119,6 +119,7 @@ def construct_docker_options(
   ]
   _bentoml_config_options += " " if _bentoml_config_options else "" + " ".join(_bentoml_config_options_opts)
   env: openllm.utils.EnvVarMixin = llm.config["env"]
+  if env["framework_value"] == "vllm": serialisation_format = "legacy"
   env_dict = {
       env.framework: env["framework_value"], env.config: f"'{llm.config.model_dump_json().decode()}'", "OPENLLM_MODEL": llm.config["model_name"], "OPENLLM_SERIALIZATION": serialisation_format, "OPENLLM_ADAPTER_MAP": f"'{orjson.dumps(adapter_map).decode()}'", "BENTOML_DEBUG": str(True), "BENTOML_QUIET": str(False), "BENTOML_CONFIG_OPTIONS": f"'{_bentoml_config_options}'",
       env.model_id: f"/home/bentoml/bento/models/{llm.tag.path()}"