diff --git a/examples/cohere_client.py b/examples/cohere_client.py index a7b75b10..c9a73e6e 100644 --- a/examples/cohere_client.py +++ b/examples/cohere_client.py @@ -6,24 +6,12 @@ from cohere.responses.chat import StreamTextGeneration co = cohere.Client(api_key='na', api_url=os.getenv('OPENLLM_ENDPOINT', 'http://localhost:3000') + '/cohere') generation = co.generate(prompt='Write me a tag line for an ice cream shop.') -print(generation.generations[0].text) +print('One shot generation:', generation.generations[0].text) +print('\nStreaming response: ', flush=True, end='') for it in co.generate(prompt='Write me a tag line for an ice cream shop.', stream=True): print(it.text, flush=True, end='') -response = co.chat( - message="What is Epicurus's philosophy of life?", - temperature=0.6, - chat_history=[ - {'role': 'User', 'message': 'What is the meaning of life?'}, - { - 'role': 'Chatbot', - 'message': "Many thinkers have proposed theories about the meaning of life. \n\nFor instance, Jean-Paul Sartre believed that existence precedes essence, meaning that the essence, or meaning, of one's life arises after birth. Søren Kierkegaard argued that life is full of absurdity and that one must make one's own values in an indifferent world. Arthur Schopenhauer stated that one's life reflects one's will, and that the will (or life) is without aim, irrational, and full of pain. \n\nEarly thinkers such as John Locke, Jean-Jacques Rousseau and Adam Smith believed that humankind should find meaning through labour, property and social contracts. \n\nAnother way of thinking about the meaning of life is to focus on the pursuit of happiness or pleasure. Aristippus of Cyrene, a student of Socrates, founded an early Socratic school that emphasised one aspect of Socrates's teachings: that happiness is the end goal of moral action and that pleasure is the supreme good. Epicurus taught that the pursuit of modest pleasures was the greatest good, as it leads to tranquility, freedom from fear and absence of bodily pain. \n\nUltimately, the meaning of life is a subjective concept and what provides life with meaning differs for each individual.", - }, - ], -) -print(response) - for it in co.chat( message="What is Epicurus's philosophy of life?", temperature=0.6, @@ -38,3 +26,5 @@ for it in co.chat( ): if isinstance(it, StreamTextGeneration): print(it.text, flush=True, end='') + else: + print(f'\nGenerated object: {it}') diff --git a/openllm-python/src/openllm/entrypoints/cohere.py b/openllm-python/src/openllm/entrypoints/cohere.py index e79ffc77..6aee87af 100644 --- a/openllm-python/src/openllm/entrypoints/cohere.py +++ b/openllm-python/src/openllm/entrypoints/cohere.py @@ -11,7 +11,7 @@ from starlette.applications import Starlette from starlette.responses import JSONResponse, StreamingResponse from starlette.routing import Route -from openllm_core.utils import converter, gen_random_uuid +from openllm_core.utils import DEBUG, converter, gen_random_uuid from ._openapi import add_schema_definitions, append_schemas, get_generator from ..protocol.cohere import ( @@ -50,7 +50,7 @@ schemas = get_generator( 'externalDocs': 'https://docs.cohere.com/docs/the-cohere-platform', } ], - inject=False, + inject=DEBUG, ) logger = logging.getLogger(__name__) @@ -97,7 +97,7 @@ def mount_to_svc(svc: bentoml.Service, llm: openllm.LLM[M, T]) -> bentoml.Servic svc.mount_asgi_app(app, path=mount_path) return append_schemas( - svc, schemas.get_schema(routes=app.routes, mount_path=mount_path), tags_order='append', inject=False + svc, schemas.get_schema(routes=app.routes, mount_path=mount_path), tags_order='append', inject=DEBUG )