From ef724a3c9db8bfdf8114a389f6339033c1d61e76 Mon Sep 17 00:00:00 2001 From: Adira Date: Thu, 30 Jul 2026 17:01:47 +0300 Subject: [PATCH] feat(api): add /v1/detokenize endpoint (#9620) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(api): add /v1/detokenize endpoint Closes #1649. Mirror of the existing /v1/tokenize path, requested by @benniekiss in the issue thread for "complete API workflow" use cases that need to turn token IDs back into text without local processing. - Add Detokenize gRPC RPC with DetokenizeRequest{tokens} / DetokenizeResponse{content} messages. - Implement in the llama.cpp backend using common_token_to_piece, the same primitive TokenizeString already uses internally. - Other backends inherit the default Unimplemented from base.Base, in line with how Detect, Rerank, etc. are gated per-backend. - Wire up the Go gRPC interface, server, client, and in-process embed wrapper alongside their TokenizeString counterparts. - Add the schema types, ModelDetokenize wrapper, HTTP handler, route registration, RouteFeatureRegistry entry (gated by FeatureTokenize so no new feature flag is needed), and the discovery map entry under ai_functions. - Regenerated swagger reflects the new endpoint and types. - Update authentication.md to list /v1/detokenize alongside /v1/tokenize. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Adira Denis Muhando * test(e2e): add mock backend tests for /v1/detokenize Add Detokenize to the mock gRPC backend and wire up two e2e tests in the MockBackend suite: one that posts known token IDs and asserts a non-empty content response, and a round-trip that tokenizes first then detokenizes the returned IDs. Addresses reviewer feedback on #9620. Assisted-by: Claude:claude-sonnet-4-6 Signed-off-by: Adira Denis Muhando * fix(kokoros): implement detokenize in the Rust backend service The Detokenize RPC added in this PR grows the tonic-generated Backend trait. Unlike the other languages there is nothing to inherit a default from — Rust trait impls must list every method — so backend/rust/kokoros failed to compile: error[E0046]: not all trait items implemented, missing: `detokenize` --> src/service.rs:72:1 72 | impl Backend for KokorosService { Go backends pick up the Unimplemented default from base.Base, and the generated C++/Python servicer bases default to UNIMPLEMENTED, which is why the Rust backend was the only one that broke. kokoros is the sole Rust crate in the tree, so this is the full extent of the fallout. Return Status::unimplemented("Not supported"), matching how this same file already gates tokenize_string and ~20 other unsupported RPCs. Fixes the tests-kokoros and backend-jobs-singlearch-4 (-cpu-kokoros) failures on the previous head. Assisted-by: Claude:claude-opus-5 cargo Signed-off-by: Adira Denis Muhando --------- Signed-off-by: Adira Denis Muhando Co-authored-by: localai-org-maint-bot --- backend/backend.proto | 9 +++ backend/cpp/llama-cpp/grpc-server.cpp | 15 +++++ backend/rust/kokoros/src/service.rs | 7 +++ core/backend/detokenize.go | 67 +++++++++++++++++++++ core/http/auth/features.go | 1 + core/http/endpoints/localai/detokenize.go | 36 ++++++++++++ core/http/routes/localai.go | 7 +++ core/schema/tokenize.go | 9 +++ core/services/nodes/health_mock_test.go | 3 + core/services/nodes/inflight_test.go | 4 ++ docs/content/features/authentication.md | 2 +- pkg/grpc/backend.go | 1 + pkg/grpc/base/base.go | 4 ++ pkg/grpc/client.go | 22 +++++++ pkg/grpc/embed.go | 4 ++ pkg/grpc/interface.go | 1 + pkg/grpc/server.go | 12 ++++ swagger/docs.go | 51 ++++++++++++++++ swagger/swagger.json | 51 ++++++++++++++++ swagger/swagger.yaml | 33 +++++++++++ tests/e2e/mock-backend/main.go | 11 ++++ tests/e2e/mock_backend_test.go | 72 +++++++++++++++++++++++ 22 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 core/backend/detokenize.go create mode 100644 core/http/endpoints/localai/detokenize.go diff --git a/backend/backend.proto b/backend/backend.proto index bd59159b2..5d926b644 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -35,6 +35,7 @@ service Backend { rpc TTSStream(TTSRequest) returns (stream Reply) {} rpc SoundGeneration(SoundGenerationRequest) returns (Result) {} rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {} + rpc Detokenize(DetokenizeRequest) returns (DetokenizeResponse) {} rpc Status(HealthMessage) returns (StatusResponse) {} rpc Detect(DetectOptions) returns (DetectResponse) {} // SoundDetection runs an audio-tagging / sound-event-classification model @@ -796,6 +797,14 @@ message TokenizationResponse { repeated int32 tokens = 2; } +message DetokenizeRequest { + repeated int32 tokens = 1; +} + +message DetokenizeResponse { + string content = 1; +} + message MemoryUsageData { uint64 total = 1; map breakdown = 2; diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index adb1bf063..67d5ef11f 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -3279,6 +3279,21 @@ public: return grpc::Status::OK; } + grpc::Status Detokenize(ServerContext* context, const backend::DetokenizeRequest* request, backend::DetokenizeResponse* response) override { + auto auth = checkAuth(context); + if (!auth.ok()) return auth; + if (params_base.model.path.empty()) { + return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded"); + } + + std::string content; + for (const auto token : request->tokens()) { + content.append(common_token_to_piece(ctx_server.get_llama_context(), token)); + } + response->set_content(content); + return grpc::Status::OK; + } + grpc::Status GetMetrics(ServerContext* /*context*/, const backend::MetricsRequest* /*request*/, backend::MetricsResponse* response) override { diff --git a/backend/rust/kokoros/src/service.rs b/backend/rust/kokoros/src/service.rs index 80fd3ceb8..f2fbe6cc4 100644 --- a/backend/rust/kokoros/src/service.rs +++ b/backend/rust/kokoros/src/service.rs @@ -415,6 +415,13 @@ impl Backend for KokorosService { Err(Status::unimplemented("Not supported")) } + async fn detokenize( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not supported")) + } + async fn detect( &self, _: Request, diff --git a/core/backend/detokenize.go b/core/backend/detokenize.go new file mode 100644 index 000000000..05c16c4a5 --- /dev/null +++ b/core/backend/detokenize.go @@ -0,0 +1,67 @@ +package backend + +import ( + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/trace" + "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" +) + +func ModelDetokenize(tokens []int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.DetokenizeResponse, error) { + + var inferenceModel grpc.Backend + var err error + + opts := ModelOptions(modelConfig, appConfig) + inferenceModel, err = loader.Load(opts...) + if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) + return schema.DetokenizeResponse{}, err + } + + var startTime time.Time + if appConfig.EnableTracing { + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) + startTime = time.Now() + } + + resp, err := inferenceModel.Detokenize(appConfig.Context, &pb.DetokenizeRequest{Tokens: tokens}) + + if appConfig.EnableTracing { + errStr := "" + if err != nil { + errStr = err.Error() + } + + content := "" + if resp != nil { + content = resp.Content + } + + trace.RecordBackendTrace(trace.BackendTrace{ + Timestamp: startTime, + Duration: time.Since(startTime), + Type: trace.BackendTraceTokenize, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(content, 200), + Error: errStr, + Data: map[string]any{ + "token_count": len(tokens), + "output_text": trace.TruncateString(content, 1000), + }, + }) + } + + if err != nil { + return schema.DetokenizeResponse{}, err + } + + return schema.DetokenizeResponse{ + Content: resp.Content, + }, nil +} diff --git a/core/http/auth/features.go b/core/http/auth/features.go index 063b4d76e..36fdafe12 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -111,6 +111,7 @@ var RouteFeatureRegistry = []RouteFeature{ // Tokenize {"POST", "/v1/tokenize", FeatureTokenize}, + {"POST", "/v1/detokenize", FeatureTokenize}, // Rerank {"POST", "/v1/rerank", FeatureRerank}, diff --git a/core/http/endpoints/localai/detokenize.go b/core/http/endpoints/localai/detokenize.go new file mode 100644 index 000000000..a3ff47963 --- /dev/null +++ b/core/http/endpoints/localai/detokenize.go @@ -0,0 +1,36 @@ +package localai + +import ( + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/model" +) + +// DetokenizeEndpoint exposes a REST API to convert token IDs back to text. +// @Summary Detokenize the input. +// @Tags tokenize +// @Param request body schema.DetokenizeRequest true "Request" +// @Success 200 {object} schema.DetokenizeResponse "Response" +// @Router /v1/detokenize [post] +func DetokenizeEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.DetokenizeRequest) + if !ok || input.Model == "" { + return echo.ErrBadRequest + } + + cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || cfg == nil { + return echo.ErrBadRequest + } + + resp, err := backend.ModelDetokenize(input.Tokens, ml, *cfg, appConfig) + if err != nil { + return err + } + return c.JSON(200, resp) + } +} diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 498e29e8f..f5b34a831 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -348,6 +348,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, "3d_generation": "/3d/generations", "detection": "/v1/detection", "tokenize": "/v1/tokenize", + "detokenize": "/v1/detokenize", }, "monitoring": monitoringRoutes, "mcp": map[string]string{ @@ -417,6 +418,12 @@ func RegisterLocalAIRoutes(router *echo.Echo, requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)), requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.TokenizeRequest) })) + detokenizeHandler := localai.DetokenizeEndpoint(cl, ml, appConfig) + router.POST("/v1/detokenize", + detokenizeHandler, + requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)), + requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.DetokenizeRequest) })) + // MCP endpoint - supports both streaming and non-streaming modes // Note: streaming mode is NOT compatible with the OpenAI apis. We have a set which streams more states. if evaluator != nil && !appConfig.DisableMCP { diff --git a/core/schema/tokenize.go b/core/schema/tokenize.go index 5129b6ab7..f9a2fb13b 100644 --- a/core/schema/tokenize.go +++ b/core/schema/tokenize.go @@ -8,3 +8,12 @@ type TokenizeRequest struct { type TokenizeResponse struct { Tokens []int32 `json:"tokens"` // token IDs } + +type DetokenizeRequest struct { + BasicModelRequest + Tokens []int32 `json:"tokens"` // token IDs to convert back to text +} + +type DetokenizeResponse struct { + Content string `json:"content"` // detokenized text +} diff --git a/core/services/nodes/health_mock_test.go b/core/services/nodes/health_mock_test.go index 5e2571194..b2583fadd 100644 --- a/core/services/nodes/health_mock_test.go +++ b/core/services/nodes/health_mock_test.go @@ -202,6 +202,9 @@ func (c *fakeBackendClient) AudioTranscriptionStream(_ context.Context, _ *pb.Tr func (c *fakeBackendClient) TokenizeString(_ context.Context, _ *pb.PredictOptions, _ ...ggrpc.CallOption) (*pb.TokenizationResponse, error) { return nil, nil } +func (c *fakeBackendClient) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) { + return nil, nil +} func (c *fakeBackendClient) Status(_ context.Context) (*pb.StatusResponse, error) { return nil, nil } diff --git a/core/services/nodes/inflight_test.go b/core/services/nodes/inflight_test.go index a79237a02..98cc03557 100644 --- a/core/services/nodes/inflight_test.go +++ b/core/services/nodes/inflight_test.go @@ -146,6 +146,10 @@ func (f *fakeGRPCBackend) TokenizeString(_ context.Context, _ *pb.PredictOptions return &pb.TokenizationResponse{}, nil } +func (f *fakeGRPCBackend) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) { + return &pb.DetokenizeResponse{}, nil +} + func (f *fakeGRPCBackend) Status(_ context.Context) (*pb.StatusResponse, error) { return &pb.StatusResponse{}, nil } diff --git a/docs/content/features/authentication.md b/docs/content/features/authentication.md index 8f27d6c62..6c206adbb 100644 --- a/docs/content/features/authentication.md +++ b/docs/content/features/authentication.md @@ -179,7 +179,7 @@ When authentication is enabled, the following endpoints require admin role: **User-Accessible Endpoints (all authenticated users):** - `POST /v1/chat/completions`, `POST /v1/embeddings`, `POST /v1/completions` - `POST /v1/images/generations`, `POST /v1/audio/*`, `POST /tts`, `POST /vad`, `POST /video` -- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detection` +- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detokenize`, `POST /v1/detection` - `POST /v1/mcp/chat/completions`, `POST /v1/messages`, `POST /v1/responses` - `POST /stores/*`, `GET /api/cors-proxy` - `GET /version`, `GET /api/features`, `GET /swagger/*`, `GET /metrics` diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index 00e449193..32564655f 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -106,6 +106,7 @@ type ControlBackend interface { HealthCheck(ctx context.Context) (bool, error) LoadModel(ctx context.Context, in *pb.ModelOptions, opts ...grpc.CallOption) (*pb.Result, error) TokenizeString(ctx context.Context, in *pb.PredictOptions, opts ...grpc.CallOption) (*pb.TokenizationResponse, error) + Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error) Status(ctx context.Context) (*pb.StatusResponse, error) StoresSet(ctx context.Context, in *pb.StoresSetOptions, opts ...grpc.CallOption) (*pb.Result, error) diff --git a/pkg/grpc/base/base.go b/pkg/grpc/base/base.go index 0cf8d9162..94e4e7462 100644 --- a/pkg/grpc/base/base.go +++ b/pkg/grpc/base/base.go @@ -123,6 +123,10 @@ func (llm *Base) TokenizeString(opts *pb.PredictOptions) (pb.TokenizationRespons return pb.TokenizationResponse{}, fmt.Errorf("unimplemented") } +func (llm *Base) Detokenize(req *pb.DetokenizeRequest) (pb.DetokenizeResponse, error) { + return pb.DetokenizeResponse{}, fmt.Errorf("unimplemented") +} + func (llm *Base) ModelMetadata(opts *pb.ModelOptions) (*pb.ModelMetadataResponse, error) { return nil, fmt.Errorf("unimplemented") } diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index 6ec3e57a8..d42978c4e 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -426,6 +426,28 @@ func (c *Client) TokenizeString(ctx context.Context, in *pb.PredictOptions, opts return res, nil } +func (c *Client) Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error) { + if !c.parallel { + c.opMutex.Lock() + defer c.opMutex.Unlock() + } + c.setBusy(true) + defer c.setBusy(false) + defer c.wdMark()() + conn, err := c.dial() + if err != nil { + return nil, err + } + defer func() { _ = conn.Close() }() + client := pb.NewBackendClient(conn) + + res, err := client.Detokenize(ctx, in, opts...) + if err != nil { + return nil, err + } + return res, nil +} + func (c *Client) Status(ctx context.Context) (*pb.StatusResponse, error) { if !c.parallel { c.opMutex.Lock() diff --git a/pkg/grpc/embed.go b/pkg/grpc/embed.go index 4c0bb2404..a819d3a53 100644 --- a/pkg/grpc/embed.go +++ b/pkg/grpc/embed.go @@ -117,6 +117,10 @@ func (e *embedBackend) TokenizeString(ctx context.Context, in *pb.PredictOptions return e.s.TokenizeString(ctx, in) } +func (e *embedBackend) Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error) { + return e.s.Detokenize(ctx, in) +} + func (e *embedBackend) Status(ctx context.Context) (*pb.StatusResponse, error) { return e.s.Status(ctx, &pb.HealthMessage{}) } diff --git a/pkg/grpc/interface.go b/pkg/grpc/interface.go index 25df96233..974474546 100644 --- a/pkg/grpc/interface.go +++ b/pkg/grpc/interface.go @@ -33,6 +33,7 @@ type AIModel interface { TTSStream(*pb.TTSRequest, chan []byte) error SoundGeneration(*pb.SoundGenerationRequest) error TokenizeString(*pb.PredictOptions) (pb.TokenizationResponse, error) + Detokenize(*pb.DetokenizeRequest) (pb.DetokenizeResponse, error) Status() (pb.StatusResponse, error) StoresSet(*pb.StoresSetOptions) error diff --git a/pkg/grpc/server.go b/pkg/grpc/server.go index f9d1020e1..903532e4b 100644 --- a/pkg/grpc/server.go +++ b/pkg/grpc/server.go @@ -544,6 +544,18 @@ func (s *server) TokenizeString(ctx context.Context, in *pb.PredictOptions) (*pb }, err } +func (s *server) Detokenize(ctx context.Context, in *pb.DetokenizeRequest) (*pb.DetokenizeResponse, error) { + if s.llm.Locking() { + s.llm.Lock() + defer s.llm.Unlock() + } + res, err := s.llm.Detokenize(in) + if err != nil { + return nil, err + } + return &res, nil +} + func (s *server) Status(ctx context.Context, in *pb.HealthMessage) (*pb.StatusResponse, error) { res, err := s.llm.Status() if err != nil { diff --git a/swagger/docs.go b/swagger/docs.go index 452dc6516..8da82f29c 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -2778,6 +2778,33 @@ const docTemplate = `{ } } }, + "/v1/detokenize": { + "post": { + "tags": [ + "tokenize" + ], + "summary": "Detokenize the input.", + "parameters": [ + { + "description": "Request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.DetokenizeRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.DetokenizeResponse" + } + } + } + } + }, "/v1/edits": { "post": { "tags": [ @@ -4872,6 +4899,30 @@ const docTemplate = `{ } } }, + "schema.DetokenizeRequest": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "tokens": { + "description": "token IDs to convert back to text", + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + "schema.DetokenizeResponse": { + "type": "object", + "properties": { + "content": { + "description": "detokenized text", + "type": "string" + } + } + }, "schema.DiarizationResult": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index 7b452f7cd..9042118a4 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -2775,6 +2775,33 @@ } } }, + "/v1/detokenize": { + "post": { + "tags": [ + "tokenize" + ], + "summary": "Detokenize the input.", + "parameters": [ + { + "description": "Request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.DetokenizeRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.DetokenizeResponse" + } + } + } + } + }, "/v1/edits": { "post": { "tags": [ @@ -4869,6 +4896,30 @@ } } }, + "schema.DetokenizeRequest": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "tokens": { + "description": "token IDs to convert back to text", + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + "schema.DetokenizeResponse": { + "type": "object", + "properties": { + "content": { + "description": "detokenized text", + "type": "string" + } + } + }, "schema.DiarizationResult": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index ebbcf92ca..9c0f7ea32 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -851,6 +851,22 @@ definitions: $ref: '#/definitions/schema.Detection' type: array type: object + schema.DetokenizeRequest: + properties: + model: + type: string + tokens: + description: token IDs to convert back to text + items: + type: integer + type: array + type: object + schema.DetokenizeResponse: + properties: + content: + description: detokenized text + type: string + type: object schema.DiarizationResult: properties: duration: @@ -4617,6 +4633,23 @@ paths: summary: Detects objects in the input image. tags: - detection + /v1/detokenize: + post: + parameters: + - description: Request + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.DetokenizeRequest' + responses: + "200": + description: Response + schema: + $ref: '#/definitions/schema.DetokenizeResponse' + summary: Detokenize the input. + tags: + - tokenize /v1/edits: post: parameters: diff --git a/tests/e2e/mock-backend/main.go b/tests/e2e/mock-backend/main.go index dbe405e10..290205d5e 100644 --- a/tests/e2e/mock-backend/main.go +++ b/tests/e2e/mock-backend/main.go @@ -712,6 +712,17 @@ func extractRouteLabel(candidate string) string { return label } +func (m *MockBackend) Detokenize(ctx context.Context, in *pb.DetokenizeRequest) (*pb.DetokenizeResponse, error) { + xlog.Debug("Detokenize called", "tokens", in.Tokens) + parts := make([]string, len(in.Tokens)) + for i, t := range in.Tokens { + parts[i] = strconv.Itoa(int(t)) + } + return &pb.DetokenizeResponse{ + Content: "detokenized: " + strings.Join(parts, " "), + }, nil +} + func (m *MockBackend) Status(ctx context.Context, in *pb.HealthMessage) (*pb.StatusResponse, error) { xlog.Debug("Status called") return &pb.StatusResponse{ diff --git a/tests/e2e/mock_backend_test.go b/tests/e2e/mock_backend_test.go index 24f4cbd94..e95dd875e 100644 --- a/tests/e2e/mock_backend_test.go +++ b/tests/e2e/mock_backend_test.go @@ -387,6 +387,78 @@ var _ = Describe("Mock Backend E2E Tests", Label("MockBackend"), func() { }) }) + Describe("Detokenization API", func() { + It("should return content for known token IDs", func() { + body := `{"model":"mock-model","tokens":[101,2023,2003,1037,3231,1012]}` + req, err := http.NewRequest("POST", apiURL+"/detokenize", strings.NewReader(body)) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + + httpClient := &http.Client{Timeout: 30 * time.Second} + resp, err := httpClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(200)) + + data, err := io.ReadAll(resp.Body) + Expect(err).ToNot(HaveOccurred()) + var result map[string]any + Expect(json.Unmarshal(data, &result)).To(Succeed()) + content, ok := result["content"].(string) + Expect(ok).To(BeTrue(), "response missing 'content' field: %s", string(data)) + Expect(content).ToNot(BeEmpty()) + }) + + It("should round-trip tokenize then detokenize", func() { + httpClient := &http.Client{Timeout: 30 * time.Second} + + // Step 1: tokenize + tokenizeReq, err := http.NewRequest("POST", apiURL+"/tokenize", + strings.NewReader(`{"model":"mock-model","content":"Hello world"}`)) + Expect(err).ToNot(HaveOccurred()) + tokenizeReq.Header.Set("Content-Type", "application/json") + + tokenizeResp, err := httpClient.Do(tokenizeReq) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = tokenizeResp.Body.Close() }() + Expect(tokenizeResp.StatusCode).To(Equal(200)) + + tokenizeData, err := io.ReadAll(tokenizeResp.Body) + Expect(err).ToNot(HaveOccurred()) + var tokenizeResult map[string]any + Expect(json.Unmarshal(tokenizeData, &tokenizeResult)).To(Succeed()) + + tokensRaw, ok := tokenizeResult["tokens"].([]any) + Expect(ok).To(BeTrue(), "tokenize response missing 'tokens': %s", string(tokenizeData)) + Expect(tokensRaw).ToNot(BeEmpty()) + + // Step 2: detokenize the returned token IDs + tokens := make([]int, len(tokensRaw)) + for i, t := range tokensRaw { + tokens[i] = int(t.(float64)) + } + tokenJSON, err := json.Marshal(map[string]any{"model": "mock-model", "tokens": tokens}) + Expect(err).ToNot(HaveOccurred()) + + detokenizeReq, err := http.NewRequest("POST", apiURL+"/detokenize", strings.NewReader(string(tokenJSON))) + Expect(err).ToNot(HaveOccurred()) + detokenizeReq.Header.Set("Content-Type", "application/json") + + detokenizeResp, err := httpClient.Do(detokenizeReq) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = detokenizeResp.Body.Close() }() + Expect(detokenizeResp.StatusCode).To(Equal(200)) + + detokenizeData, err := io.ReadAll(detokenizeResp.Body) + Expect(err).ToNot(HaveOccurred()) + var detokenizeResult map[string]any + Expect(json.Unmarshal(detokenizeData, &detokenizeResult)).To(Succeed()) + content, ok := detokenizeResult["content"].(string) + Expect(ok).To(BeTrue(), "detokenize response missing 'content': %s", string(detokenizeData)) + Expect(content).ToNot(BeEmpty()) + }) + }) + Describe("Autoparser ChatDelta Streaming", Label("Autoparser"), func() { // These tests verify that when the C++ autoparser handles tool calls // and content via ChatDeltas (with empty raw message), the streaming