feat(api): add /v1/detokenize endpoint (#9620)

* feat(api): add /v1/detokenize endpoint Closes #1649. Mirror of the existing /v1/tokenize path, requested by @benniekiss in the issue thread for "complete API workflow" use cases that need to turn token IDs back into text without local processing. - Add Detokenize gRPC RPC with DetokenizeRequest{tokens} / DetokenizeResponse{content} messages. - Implement in the llama.cpp backend using common_token_to_piece, the same primitive TokenizeString already uses internally. - Other backends inherit the default Unimplemented from base.Base, in line with how Detect, Rerank, etc. are gated per-backend. - Wire up the Go gRPC interface, server, client, and in-process embed wrapper alongside their TokenizeString counterparts. - Add the schema types, ModelDetokenize wrapper, HTTP handler, route registration, RouteFeatureRegistry entry (gated by FeatureTokenize so no new feature flag is needed), and the discovery map entry under ai_functions. - Regenerated swagger reflects the new endpoint and types. - Update authentication.md to list /v1/detokenize alongside /v1/tokenize. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Adira Denis Muhando <dennisadira@gmail.com> * test(e2e): add mock backend tests for /v1/detokenize Add Detokenize to the mock gRPC backend and wire up two e2e tests in the MockBackend suite: one that posts known token IDs and asserts a non-empty content response, and a round-trip that tokenizes first then detokenizes the returned IDs. Addresses reviewer feedback on #9620. Assisted-by: Claude:claude-sonnet-4-6 Signed-off-by: Adira Denis Muhando <dennisadira@gmail.com> * fix(kokoros): implement detokenize in the Rust backend service The Detokenize RPC added in this PR grows the tonic-generated Backend trait. Unlike the other languages there is nothing to inherit a default from — Rust trait impls must list every method — so backend/rust/kokoros failed to compile: error[E0046]: not all trait items implemented, missing: `detokenize` --> src/service.rs:72:1 72 | impl Backend for KokorosService { Go backends pick up the Unimplemented default from base.Base, and the generated C++/Python servicer bases default to UNIMPLEMENTED, which is why the Rust backend was the only one that broke. kokoros is the sole Rust crate in the tree, so this is the full extent of the fallout. Return Status::unimplemented("Not supported"), matching how this same file already gates tokenize_string and ~20 other unsupported RPCs. Fixes the tests-kokoros and backend-jobs-singlearch-4 (-cpu-kokoros) failures on the previous head. Assisted-by: Claude:claude-opus-5 cargo Signed-off-by: Adira Denis Muhando <dennisadira@gmail.com> --------- Signed-off-by: Adira Denis Muhando <dennisadira@gmail.com> Co-authored-by: localai-org-maint-bot <bot-opensource@localaisrl.com>
2026-08-02 11:30:44 -04:00 · 2026-07-30 17:01:47 +03:00
parent 965180581b
commit ef724a3c9d
22 changed files with 421 additions and 1 deletions
--- a/backend/backend.proto
+++ b/backend/backend.proto
@@ -35,6 +35,7 @@ service Backend {
  rpc TTSStream(TTSRequest) returns (stream Reply) {}
  rpc SoundGeneration(SoundGenerationRequest) returns (Result) {}
  rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {}
+  rpc Detokenize(DetokenizeRequest) returns (DetokenizeResponse) {}
  rpc Status(HealthMessage) returns (StatusResponse) {}
  rpc Detect(DetectOptions) returns (DetectResponse) {}
  // SoundDetection runs an audio-tagging / sound-event-classification model
@@ -796,6 +797,14 @@ message TokenizationResponse {
  repeated int32 tokens = 2;
 }

+message DetokenizeRequest {
+  repeated int32 tokens = 1;
+}
+
+message DetokenizeResponse {
+  string content = 1;
+}
+
 message MemoryUsageData {
  uint64 total = 1;
  map<string, uint64> breakdown = 2;
--- a/backend/cpp/llama-cpp/grpc-server.cpp
+++ b/backend/cpp/llama-cpp/grpc-server.cpp
@@ -3279,6 +3279,21 @@ public:
        return grpc::Status::OK;
    }

+    grpc::Status Detokenize(ServerContext* context, const backend::DetokenizeRequest* request, backend::DetokenizeResponse* response) override {
+        auto auth = checkAuth(context);
+        if (!auth.ok()) return auth;
+        if (params_base.model.path.empty()) {
+            return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
+        }
+
+        std::string content;
+        for (const auto token : request->tokens()) {
+            content.append(common_token_to_piece(ctx_server.get_llama_context(), token));
+        }
+        response->set_content(content);
+        return grpc::Status::OK;
+    }
+
    grpc::Status GetMetrics(ServerContext* /*context*/, const backend::MetricsRequest* /*request*/, backend::MetricsResponse* response) override {


--- a/backend/rust/kokoros/src/service.rs
+++ b/backend/rust/kokoros/src/service.rs
@@ -415,6 +415,13 @@ impl Backend for KokorosService {
        Err(Status::unimplemented("Not supported"))
    }

+    async fn detokenize(
+        &self,
+        _: Request<backend::DetokenizeRequest>,
+    ) -> Result<Response<backend::DetokenizeResponse>, Status> {
+        Err(Status::unimplemented("Not supported"))
+    }
+
    async fn detect(
        &self,
        _: Request<backend::DetectOptions>,
--- a/core/backend/detokenize.go
+++ b/core/backend/detokenize.go
@@ -0,0 +1,67 @@
+package backend
+
+import (
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/core/trace"
+	"github.com/mudler/LocalAI/pkg/grpc"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/model"
+)
+
+func ModelDetokenize(tokens []int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.DetokenizeResponse, error) {
+
+	var inferenceModel grpc.Backend
+	var err error
+
+	opts := ModelOptions(modelConfig, appConfig)
+	inferenceModel, err = loader.Load(opts...)
+	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
+		return schema.DetokenizeResponse{}, err
+	}
+
+	var startTime time.Time
+	if appConfig.EnableTracing {
+		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes)
+		startTime = time.Now()
+	}
+
+	resp, err := inferenceModel.Detokenize(appConfig.Context, &pb.DetokenizeRequest{Tokens: tokens})
+
+	if appConfig.EnableTracing {
+		errStr := ""
+		if err != nil {
+			errStr = err.Error()
+		}
+
+		content := ""
+		if resp != nil {
+			content = resp.Content
+		}
+
+		trace.RecordBackendTrace(trace.BackendTrace{
+			Timestamp: startTime,
+			Duration:  time.Since(startTime),
+			Type:      trace.BackendTraceTokenize,
+			ModelName: modelConfig.Name,
+			Backend:   modelConfig.Backend,
+			Summary:   trace.TruncateString(content, 200),
+			Error:     errStr,
+			Data: map[string]any{
+				"token_count": len(tokens),
+				"output_text": trace.TruncateString(content, 1000),
+			},
+		})
+	}
+
+	if err != nil {
+		return schema.DetokenizeResponse{}, err
+	}
+
+	return schema.DetokenizeResponse{
+		Content: resp.Content,
+	}, nil
+}
--- a/core/http/auth/features.go
+++ b/core/http/auth/features.go
@@ -111,6 +111,7 @@ var RouteFeatureRegistry = []RouteFeature{

 	// Tokenize
 	{"POST", "/v1/tokenize", FeatureTokenize},
+	{"POST", "/v1/detokenize", FeatureTokenize},

 	// Rerank
 	{"POST", "/v1/rerank", FeatureRerank},
--- a/core/http/endpoints/localai/detokenize.go
+++ b/core/http/endpoints/localai/detokenize.go
@@ -0,0 +1,36 @@
+package localai
+
+import (
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/backend"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/middleware"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/model"
+)
+
+// DetokenizeEndpoint exposes a REST API to convert token IDs back to text.
+// @Summary Detokenize the input.
+// @Tags tokenize
+// @Param request body schema.DetokenizeRequest true "Request"
+// @Success 200 {object} schema.DetokenizeResponse "Response"
+// @Router /v1/detokenize [post]
+func DetokenizeEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.DetokenizeRequest)
+		if !ok || input.Model == "" {
+			return echo.ErrBadRequest
+		}
+
+		cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
+		if !ok || cfg == nil {
+			return echo.ErrBadRequest
+		}
+
+		resp, err := backend.ModelDetokenize(input.Tokens, ml, *cfg, appConfig)
+		if err != nil {
+			return err
+		}
+		return c.JSON(200, resp)
+	}
+}
--- a/core/http/routes/localai.go
+++ b/core/http/routes/localai.go
@@ -348,6 +348,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 					"3d_generation":  "/3d/generations",
 					"detection":      "/v1/detection",
 					"tokenize":       "/v1/tokenize",
+					"detokenize":     "/v1/detokenize",
 				},
 				"monitoring": monitoringRoutes,
 				"mcp": map[string]string{
@@ -417,6 +418,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 		requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
 		requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.TokenizeRequest) }))

+	detokenizeHandler := localai.DetokenizeEndpoint(cl, ml, appConfig)
+	router.POST("/v1/detokenize",
+		detokenizeHandler,
+		requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TOKENIZE)),
+		requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.DetokenizeRequest) }))
+
 	// MCP endpoint - supports both streaming and non-streaming modes
 	// Note: streaming mode is NOT compatible with the OpenAI apis. We have a set which streams more states.
 	if evaluator != nil && !appConfig.DisableMCP {
--- a/core/schema/tokenize.go
+++ b/core/schema/tokenize.go
@@ -8,3 +8,12 @@ type TokenizeRequest struct {
 type TokenizeResponse struct {
 	Tokens []int32 `json:"tokens"` // token IDs
 }
+
+type DetokenizeRequest struct {
+	BasicModelRequest
+	Tokens []int32 `json:"tokens"` // token IDs to convert back to text
+}
+
+type DetokenizeResponse struct {
+	Content string `json:"content"` // detokenized text
+}
--- a/core/services/nodes/health_mock_test.go
+++ b/core/services/nodes/health_mock_test.go
@@ -202,6 +202,9 @@ func (c *fakeBackendClient) AudioTranscriptionStream(_ context.Context, _ *pb.Tr
 func (c *fakeBackendClient) TokenizeString(_ context.Context, _ *pb.PredictOptions, _ ...ggrpc.CallOption) (*pb.TokenizationResponse, error) {
 	return nil, nil
 }
+func (c *fakeBackendClient) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
+	return nil, nil
+}
 func (c *fakeBackendClient) Status(_ context.Context) (*pb.StatusResponse, error) {
 	return nil, nil
 }
--- a/core/services/nodes/inflight_test.go
+++ b/core/services/nodes/inflight_test.go
@@ -146,6 +146,10 @@ func (f *fakeGRPCBackend) TokenizeString(_ context.Context, _ *pb.PredictOptions
 	return &pb.TokenizationResponse{}, nil
 }

+func (f *fakeGRPCBackend) Detokenize(_ context.Context, _ *pb.DetokenizeRequest, _ ...ggrpc.CallOption) (*pb.DetokenizeResponse, error) {
+	return &pb.DetokenizeResponse{}, nil
+}
+
 func (f *fakeGRPCBackend) Status(_ context.Context) (*pb.StatusResponse, error) {
 	return &pb.StatusResponse{}, nil
 }
--- a/docs/content/features/authentication.md
+++ b/docs/content/features/authentication.md
@@ -179,7 +179,7 @@ When authentication is enabled, the following endpoints require admin role:
 **User-Accessible Endpoints (all authenticated users):**
 - `POST /v1/chat/completions`, `POST /v1/embeddings`, `POST /v1/completions`
 - `POST /v1/images/generations`, `POST /v1/audio/*`, `POST /tts`, `POST /vad`, `POST /video`
- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detection`
+- `GET /v1/models`, `POST /v1/tokenize`, `POST /v1/detokenize`, `POST /v1/detection`
 - `POST /v1/mcp/chat/completions`, `POST /v1/messages`, `POST /v1/responses`
 - `POST /stores/*`, `GET /api/cors-proxy`
 - `GET /version`, `GET /api/features`, `GET /swagger/*`, `GET /metrics`
--- a/pkg/grpc/backend.go
+++ b/pkg/grpc/backend.go
@@ -106,6 +106,7 @@ type ControlBackend interface {
 	HealthCheck(ctx context.Context) (bool, error)
 	LoadModel(ctx context.Context, in *pb.ModelOptions, opts ...grpc.CallOption) (*pb.Result, error)
 	TokenizeString(ctx context.Context, in *pb.PredictOptions, opts ...grpc.CallOption) (*pb.TokenizationResponse, error)
+	Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error)
 	Status(ctx context.Context) (*pb.StatusResponse, error)

 	StoresSet(ctx context.Context, in *pb.StoresSetOptions, opts ...grpc.CallOption) (*pb.Result, error)
--- a/pkg/grpc/base/base.go
+++ b/pkg/grpc/base/base.go
@@ -123,6 +123,10 @@ func (llm *Base) TokenizeString(opts *pb.PredictOptions) (pb.TokenizationRespons
 	return pb.TokenizationResponse{}, fmt.Errorf("unimplemented")
 }

+func (llm *Base) Detokenize(req *pb.DetokenizeRequest) (pb.DetokenizeResponse, error) {
+	return pb.DetokenizeResponse{}, fmt.Errorf("unimplemented")
+}
+
 func (llm *Base) ModelMetadata(opts *pb.ModelOptions) (*pb.ModelMetadataResponse, error) {
 	return nil, fmt.Errorf("unimplemented")
 }
--- a/pkg/grpc/client.go
+++ b/pkg/grpc/client.go
@@ -426,6 +426,28 @@ func (c *Client) TokenizeString(ctx context.Context, in *pb.PredictOptions, opts
 	return res, nil
 }

+func (c *Client) Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error) {
+	if !c.parallel {
+		c.opMutex.Lock()
+		defer c.opMutex.Unlock()
+	}
+	c.setBusy(true)
+	defer c.setBusy(false)
+	defer c.wdMark()()
+	conn, err := c.dial()
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = conn.Close() }()
+	client := pb.NewBackendClient(conn)
+
+	res, err := client.Detokenize(ctx, in, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
 func (c *Client) Status(ctx context.Context) (*pb.StatusResponse, error) {
 	if !c.parallel {
 		c.opMutex.Lock()
--- a/pkg/grpc/embed.go
+++ b/pkg/grpc/embed.go
@@ -117,6 +117,10 @@ func (e *embedBackend) TokenizeString(ctx context.Context, in *pb.PredictOptions
 	return e.s.TokenizeString(ctx, in)
 }

+func (e *embedBackend) Detokenize(ctx context.Context, in *pb.DetokenizeRequest, opts ...grpc.CallOption) (*pb.DetokenizeResponse, error) {
+	return e.s.Detokenize(ctx, in)
+}
+
 func (e *embedBackend) Status(ctx context.Context) (*pb.StatusResponse, error) {
 	return e.s.Status(ctx, &pb.HealthMessage{})
 }
--- a/pkg/grpc/interface.go
+++ b/pkg/grpc/interface.go
@@ -33,6 +33,7 @@ type AIModel interface {
 	TTSStream(*pb.TTSRequest, chan []byte) error
 	SoundGeneration(*pb.SoundGenerationRequest) error
 	TokenizeString(*pb.PredictOptions) (pb.TokenizationResponse, error)
+	Detokenize(*pb.DetokenizeRequest) (pb.DetokenizeResponse, error)
 	Status() (pb.StatusResponse, error)

 	StoresSet(*pb.StoresSetOptions) error
--- a/pkg/grpc/server.go
+++ b/pkg/grpc/server.go
@@ -544,6 +544,18 @@ func (s *server) TokenizeString(ctx context.Context, in *pb.PredictOptions) (*pb
 	}, err
 }

+func (s *server) Detokenize(ctx context.Context, in *pb.DetokenizeRequest) (*pb.DetokenizeResponse, error) {
+	if s.llm.Locking() {
+		s.llm.Lock()
+		defer s.llm.Unlock()
+	}
+	res, err := s.llm.Detokenize(in)
+	if err != nil {
+		return nil, err
+	}
+	return &res, nil
+}
+
 func (s *server) Status(ctx context.Context, in *pb.HealthMessage) (*pb.StatusResponse, error) {
 	res, err := s.llm.Status()
 	if err != nil {
--- a/swagger/docs.go
+++ b/swagger/docs.go
@@ -2778,6 +2778,33 @@ const docTemplate = `{
                }
            }
        },
+        "/v1/detokenize": {
+            "post": {
+                "tags": [
+                    "tokenize"
+                ],
+                "summary": "Detokenize the input.",
+                "parameters": [
+                    {
+                        "description": "Request",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.DetokenizeRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.DetokenizeResponse"
+                        }
+                    }
+                }
+            }
+        },
        "/v1/edits": {
            "post": {
                "tags": [
@@ -4872,6 +4899,30 @@ const docTemplate = `{
                }
            }
        },
+        "schema.DetokenizeRequest": {
+            "type": "object",
+            "properties": {
+                "model": {
+                    "type": "string"
+                },
+                "tokens": {
+                    "description": "token IDs to convert back to text",
+                    "type": "array",
+                    "items": {
+                        "type": "integer"
+                    }
+                }
+            }
+        },
+        "schema.DetokenizeResponse": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "description": "detokenized text",
+                    "type": "string"
+                }
+            }
+        },
        "schema.DiarizationResult": {
            "type": "object",
            "properties": {
--- a/swagger/swagger.json
+++ b/swagger/swagger.json
@@ -2775,6 +2775,33 @@
                }
            }
        },
+        "/v1/detokenize": {
+            "post": {
+                "tags": [
+                    "tokenize"
+                ],
+                "summary": "Detokenize the input.",
+                "parameters": [
+                    {
+                        "description": "Request",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.DetokenizeRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.DetokenizeResponse"
+                        }
+                    }
+                }
+            }
+        },
        "/v1/edits": {
            "post": {
                "tags": [
@@ -4869,6 +4896,30 @@
                }
            }
        },
+        "schema.DetokenizeRequest": {
+            "type": "object",
+            "properties": {
+                "model": {
+                    "type": "string"
+                },
+                "tokens": {
+                    "description": "token IDs to convert back to text",
+                    "type": "array",
+                    "items": {
+                        "type": "integer"
+                    }
+                }
+            }
+        },
+        "schema.DetokenizeResponse": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "description": "detokenized text",
+                    "type": "string"
+                }
+            }
+        },
        "schema.DiarizationResult": {
            "type": "object",
            "properties": {
--- a/swagger/swagger.yaml
+++ b/swagger/swagger.yaml
@@ -851,6 +851,22 @@ definitions:
          $ref: '#/definitions/schema.Detection'
        type: array
    type: object
+  schema.DetokenizeRequest:
+    properties:
+      model:
+        type: string
+      tokens:
+        description: token IDs to convert back to text
+        items:
+          type: integer
+        type: array
+    type: object
+  schema.DetokenizeResponse:
+    properties:
+      content:
+        description: detokenized text
+        type: string
+    type: object
  schema.DiarizationResult:
    properties:
      duration:
@@ -4617,6 +4633,23 @@ paths:
      summary: Detects objects in the input image.
      tags:
      - detection
+  /v1/detokenize:
+    post:
+      parameters:
+      - description: Request
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.DetokenizeRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.DetokenizeResponse'
+      summary: Detokenize the input.
+      tags:
+      - tokenize
  /v1/edits:
    post:
      parameters:
--- a/tests/e2e/mock-backend/main.go
+++ b/tests/e2e/mock-backend/main.go
@@ -712,6 +712,17 @@ func extractRouteLabel(candidate string) string {
 	return label
 }

+func (m *MockBackend) Detokenize(ctx context.Context, in *pb.DetokenizeRequest) (*pb.DetokenizeResponse, error) {
+	xlog.Debug("Detokenize called", "tokens", in.Tokens)
+	parts := make([]string, len(in.Tokens))
+	for i, t := range in.Tokens {
+		parts[i] = strconv.Itoa(int(t))
+	}
+	return &pb.DetokenizeResponse{
+		Content: "detokenized: " + strings.Join(parts, " "),
+	}, nil
+}
+
 func (m *MockBackend) Status(ctx context.Context, in *pb.HealthMessage) (*pb.StatusResponse, error) {
 	xlog.Debug("Status called")
 	return &pb.StatusResponse{
--- a/tests/e2e/mock_backend_test.go
+++ b/tests/e2e/mock_backend_test.go
@@ -387,6 +387,78 @@ var _ = Describe("Mock Backend E2E Tests", Label("MockBackend"), func() {
 		})
 	})

+	Describe("Detokenization API", func() {
+		It("should return content for known token IDs", func() {
+			body := `{"model":"mock-model","tokens":[101,2023,2003,1037,3231,1012]}`
+			req, err := http.NewRequest("POST", apiURL+"/detokenize", strings.NewReader(body))
+			Expect(err).ToNot(HaveOccurred())
+			req.Header.Set("Content-Type", "application/json")
+
+			httpClient := &http.Client{Timeout: 30 * time.Second}
+			resp, err := httpClient.Do(req)
+			Expect(err).ToNot(HaveOccurred())
+			defer func() { _ = resp.Body.Close() }()
+			Expect(resp.StatusCode).To(Equal(200))
+
+			data, err := io.ReadAll(resp.Body)
+			Expect(err).ToNot(HaveOccurred())
+			var result map[string]any
+			Expect(json.Unmarshal(data, &result)).To(Succeed())
+			content, ok := result["content"].(string)
+			Expect(ok).To(BeTrue(), "response missing 'content' field: %s", string(data))
+			Expect(content).ToNot(BeEmpty())
+		})
+
+		It("should round-trip tokenize then detokenize", func() {
+			httpClient := &http.Client{Timeout: 30 * time.Second}
+
+			// Step 1: tokenize
+			tokenizeReq, err := http.NewRequest("POST", apiURL+"/tokenize",
+				strings.NewReader(`{"model":"mock-model","content":"Hello world"}`))
+			Expect(err).ToNot(HaveOccurred())
+			tokenizeReq.Header.Set("Content-Type", "application/json")
+
+			tokenizeResp, err := httpClient.Do(tokenizeReq)
+			Expect(err).ToNot(HaveOccurred())
+			defer func() { _ = tokenizeResp.Body.Close() }()
+			Expect(tokenizeResp.StatusCode).To(Equal(200))
+
+			tokenizeData, err := io.ReadAll(tokenizeResp.Body)
+			Expect(err).ToNot(HaveOccurred())
+			var tokenizeResult map[string]any
+			Expect(json.Unmarshal(tokenizeData, &tokenizeResult)).To(Succeed())
+
+			tokensRaw, ok := tokenizeResult["tokens"].([]any)
+			Expect(ok).To(BeTrue(), "tokenize response missing 'tokens': %s", string(tokenizeData))
+			Expect(tokensRaw).ToNot(BeEmpty())
+
+			// Step 2: detokenize the returned token IDs
+			tokens := make([]int, len(tokensRaw))
+			for i, t := range tokensRaw {
+				tokens[i] = int(t.(float64))
+			}
+			tokenJSON, err := json.Marshal(map[string]any{"model": "mock-model", "tokens": tokens})
+			Expect(err).ToNot(HaveOccurred())
+
+			detokenizeReq, err := http.NewRequest("POST", apiURL+"/detokenize", strings.NewReader(string(tokenJSON)))
+			Expect(err).ToNot(HaveOccurred())
+			detokenizeReq.Header.Set("Content-Type", "application/json")
+
+			detokenizeResp, err := httpClient.Do(detokenizeReq)
+			Expect(err).ToNot(HaveOccurred())
+			defer func() { _ = detokenizeResp.Body.Close() }()
+			Expect(detokenizeResp.StatusCode).To(Equal(200))
+
+			detokenizeData, err := io.ReadAll(detokenizeResp.Body)
+			Expect(err).ToNot(HaveOccurred())
+			var detokenizeResult map[string]any
+			Expect(json.Unmarshal(detokenizeData, &detokenizeResult)).To(Succeed())
+			content, ok := detokenizeResult["content"].(string)
+			Expect(ok).To(BeTrue(), "detokenize response missing 'content': %s", string(detokenizeData))
+			Expect(content).ToNot(BeEmpty())
+		})
+	})
+
 	Describe("Autoparser ChatDelta Streaming", Label("Autoparser"), func() {
 		// These tests verify that when the C++ autoparser handles tool calls
 		// and content via ChatDeltas (with empty raw message), the streaming