diff --git a/core/http/auth/features.go b/core/http/auth/features.go index dac7259fe..d83c9b25d 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -118,6 +118,10 @@ var RouteFeatureRegistry = []RouteFeature{ // Rerank {"POST", "/v1/rerank", FeatureRerank}, + // Moderation + {"POST", "/v1/moderations", FeatureModeration}, + {"POST", "/moderations", FeatureModeration}, + // Stores {"POST", "/stores/set", FeatureStores}, {"POST", "/stores/delete", FeatureStores}, @@ -193,6 +197,7 @@ func APIFeatureMetas() []FeatureMeta { {FeatureEmbeddings, "Embeddings", true}, {FeatureSound, "Sound Generation", true}, {FeatureRealtime, "Realtime", true}, + {FeatureModeration, "Moderation", true}, {FeatureRerank, "Rerank", true}, {FeatureTokenize, "Tokenize", true}, {FeatureMCP, "MCP", true}, diff --git a/core/http/auth/features_moderation_test.go b/core/http/auth/features_moderation_test.go new file mode 100644 index 000000000..8f8b63f11 --- /dev/null +++ b/core/http/auth/features_moderation_test.go @@ -0,0 +1,24 @@ +package auth_test + +import ( + . "github.com/mudler/LocalAI/core/http/auth" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Moderation feature registration", func() { + It("registers both moderation routes as default-on API features", func() { + Expect(APIFeatures).To(ContainElement(FeatureModeration)) + + patterns := []string{} + for _, route := range RouteFeatureRegistry { + if route.Feature == FeatureModeration { + patterns = append(patterns, route.Pattern) + } + } + Expect(patterns).To(ConsistOf("/v1/moderations", "/moderations")) + + metas := APIFeatureMetas() + Expect(metas).To(ContainElement(FeatureMeta{Key: FeatureModeration, Label: "Moderation", DefaultValue: true})) + }) +}) diff --git a/core/http/auth/helpers_test.go b/core/http/auth/helpers_test.go index 1fcf9e449..9df64cd13 100644 --- a/core/http/auth/helpers_test.go +++ b/core/http/auth/helpers_test.go @@ -59,10 +59,14 @@ func ok(c echo.Context) error { func newAuthTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo { e := echo.New() e.Use(auth.Middleware(db, appConfig)) + if db != nil { + e.Use(auth.RequireRouteFeature(db)) + } // API routes (require auth) e.GET("/v1/models", ok) e.POST("/v1/chat/completions", ok) + e.POST("/v1/moderations", ok) e.GET("/api/settings", ok) e.POST("/api/settings", ok) @@ -81,10 +85,14 @@ func newAuthTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo func newAdminTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo { e := echo.New() e.Use(auth.Middleware(db, appConfig)) + if db != nil { + e.Use(auth.RequireRouteFeature(db)) + } // Regular routes e.GET("/v1/models", ok) e.POST("/v1/chat/completions", ok) + e.POST("/v1/moderations", ok) // Admin-only routes adminMw := auth.RequireAdmin() diff --git a/core/http/auth/middleware_test.go b/core/http/auth/middleware_test.go index 94be9336a..7f919cc8c 100644 --- a/core/http/auth/middleware_test.go +++ b/core/http/auth/middleware_test.go @@ -91,6 +91,19 @@ var _ = Describe("Auth Middleware", func() { Expect(rec.Code).To(Equal(http.StatusOK)) }) + It("allows authenticated users to call moderation by default", func() { + sessionID := createTestSession(db, user.ID) + rec := doRequest(app, http.MethodPost, "/v1/moderations", withSessionCookie(sessionID)) + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + + It("blocks moderation when the user's feature is disabled", func() { + Expect(auth.UpdateUserPermissions(db, user.ID, auth.PermissionMap{auth.FeatureModeration: false})).To(Succeed()) + sessionID := createTestSession(db, user.ID) + rec := doRequest(app, http.MethodPost, "/v1/moderations", withSessionCookie(sessionID)) + Expect(rec.Code).To(Equal(http.StatusForbidden)) + }) + It("allows requests with valid session as Bearer token", func() { sessionID := createTestSession(db, user.ID) rec := doRequest(app, http.MethodGet, "/v1/models", withBearerToken(sessionID)) @@ -156,6 +169,11 @@ var _ = Describe("Auth Middleware", func() { Expect(rec.Code).To(Equal(http.StatusUnauthorized)) }) + It("returns 401 for unauthenticated moderation requests", func() { + rec := doRequest(app, http.MethodPost, "/v1/moderations") + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + }) + It("returns 401 for unauthenticated 3D generation requests", func() { rec := doRequest(app, http.MethodPost, "/3d/generations") Expect(rec.Code).To(Equal(http.StatusUnauthorized)) diff --git a/core/http/auth/permissions.go b/core/http/auth/permissions.go index 6c01e03a3..95e76f572 100644 --- a/core/http/auth/permissions.go +++ b/core/http/auth/permissions.go @@ -51,6 +51,7 @@ const ( FeatureEmbeddings = "embeddings" FeatureSound = "sound" FeatureRealtime = "realtime" + FeatureModeration = "moderation" FeatureRerank = "rerank" FeatureTokenize = "tokenize" FeatureMCP = "mcp" @@ -75,7 +76,7 @@ var APIFeatures = []string{ FeatureChat, FeatureImages, FeatureAudioSpeech, FeatureAudioTranscription, FeatureAudioDiarization, FeatureAudioClassification, FeatureVAD, FeatureDetection, FeatureVideo, Feature3D, FeatureEmbeddings, FeatureSound, - FeatureRealtime, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores, + FeatureRealtime, FeatureModeration, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores, FeatureFaceRecognition, FeatureVoiceRecognition, FeatureAudioTransform, FeaturePIIFilter, } diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 62f264cf0..c8a5fb74b 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -30,6 +30,12 @@ var instructionDefs = []instructionDef{ Tags: []string{"inference", "embeddings"}, Intro: "Set \"stream\": true for SSE streaming. Supports tool/function calling when the model config has function templates configured.", }, + { + Name: "moderation", + Description: "OpenAI-compatible text moderation using a local completion model", + Tags: []string{"moderation"}, + Intro: "POST /v1/moderations accepts a text string or array plus a LocalAI completion model. LocalAI constrains the model to the OpenAI moderation category schema and returns one result per input. Multimodal moderation inputs are not yet supported.", + }, { Name: "audio", Description: "Text-to-speech, voice activity detection, transcription, speaker diarization, sound classification, and sound generation", diff --git a/core/http/endpoints/localai/api_instructions_test.go b/core/http/endpoints/localai/api_instructions_test.go index 9648e674a..710d4d982 100644 --- a/core/http/endpoints/localai/api_instructions_test.go +++ b/core/http/endpoints/localai/api_instructions_test.go @@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() { instructions, ok := resp["instructions"].([]any) Expect(ok).To(BeTrue()) - Expect(instructions).To(HaveLen(18)) + Expect(instructions).To(HaveLen(19)) // Verify each instruction has required fields and correct URL format for _, s := range instructions { @@ -69,6 +69,7 @@ var _ = Describe("API Instructions Endpoints", func() { Expect(names).To(ContainElements( "chat-inference", + "moderation", "config-management", "model-management", "monitoring", diff --git a/core/http/endpoints/mcp/localai_assistant_test.go b/core/http/endpoints/mcp/localai_assistant_test.go index 2231350d0..213f6fa0c 100644 --- a/core/http/endpoints/mcp/localai_assistant_test.go +++ b/core/http/endpoints/mcp/localai_assistant_test.go @@ -84,6 +84,22 @@ func (stubClient) ListNodes(_ context.Context) ([]localaitools.Node, error) { return []localaitools.Node{}, nil } +func (stubClient) ListScheduling(_ context.Context) ([]localaitools.ModelSchedulingConfig, error) { + return []localaitools.ModelSchedulingConfig{}, nil +} + +func (stubClient) GetScheduling(_ context.Context, _ string) (*localaitools.ModelSchedulingConfig, error) { + return &localaitools.ModelSchedulingConfig{}, nil +} + +func (stubClient) SetScheduling(_ context.Context, _ localaitools.SetSchedulingRequest) (*localaitools.ModelSchedulingConfig, error) { + return &localaitools.ModelSchedulingConfig{}, nil +} + +func (stubClient) DeleteScheduling(_ context.Context, _ string) error { + return nil +} + func (stubClient) SetNodeVRAMBudget(_ context.Context, _, _ string) error { return nil } diff --git a/core/http/endpoints/openai/moderations.go b/core/http/endpoints/openai/moderations.go new file mode 100644 index 000000000..a42403686 --- /dev/null +++ b/core/http/endpoints/openai/moderations.go @@ -0,0 +1,190 @@ +package openai + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "strings" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/templates" + "github.com/mudler/LocalAI/pkg/functions" + "github.com/mudler/LocalAI/pkg/model" +) + +var moderationCategories = []string{ + "harassment", + "harassment/threatening", + "hate", + "hate/threatening", + "illicit", + "illicit/violent", + "self-harm", + "self-harm/intent", + "self-harm/instructions", + "sexual", + "sexual/minors", + "violence", + "violence/graphic", +} + +type moderationGenerator func(context.Context, string, *config.ModelConfig) (string, backend.TokenUsage, error) + +type generatedModeration struct { + Categories map[string]bool `json:"categories"` + CategoryScores map[string]float64 `json:"category_scores"` +} + +// ModerationEndpoint implements the text input subset of OpenAI's moderation +// API using any LocalAI completion model and constrained JSON generation. +// @Summary Classify text for potentially harmful content. +// @Tags moderation +// @Param request body schema.ModerationRequest true "query params" +// @Success 200 {object} schema.ModerationResponse "Response" +// @Router /v1/moderations [post] +func ModerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return moderationEndpoint(func(ctx context.Context, input string, cfg *config.ModelConfig) (string, backend.TokenUsage, error) { + prompt := moderationPrompt(input) + var messages schema.Messages + if cfg.TemplateConfig.UseTokenizerTemplate { + messages = schema.Messages{{Role: "user", Content: prompt}} + prompt = "" + } else if evaluator != nil { + if rendered, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *cfg, templates.PromptTemplateData{Input: prompt, SystemPrompt: cfg.SystemPrompt}); err == nil { + prompt = rendered + } + } + + predict, err := backend.ModelInferenceFunc(ctx, prompt, messages, nil, nil, nil, ml, cfg, cl, appConfig, nil, "", "", nil, nil, nil, nil) + if err != nil { + return "", backend.TokenUsage{}, err + } + response, err := predict() + return response.Response, response.Usage, err + }) +} + +func moderationEndpoint(generate moderationGenerator) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.ModerationRequest) + if !ok || input == nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid moderation request") + } + if len(input.Input) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "input must contain at least one text string") + } + if generate == nil { + return echo.NewHTTPError(http.StatusInternalServerError, "moderation generator is unavailable") + } + + modelConfig, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || modelConfig == nil { + return echo.NewHTTPError(http.StatusBadRequest, "moderation model configuration is unavailable") + } + + grammar, err := moderationGrammar() + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "failed to build moderation grammar").SetInternal(err) + } + cfg := *modelConfig + cfg.Grammar = grammar + maxTokens := 512 + cfg.Maxtokens = &maxTokens + + results := make([]schema.ModerationResult, 0, len(input.Input)) + promptTokens, completionTokens := 0, 0 + for _, text := range input.Input { + if strings.TrimSpace(text) == "" { + return echo.NewHTTPError(http.StatusBadRequest, "input strings must not be empty") + } + raw, usage, err := generate(c.Request().Context(), text, &cfg) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "moderation inference failed").SetInternal(err) + } + result, err := parseModerationResult(raw) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "invalid moderation result: "+err.Error()).SetInternal(err) + } + results = append(results, result) + promptTokens += usage.Prompt + completionTokens += usage.Completion + } + + middleware.StampUsage(c, input.Model, promptTokens, completionTokens) + return c.JSON(http.StatusOK, schema.ModerationResponse{ + ID: "modr-" + uuid.NewString(), + Model: input.Model, + Results: results, + }) + } +} + +func moderationPrompt(input string) string { + encoded, _ := json.Marshal(input) + return `Classify the supplied text for safety moderation. Return only the JSON object required by the grammar. +For each category, set the boolean to true only when the text matches that category. Give each category a confidence score from 0 to 1. +The categories are harassment, harassment/threatening, hate, hate/threatening, illicit, illicit/violent, self-harm, self-harm/intent, self-harm/instructions, sexual, sexual/minors, violence, and violence/graphic. +Text to classify: ` + string(encoded) +} + +func moderationGrammar() (string, error) { + boolProperties := map[string]any{} + scoreProperties := map[string]any{} + for _, category := range moderationCategories { + boolProperties[category] = map[string]any{"type": "boolean"} + scoreProperties[category] = map[string]any{"type": "number"} + } + structure := functions.JSONFunctionStructure{AnyOf: []functions.Item{{ + Type: "object", + Properties: map[string]any{ + "categories": map[string]any{ + "type": "object", + "properties": boolProperties, + "required": moderationCategories, + "additionalProperties": false, + }, + "category_scores": map[string]any{ + "type": "object", + "properties": scoreProperties, + "required": moderationCategories, + "additionalProperties": false, + }, + }, + }}} + return structure.Grammar() +} + +func parseModerationResult(raw string) (schema.ModerationResult, error) { + var generated generatedModeration + if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &generated); err != nil { + return schema.ModerationResult{}, err + } + + result := schema.ModerationResult{ + Categories: make(map[string]bool, len(moderationCategories)), + CategoryScores: make(map[string]float64, len(moderationCategories)), + CategoryAppliedInputTypes: make(map[string][]string, len(moderationCategories)), + } + for _, category := range moderationCategories { + flagged, exists := generated.Categories[category] + if !exists { + return schema.ModerationResult{}, fmt.Errorf("missing category %q", category) + } + score, exists := generated.CategoryScores[category] + if !exists || math.IsNaN(score) || math.IsInf(score, 0) || score < 0 || score > 1 { + return schema.ModerationResult{}, fmt.Errorf("category %q has an invalid score", category) + } + result.Categories[category] = flagged + result.CategoryScores[category] = score + result.CategoryAppliedInputTypes[category] = []string{"text"} + result.Flagged = result.Flagged || flagged + } + return result, nil +} diff --git a/core/http/endpoints/openai/moderations_test.go b/core/http/endpoints/openai/moderations_test.go new file mode 100644 index 000000000..b6f5130e7 --- /dev/null +++ b/core/http/endpoints/openai/moderations_test.go @@ -0,0 +1,105 @@ +package openai + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Moderations endpoint", func() { + It("classifies each text input and returns the OpenAI response shape", func() { + inputs := []string{} + generate := func(_ context.Context, input string, cfg *config.ModelConfig) (string, backend.TokenUsage, error) { + inputs = append(inputs, input) + Expect(cfg.Grammar).To(ContainSubstring("harassment")) + return `{ + "categories":{"harassment":true,"harassment/threatening":false,"hate":false,"hate/threatening":false,"illicit":false,"illicit/violent":false,"self-harm":false,"self-harm/intent":false,"self-harm/instructions":false,"sexual":false,"sexual/minors":false,"violence":false,"violence/graphic":false}, + "category_scores":{"harassment":0.9,"harassment/threatening":0.1,"hate":0,"hate/threatening":0,"illicit":0,"illicit/violent":0,"self-harm":0,"self-harm/intent":0,"self-harm/instructions":0,"sexual":0,"sexual/minors":0,"violence":0,"violence/graphic":0} + }`, backend.TokenUsage{Prompt: 12, Completion: 8}, nil + } + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/v1/moderations", strings.NewReader(`{"model":"guard","input":["first","second"]}`)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + ctx := e.NewContext(req, rec) + ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "guard"}, + Input: schema.ModerationInput{"first", "second"}, + }) + modelConfig := &config.ModelConfig{Name: "guard"} + modelConfig.Model = "guard.gguf" + ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, modelConfig) + + Expect(moderationEndpoint(generate)(ctx)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(inputs).To(Equal([]string{"first", "second"})) + + var response schema.ModerationResponse + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response.ID).To(HavePrefix("modr-")) + Expect(response.Model).To(Equal("guard")) + Expect(response.Results).To(HaveLen(2)) + Expect(response.Results[0].Flagged).To(BeTrue()) + Expect(response.Results[0].Categories["harassment"]).To(BeTrue()) + Expect(response.Results[0].CategoryAppliedInputTypes["harassment"]).To(Equal([]string{"text"})) + }) + + It("rejects an empty input list", func() { + e := echo.New() + ctx := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/moderations", nil), httptest.NewRecorder()) + ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "guard"}, + }) + ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "guard"}) + + err := moderationEndpoint(nil)(ctx) + Expect(err).To(MatchError(ContainSubstring("input must contain at least one text string"))) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + }) + + It("surfaces malformed classifier output without returning a partial result", func() { + generate := func(context.Context, string, *config.ModelConfig) (string, backend.TokenUsage, error) { + return "not-json", backend.TokenUsage{}, nil + } + e := echo.New() + ctx := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/moderations", nil), httptest.NewRecorder()) + ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "guard"}, + Input: schema.ModerationInput{"text"}, + }) + ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "guard"}) + + err := moderationEndpoint(generate)(ctx) + Expect(err).To(MatchError(ContainSubstring("invalid moderation result"))) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusInternalServerError)) + }) +}) + +var _ = Describe("Moderation input", func() { + DescribeTable("accepts OpenAI text input forms", + func(body string, expected schema.ModerationInput) { + var req schema.ModerationRequest + Expect(json.Unmarshal([]byte(body), &req)).To(Succeed()) + Expect(req.Input).To(Equal(expected)) + }, + Entry("single text", `{"input":"hello"}`, schema.ModerationInput{"hello"}), + Entry("text array", `{"input":["hello","world"]}`, schema.ModerationInput{"hello", "world"}), + ) + + It("rejects multimodal input in the text-only MVP", func() { + var req schema.ModerationRequest + err := json.Unmarshal([]byte(`{"input":[{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}`), &req) + Expect(err).To(MatchError(ContainSubstring("text string or array of text strings"))) + }) +}) diff --git a/core/http/routes/openai.go b/core/http/routes/openai.go index b1671077a..7f3caf072 100644 --- a/core/http/routes/openai.go +++ b/core/http/routes/openai.go @@ -141,6 +141,20 @@ func RegisterOpenAIRoutes(app *echo.Echo, app.POST("/completions", completionHandler, completionMiddleware...) app.POST("/v1/engines/:model/completions", completionHandler, completionMiddleware...) + // moderation + moderationHandler := openai.ModerationEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig()) + moderationMiddleware := []echo.MiddlewareFunc{ + nodeHeaderMiddleware, + usageMiddleware, + traceMiddleware, + re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_COMPLETION)), + re.BuildConstantDefaultModelNameMiddleware("gpt-4o"), + re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.ModerationRequest) }), + middleware.AdmissionControl(application.AdmissionLimiter(), application.PIIEvents()), + } + app.POST("/v1/moderations", moderationHandler, moderationMiddleware...) + app.POST("/moderations", moderationHandler, moderationMiddleware...) + // embeddings embeddingHandler := openai.EmbeddingsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()) embeddingMiddleware := []echo.MiddlewareFunc{ diff --git a/core/schema/moderation.go b/core/schema/moderation.go new file mode 100644 index 000000000..c7e00ac26 --- /dev/null +++ b/core/schema/moderation.go @@ -0,0 +1,52 @@ +package schema + +import ( + "encoding/json" + "fmt" +) + +// ModerationInput accepts the text-only forms supported by the OpenAI +// moderations API. Multimodal moderation can be added without changing the +// response contract once LocalAI has a moderation-capable vision path. +type ModerationInput []string + +func (i *ModerationInput) UnmarshalJSON(data []byte) error { + var single string + if err := json.Unmarshal(data, &single); err == nil { + *i = ModerationInput{single} + return nil + } + + var multiple []string + if err := json.Unmarshal(data, &multiple); err == nil { + *i = ModerationInput(multiple) + return nil + } + + return fmt.Errorf("input must be a text string or array of text strings") +} + +func (i ModerationInput) MarshalJSON() ([]byte, error) { + if len(i) == 1 { + return json.Marshal(i[0]) + } + return json.Marshal([]string(i)) +} + +type ModerationRequest struct { + BasicModelRequest + Input ModerationInput `json:"input"` +} + +type ModerationResult struct { + Flagged bool `json:"flagged"` + Categories map[string]bool `json:"categories"` + CategoryScores map[string]float64 `json:"category_scores"` + CategoryAppliedInputTypes map[string][]string `json:"category_applied_input_types"` +} + +type ModerationResponse struct { + ID string `json:"id"` + Model string `json:"model"` + Results []ModerationResult `json:"results"` +} diff --git a/docs/content/features/constrained_grammars.md b/docs/content/features/constrained_grammars.md index b04170ee3..3769d5c1e 100644 --- a/docs/content/features/constrained_grammars.md +++ b/docs/content/features/constrained_grammars.md @@ -69,4 +69,5 @@ For more complex grammars, you can define multi-line BNF rules. The grammar pars ## Related Features - [OpenAI Functions]({{%relref "features/openai-functions" %}}) - Function calling with structured outputs -- [Text Generation]({{%relref "features/text-generation" %}}) - General text generation capabilities \ No newline at end of file +- [Text Generation]({{%relref "features/text-generation" %}}) - General text generation capabilities +- [Moderation]({{%relref "features/moderation" %}}) - OpenAI-compatible safety classification whose response is constrained to the moderation schema diff --git a/docs/content/features/moderation.md b/docs/content/features/moderation.md new file mode 100644 index 000000000..a8817cc84 --- /dev/null +++ b/docs/content/features/moderation.md @@ -0,0 +1,37 @@ ++++ +disableToc = false +title = "Moderation" +weight = 65 +url = "/features/moderation/" ++++ + +LocalAI exposes an OpenAI-compatible text moderation endpoint at +`POST /v1/moderations`. It uses a local text-generation model with a constrained +JSON grammar, so no separate moderation service or cloud API is required. + +```bash +curl http://localhost:8080/v1/moderations \ + -H "Content-Type: application/json" \ + -d '{ + "model": "your-instruct-model", + "input": "Text to classify" + }' +``` + +`input` may be one string or an array of strings. The response contains one +result per input with `flagged`, `categories`, `category_scores`, and +`category_applied_input_types` fields. The category names match the OpenAI +moderation API, including harassment, hate, illicit activity, self-harm, +sexual content, and violence categories. + +The selected model must support text completion. For consistent results, use +an instruction-tuned model that follows safety-classification prompts well. +LocalAI constrains the output shape, but the model determines the classification +quality and confidence scores. + +{{% notice note %}} + +This first implementation supports text only. OpenAI-style multimodal input +objects containing images return a validation error. + +{{% /notice %}} diff --git a/docs/content/whats-new.md b/docs/content/whats-new.md index a947642d0..c9ab1823a 100644 --- a/docs/content/whats-new.md +++ b/docs/content/whats-new.md @@ -12,6 +12,7 @@ You can see the release notes [here](https://github.com/mudler/LocalAI/releases) ## 2026 Highlights +- **August 2026**: [Text moderation](/features/moderation/) - new OpenAI-compatible `POST /v1/moderations` endpoint. It uses any local completion model with a constrained JSON grammar and returns the standard safety categories, scores, and per-input flags. - **July 2026**: [LongCat video and avatar generation](/features/video-generation/) - dedicated CUDA backend for `LongCat-Video` text/image-to-video and `LongCat-Video-Avatar-1.5` speech-driven avatars. Includes multi-segment continuation, portrait and recorded-audio inputs in Studio, and an SDPA CUDA 13 ARM64 build for DGX Spark. - **April 2026**: [Audio Transform](/features/audio-transform/) - generic audio-in / audio-out endpoint with optional reference signal. First implementation: [LocalVQE](https://github.com/localai-org/LocalVQE) C++ backend (joint AEC + noise suppression + dereverberation, DeepVQE-style). Both batch (`POST /audio/transformations`) and bidirectional WebSocket streaming (`/audio/transformations/stream`). Studio "Transform" tab with synchronized waveform players for input / reference / output. - **April 2026**: [Face recognition backend](/features/face-recognition/) - `insightface`-powered 1:1 verification, 1:N identification, face embedding, face detection, and demographic analysis. Ships both a non-commercial `buffalo_l` model and an Apache 2.0 OpenCV Zoo alternative. diff --git a/swagger/docs.go b/swagger/docs.go index b69860b01..4f259f284 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -1550,6 +1550,34 @@ const docTemplate = `{ } } }, + "/api/traces/summary": { + "get": { + "description": "Returns request, failure and latency totals over a recent window, plus a bucketed series for sparklines. Exists so callers wanting three numbers do not have to fetch the whole trace list and count it themselves.", + "produces": [ + "application/json" + ], + "tags": [ + "monitoring" + ], + "summary": "Summarize recent API traces", + "parameters": [ + { + "type": "integer", + "description": "Window in hours (default 24, max 168)", + "name": "hours", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Counted trace totals", + "schema": { + "$ref": "#/definitions/middleware.TraceSummary" + } + } + } + } + }, "/api/traces/{id}": { "get": { "description": "Returns a single captured API exchange, including the request and response bodies omitted from the list response", @@ -3271,6 +3299,33 @@ const docTemplate = `{ } } }, + "/v1/moderations": { + "post": { + "tags": [ + "moderation" + ], + "summary": "Classify text for potentially harmful content.", + "parameters": [ + { + "description": "query params", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.ModerationRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.ModerationResponse" + } + } + } + } + }, "/v1/rerank": { "post": { "tags": [ @@ -4357,6 +4412,43 @@ const docTemplate = `{ } } }, + "middleware.TraceBucket": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "errors": { + "type": "integer" + }, + "start": { + "type": "string" + } + } + }, + "middleware.TraceSummary": { + "type": "object", + "properties": { + "buckets": { + "type": "array", + "items": { + "$ref": "#/definitions/middleware.TraceBucket" + } + }, + "errors": { + "type": "integer" + }, + "p95_ms": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "window_hours": { + "type": "integer" + } + } + }, "model.BackendLogLine": { "type": "object", "properties": { @@ -5979,6 +6071,67 @@ const docTemplate = `{ } } }, + "schema.ModerationRequest": { + "type": "object", + "properties": { + "input": { + "type": "array", + "items": { + "type": "string" + } + }, + "model": { + "type": "string" + } + } + }, + "schema.ModerationResponse": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.ModerationResult" + } + } + } + }, + "schema.ModerationResult": { + "type": "object", + "properties": { + "categories": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + }, + "category_applied_input_types": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "category_scores": { + "type": "object", + "additionalProperties": { + "type": "number", + "format": "float64" + } + }, + "flagged": { + "type": "boolean" + } + } + }, "schema.MultimediaSourceConfig": { "type": "object", "properties": { @@ -7068,6 +7221,10 @@ const docTemplate = `{ "schema.SysInfoModel": { "type": "object", "properties": { + "backend": { + "description": "Backend is the engine serving this model. The loader knows only the ID,\nso it is resolved from the model's config; empty when the model was\nloaded without one (a loose file, or a config since removed).", + "type": "string" + }, "id": { "type": "string" } diff --git a/swagger/swagger.json b/swagger/swagger.json index 8ce062051..39dd6c03a 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -1547,6 +1547,34 @@ } } }, + "/api/traces/summary": { + "get": { + "description": "Returns request, failure and latency totals over a recent window, plus a bucketed series for sparklines. Exists so callers wanting three numbers do not have to fetch the whole trace list and count it themselves.", + "produces": [ + "application/json" + ], + "tags": [ + "monitoring" + ], + "summary": "Summarize recent API traces", + "parameters": [ + { + "type": "integer", + "description": "Window in hours (default 24, max 168)", + "name": "hours", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Counted trace totals", + "schema": { + "$ref": "#/definitions/middleware.TraceSummary" + } + } + } + } + }, "/api/traces/{id}": { "get": { "description": "Returns a single captured API exchange, including the request and response bodies omitted from the list response", @@ -3268,6 +3296,33 @@ } } }, + "/v1/moderations": { + "post": { + "tags": [ + "moderation" + ], + "summary": "Classify text for potentially harmful content.", + "parameters": [ + { + "description": "query params", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.ModerationRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.ModerationResponse" + } + } + } + } + }, "/v1/rerank": { "post": { "tags": [ @@ -4354,6 +4409,43 @@ } } }, + "middleware.TraceBucket": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "errors": { + "type": "integer" + }, + "start": { + "type": "string" + } + } + }, + "middleware.TraceSummary": { + "type": "object", + "properties": { + "buckets": { + "type": "array", + "items": { + "$ref": "#/definitions/middleware.TraceBucket" + } + }, + "errors": { + "type": "integer" + }, + "p95_ms": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "window_hours": { + "type": "integer" + } + } + }, "model.BackendLogLine": { "type": "object", "properties": { @@ -5976,6 +6068,67 @@ } } }, + "schema.ModerationRequest": { + "type": "object", + "properties": { + "input": { + "type": "array", + "items": { + "type": "string" + } + }, + "model": { + "type": "string" + } + } + }, + "schema.ModerationResponse": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.ModerationResult" + } + } + } + }, + "schema.ModerationResult": { + "type": "object", + "properties": { + "categories": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + }, + "category_applied_input_types": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "category_scores": { + "type": "object", + "additionalProperties": { + "type": "number", + "format": "float64" + } + }, + "flagged": { + "type": "boolean" + } + } + }, "schema.MultimediaSourceConfig": { "type": "object", "properties": { @@ -7065,6 +7218,10 @@ "schema.SysInfoModel": { "type": "object", "properties": { + "backend": { + "description": "Backend is the engine serving this model. The loader knows only the ID,\nso it is resolved from the model's config; empty when the model was\nloaded without one (a loose file, or a config since removed).", + "type": "string" + }, "id": { "type": "string" } diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 2963dfd2c..90e0bb3c4 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -438,6 +438,30 @@ definitions: $ref: '#/definitions/voiceprofile.Profile' type: array type: object + middleware.TraceBucket: + properties: + count: + type: integer + errors: + type: integer + start: + type: string + type: object + middleware.TraceSummary: + properties: + buckets: + items: + $ref: '#/definitions/middleware.TraceBucket' + type: array + errors: + type: integer + p95_ms: + type: integer + total: + type: integer + window_hours: + type: integer + type: object model.BackendLogLine: properties: stream: @@ -1558,6 +1582,46 @@ definitions: object: type: string type: object + schema.ModerationRequest: + properties: + input: + items: + type: string + type: array + model: + type: string + type: object + schema.ModerationResponse: + properties: + id: + type: string + model: + type: string + results: + items: + $ref: '#/definitions/schema.ModerationResult' + type: array + type: object + schema.ModerationResult: + properties: + categories: + additionalProperties: + type: boolean + type: object + category_applied_input_types: + additionalProperties: + items: + type: string + type: array + type: object + category_scores: + additionalProperties: + format: float64 + type: number + type: object + flagged: + type: boolean + type: object schema.MultimediaSourceConfig: properties: headers: @@ -2337,6 +2401,12 @@ definitions: type: object schema.SysInfoModel: properties: + backend: + description: |- + Backend is the engine serving this model. The loader knows only the ID, + so it is resolved from the model's config; empty when the model was + loaded without one (a loose file, or a config since removed). + type: string id: type: string type: object @@ -3837,6 +3907,26 @@ paths: summary: Clear API traces tags: - monitoring + /api/traces/summary: + get: + description: Returns request, failure and latency totals over a recent window, + plus a bucketed series for sparklines. Exists so callers wanting three numbers + do not have to fetch the whole trace list and count it themselves. + parameters: + - description: Window in hours (default 24, max 168) + in: query + name: hours + type: integer + produces: + - application/json + responses: + "200": + description: Counted trace totals + schema: + $ref: '#/definitions/middleware.TraceSummary' + summary: Summarize recent API traces + tags: + - monitoring /api/voice-profiles: get: description: List saved voice-cloning references without exposing filesystem @@ -4954,6 +5044,23 @@ paths: summary: List available models enriched with capabilities and input/output modalities. tags: - models + /v1/moderations: + post: + parameters: + - description: query params + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.ModerationRequest' + responses: + "200": + description: Response + schema: + $ref: '#/definitions/schema.ModerationResponse' + summary: Classify text for potentially harmful content. + tags: + - moderation /v1/rerank: post: parameters: