mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-08-04 04:12:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): add text moderation endpoint (#11316)

Browse Source 
* feat(api): add text moderation endpoint

Add an OpenAI-compatible /v1/moderations endpoint backed by constrained local text generation. Register its auth and discovery surfaces, document the text-only MVP, and cover response shaping and access control.

Assisted-by: Codex:gpt-5

* test(mcp): update assistant client stub

Keep the LocalAI Assistant holder test stub aligned with the scheduling methods added to LocalAIClient so repository-wide type checking succeeds.\n\nAssisted-by: Codex:gpt-5

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
localai-org-maint-bot
2026-08-03 18:03:46 +02:00
committed by GitHub
parent 8a68f3571c
commit 133c546c3f
18 changed files with 903 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/http/auth/features.go
View File

@@ -118,6 +118,10 @@ var RouteFeatureRegistry = []RouteFeature{
// Rerank
{"POST", "/v1/rerank", FeatureRerank},
// Moderation
{"POST", "/v1/moderations", FeatureModeration},
{"POST", "/moderations", FeatureModeration},
// Stores
{"POST", "/stores/set", FeatureStores},
{"POST", "/stores/delete", FeatureStores},
@@ -193,6 +197,7 @@ func APIFeatureMetas() []FeatureMeta {
{FeatureEmbeddings, "Embeddings", true},
{FeatureSound, "Sound Generation", true},
{FeatureRealtime, "Realtime", true},
{FeatureModeration, "Moderation", true},
{FeatureRerank, "Rerank", true},
{FeatureTokenize, "Tokenize", true},
{FeatureMCP, "MCP", true},

24
core/http/auth/features_moderation_test.go Normal file
View File

@@ -0,0 +1,24 @@
package auth_test
import (
. "github.com/mudler/LocalAI/core/http/auth"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Moderation feature registration", func() {
It("registers both moderation routes as default-on API features", func() {
Expect(APIFeatures).To(ContainElement(FeatureModeration))
patterns := []string{}
for _, route := range RouteFeatureRegistry {
if route.Feature == FeatureModeration {
patterns = append(patterns, route.Pattern)
}
}
Expect(patterns).To(ConsistOf("/v1/moderations", "/moderations"))
metas := APIFeatureMetas()
Expect(metas).To(ContainElement(FeatureMeta{Key: FeatureModeration, Label: "Moderation", DefaultValue: true}))
})
})

8
core/http/auth/helpers_test.go
View File

@@ -59,10 +59,14 @@ func ok(c echo.Context) error {
func newAuthTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo {
e := echo.New()
e.Use(auth.Middleware(db, appConfig))
if db != nil {
e.Use(auth.RequireRouteFeature(db))
}
// API routes (require auth)
e.GET("/v1/models", ok)
e.POST("/v1/chat/completions", ok)
e.POST("/v1/moderations", ok)
e.GET("/api/settings", ok)
e.POST("/api/settings", ok)
@@ -81,10 +85,14 @@ func newAuthTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo
func newAdminTestApp(db *gorm.DB, appConfig *config.ApplicationConfig) *echo.Echo {
e := echo.New()
e.Use(auth.Middleware(db, appConfig))
if db != nil {
e.Use(auth.RequireRouteFeature(db))
}
// Regular routes
e.GET("/v1/models", ok)
e.POST("/v1/chat/completions", ok)
e.POST("/v1/moderations", ok)
// Admin-only routes
adminMw := auth.RequireAdmin()

18
core/http/auth/middleware_test.go
View File

@@ -91,6 +91,19 @@ var _ = Describe("Auth Middleware", func() {
Expect(rec.Code).To(Equal(http.StatusOK))
})
It("allows authenticated users to call moderation by default", func() {
sessionID := createTestSession(db, user.ID)
rec := doRequest(app, http.MethodPost, "/v1/moderations", withSessionCookie(sessionID))
Expect(rec.Code).To(Equal(http.StatusOK))
})
It("blocks moderation when the user's feature is disabled", func() {
Expect(auth.UpdateUserPermissions(db, user.ID, auth.PermissionMap{auth.FeatureModeration: false})).To(Succeed())
sessionID := createTestSession(db, user.ID)
rec := doRequest(app, http.MethodPost, "/v1/moderations", withSessionCookie(sessionID))
Expect(rec.Code).To(Equal(http.StatusForbidden))
})
It("allows requests with valid session as Bearer token", func() {
sessionID := createTestSession(db, user.ID)
rec := doRequest(app, http.MethodGet, "/v1/models", withBearerToken(sessionID))
@@ -156,6 +169,11 @@ var _ = Describe("Auth Middleware", func() {
Expect(rec.Code).To(Equal(http.StatusUnauthorized))
})
It("returns 401 for unauthenticated moderation requests", func() {
rec := doRequest(app, http.MethodPost, "/v1/moderations")
Expect(rec.Code).To(Equal(http.StatusUnauthorized))
})
It("returns 401 for unauthenticated 3D generation requests", func() {
rec := doRequest(app, http.MethodPost, "/3d/generations")
Expect(rec.Code).To(Equal(http.StatusUnauthorized))

3
core/http/auth/permissions.go
View File

@@ -51,6 +51,7 @@ const (
FeatureEmbeddings = "embeddings"
FeatureSound = "sound"
FeatureRealtime = "realtime"
FeatureModeration = "moderation"
FeatureRerank = "rerank"
FeatureTokenize = "tokenize"
FeatureMCP = "mcp"
@@ -75,7 +76,7 @@ var APIFeatures = []string{
FeatureChat, FeatureImages, FeatureAudioSpeech, FeatureAudioTranscription,
FeatureAudioDiarization, FeatureAudioClassification,
FeatureVAD, FeatureDetection, FeatureVideo, Feature3D, FeatureEmbeddings, FeatureSound,
FeatureRealtime, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores,
FeatureRealtime, FeatureModeration, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores,
FeatureFaceRecognition, FeatureVoiceRecognition, FeatureAudioTransform,
FeaturePIIFilter,
}

6
core/http/endpoints/localai/api_instructions.go
View File

@@ -30,6 +30,12 @@ var instructionDefs = []instructionDef{
Tags: []string{"inference", "embeddings"},
Intro: "Set \"stream\": true for SSE streaming. Supports tool/function calling when the model config has function templates configured.",
},
{
Name: "moderation",
Description: "OpenAI-compatible text moderation using a local completion model",
Tags: []string{"moderation"},
Intro: "POST /v1/moderations accepts a text string or array plus a LocalAI completion model. LocalAI constrains the model to the OpenAI moderation category schema and returns one result per input. Multimodal moderation inputs are not yet supported.",
},
{
Name: "audio",
Description: "Text-to-speech, voice activity detection, transcription, speaker diarization, sound classification, and sound generation",

3
core/http/endpoints/localai/api_instructions_test.go
View File

@@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() {
instructions, ok := resp["instructions"].([]any)
Expect(ok).To(BeTrue())
Expect(instructions).To(HaveLen(18))
Expect(instructions).To(HaveLen(19))
// Verify each instruction has required fields and correct URL format
for _, s := range instructions {
@@ -69,6 +69,7 @@ var _ = Describe("API Instructions Endpoints", func() {
Expect(names).To(ContainElements(
"chat-inference",
"moderation",
"config-management",
"model-management",
"monitoring",

16
core/http/endpoints/mcp/localai_assistant_test.go
View File

@@ -84,6 +84,22 @@ func (stubClient) ListNodes(_ context.Context) ([]localaitools.Node, error) {
return []localaitools.Node{}, nil
}
func (stubClient) ListScheduling(_ context.Context) ([]localaitools.ModelSchedulingConfig, error) {
return []localaitools.ModelSchedulingConfig{}, nil
}
func (stubClient) GetScheduling(_ context.Context, _ string) (*localaitools.ModelSchedulingConfig, error) {
return &localaitools.ModelSchedulingConfig{}, nil
}
func (stubClient) SetScheduling(_ context.Context, _ localaitools.SetSchedulingRequest) (*localaitools.ModelSchedulingConfig, error) {
return &localaitools.ModelSchedulingConfig{}, nil
}
func (stubClient) DeleteScheduling(_ context.Context, _ string) error {
return nil
}
func (stubClient) SetNodeVRAMBudget(_ context.Context, _, _ string) error {
return nil
}

190
core/http/endpoints/openai/moderations.go Normal file
View File

@@ -0,0 +1,190 @@
package openai
import (
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"strings"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/model"
)
var moderationCategories = []string{
"harassment",
"harassment/threatening",
"hate",
"hate/threatening",
"illicit",
"illicit/violent",
"self-harm",
"self-harm/intent",
"self-harm/instructions",
"sexual",
"sexual/minors",
"violence",
"violence/graphic",
}
type moderationGenerator func(context.Context, string, *config.ModelConfig) (string, backend.TokenUsage, error)
type generatedModeration struct {
Categories map[string]bool `json:"categories"`
CategoryScores map[string]float64 `json:"category_scores"`
}
// ModerationEndpoint implements the text input subset of OpenAI's moderation
// API using any LocalAI completion model and constrained JSON generation.
// @Summary Classify text for potentially harmful content.
// @Tags moderation
// @Param request body schema.ModerationRequest true "query params"
// @Success 200 {object} schema.ModerationResponse "Response"
// @Router /v1/moderations [post]
func ModerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig) echo.HandlerFunc {
return moderationEndpoint(func(ctx context.Context, input string, cfg *config.ModelConfig) (string, backend.TokenUsage, error) {
prompt := moderationPrompt(input)
var messages schema.Messages
if cfg.TemplateConfig.UseTokenizerTemplate {
messages = schema.Messages{{Role: "user", Content: prompt}}
prompt = ""
} else if evaluator != nil {
if rendered, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *cfg, templates.PromptTemplateData{Input: prompt, SystemPrompt: cfg.SystemPrompt}); err == nil {
prompt = rendered
}
}
predict, err := backend.ModelInferenceFunc(ctx, prompt, messages, nil, nil, nil, ml, cfg, cl, appConfig, nil, "", "", nil, nil, nil, nil)
if err != nil {
return "", backend.TokenUsage{}, err
}
response, err := predict()
return response.Response, response.Usage, err
})
}
func moderationEndpoint(generate moderationGenerator) echo.HandlerFunc {
return func(c echo.Context) error {
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.ModerationRequest)
if !ok || input == nil {
return echo.NewHTTPError(http.StatusBadRequest, "invalid moderation request")
}
if len(input.Input) == 0 {
return echo.NewHTTPError(http.StatusBadRequest, "input must contain at least one text string")
}
if generate == nil {
return echo.NewHTTPError(http.StatusInternalServerError, "moderation generator is unavailable")
}
modelConfig, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || modelConfig == nil {
return echo.NewHTTPError(http.StatusBadRequest, "moderation model configuration is unavailable")
}
grammar, err := moderationGrammar()
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "failed to build moderation grammar").SetInternal(err)
}
cfg := *modelConfig
cfg.Grammar = grammar
maxTokens := 512
cfg.Maxtokens = &maxTokens
results := make([]schema.ModerationResult, 0, len(input.Input))
promptTokens, completionTokens := 0, 0
for _, text := range input.Input {
if strings.TrimSpace(text) == "" {
return echo.NewHTTPError(http.StatusBadRequest, "input strings must not be empty")
}
raw, usage, err := generate(c.Request().Context(), text, &cfg)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "moderation inference failed").SetInternal(err)
}
result, err := parseModerationResult(raw)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "invalid moderation result: "+err.Error()).SetInternal(err)
}
results = append(results, result)
promptTokens += usage.Prompt
completionTokens += usage.Completion
}
middleware.StampUsage(c, input.Model, promptTokens, completionTokens)
return c.JSON(http.StatusOK, schema.ModerationResponse{
ID: "modr-" + uuid.NewString(),
Model: input.Model,
Results: results,
})
}
}
func moderationPrompt(input string) string {
encoded, _ := json.Marshal(input)
return `Classify the supplied text for safety moderation. Return only the JSON object required by the grammar.
For each category, set the boolean to true only when the text matches that category. Give each category a confidence score from 0 to 1.
The categories are harassment, harassment/threatening, hate, hate/threatening, illicit, illicit/violent, self-harm, self-harm/intent, self-harm/instructions, sexual, sexual/minors, violence, and violence/graphic.
Text to classify: ` + string(encoded)
}
func moderationGrammar() (string, error) {
boolProperties := map[string]any{}
scoreProperties := map[string]any{}
for _, category := range moderationCategories {
boolProperties[category] = map[string]any{"type": "boolean"}
scoreProperties[category] = map[string]any{"type": "number"}
}
structure := functions.JSONFunctionStructure{AnyOf: []functions.Item{{
Type: "object",
Properties: map[string]any{
"categories": map[string]any{
"type": "object",
"properties": boolProperties,
"required": moderationCategories,
"additionalProperties": false,
},
"category_scores": map[string]any{
"type": "object",
"properties": scoreProperties,
"required": moderationCategories,
"additionalProperties": false,
},
},
}}}
return structure.Grammar()
}
func parseModerationResult(raw string) (schema.ModerationResult, error) {
var generated generatedModeration
if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &generated); err != nil {
return schema.ModerationResult{}, err
}
result := schema.ModerationResult{
Categories: make(map[string]bool, len(moderationCategories)),
CategoryScores: make(map[string]float64, len(moderationCategories)),
CategoryAppliedInputTypes: make(map[string][]string, len(moderationCategories)),
}
for _, category := range moderationCategories {
flagged, exists := generated.Categories[category]
if !exists {
return schema.ModerationResult{}, fmt.Errorf("missing category %q", category)
}
score, exists := generated.CategoryScores[category]
if !exists || math.IsNaN(score) || math.IsInf(score, 0) || score < 0 || score > 1 {
return schema.ModerationResult{}, fmt.Errorf("category %q has an invalid score", category)
}
result.Categories[category] = flagged
result.CategoryScores[category] = score
result.CategoryAppliedInputTypes[category] = []string{"text"}
result.Flagged = result.Flagged || flagged
}
return result, nil
}

105
core/http/endpoints/openai/moderations_test.go Normal file
View File

@@ -0,0 +1,105 @@
package openai
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Moderations endpoint", func() {
It("classifies each text input and returns the OpenAI response shape", func() {
inputs := []string{}
generate := func(_ context.Context, input string, cfg *config.ModelConfig) (string, backend.TokenUsage, error) {
inputs = append(inputs, input)
Expect(cfg.Grammar).To(ContainSubstring("harassment"))
return `{
"categories":{"harassment":true,"harassment/threatening":false,"hate":false,"hate/threatening":false,"illicit":false,"illicit/violent":false,"self-harm":false,"self-harm/intent":false,"self-harm/instructions":false,"sexual":false,"sexual/minors":false,"violence":false,"violence/graphic":false},
"category_scores":{"harassment":0.9,"harassment/threatening":0.1,"hate":0,"hate/threatening":0,"illicit":0,"illicit/violent":0,"self-harm":0,"self-harm/intent":0,"self-harm/instructions":0,"sexual":0,"sexual/minors":0,"violence":0,"violence/graphic":0}
}`, backend.TokenUsage{Prompt: 12, Completion: 8}, nil
}
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/v1/moderations", strings.NewReader(`{"model":"guard","input":["first","second"]}`))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
ctx := e.NewContext(req, rec)
ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{
BasicModelRequest: schema.BasicModelRequest{Model: "guard"},
Input: schema.ModerationInput{"first", "second"},
})
modelConfig := &config.ModelConfig{Name: "guard"}
modelConfig.Model = "guard.gguf"
ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, modelConfig)
Expect(moderationEndpoint(generate)(ctx)).To(Succeed())
Expect(rec.Code).To(Equal(http.StatusOK))
Expect(inputs).To(Equal([]string{"first", "second"}))
var response schema.ModerationResponse
Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed())
Expect(response.ID).To(HavePrefix("modr-"))
Expect(response.Model).To(Equal("guard"))
Expect(response.Results).To(HaveLen(2))
Expect(response.Results[0].Flagged).To(BeTrue())
Expect(response.Results[0].Categories["harassment"]).To(BeTrue())
Expect(response.Results[0].CategoryAppliedInputTypes["harassment"]).To(Equal([]string{"text"}))
})
It("rejects an empty input list", func() {
e := echo.New()
ctx := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/moderations", nil), httptest.NewRecorder())
ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{
BasicModelRequest: schema.BasicModelRequest{Model: "guard"},
})
ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "guard"})
err := moderationEndpoint(nil)(ctx)
Expect(err).To(MatchError(ContainSubstring("input must contain at least one text string")))
Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest))
})
It("surfaces malformed classifier output without returning a partial result", func() {
generate := func(context.Context, string, *config.ModelConfig) (string, backend.TokenUsage, error) {
return "not-json", backend.TokenUsage{}, nil
}
e := echo.New()
ctx := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/moderations", nil), httptest.NewRecorder())
ctx.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &schema.ModerationRequest{
BasicModelRequest: schema.BasicModelRequest{Model: "guard"},
Input: schema.ModerationInput{"text"},
})
ctx.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "guard"})
err := moderationEndpoint(generate)(ctx)
Expect(err).To(MatchError(ContainSubstring("invalid moderation result")))
Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusInternalServerError))
})
})
var _ = Describe("Moderation input", func() {
DescribeTable("accepts OpenAI text input forms",
func(body string, expected schema.ModerationInput) {
var req schema.ModerationRequest
Expect(json.Unmarshal([]byte(body), &req)).To(Succeed())
Expect(req.Input).To(Equal(expected))
},
Entry("single text", `{"input":"hello"}`, schema.ModerationInput{"hello"}),
Entry("text array", `{"input":["hello","world"]}`, schema.ModerationInput{"hello", "world"}),
)
It("rejects multimodal input in the text-only MVP", func() {
var req schema.ModerationRequest
err := json.Unmarshal([]byte(`{"input":[{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}`), &req)
Expect(err).To(MatchError(ContainSubstring("text string or array of text strings")))
})
})

14
core/http/routes/openai.go
View File

@@ -141,6 +141,20 @@ func RegisterOpenAIRoutes(app *echo.Echo,
app.POST("/completions", completionHandler, completionMiddleware...)
app.POST("/v1/engines/:model/completions", completionHandler, completionMiddleware...)
// moderation
moderationHandler := openai.ModerationEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig())
moderationMiddleware := []echo.MiddlewareFunc{
nodeHeaderMiddleware,
usageMiddleware,
traceMiddleware,
re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_COMPLETION)),
re.BuildConstantDefaultModelNameMiddleware("gpt-4o"),
re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.ModerationRequest) }),
middleware.AdmissionControl(application.AdmissionLimiter(), application.PIIEvents()),
}
app.POST("/v1/moderations", moderationHandler, moderationMiddleware...)
app.POST("/moderations", moderationHandler, moderationMiddleware...)
// embeddings
embeddingHandler := openai.EmbeddingsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())
embeddingMiddleware := []echo.MiddlewareFunc{

52
core/schema/moderation.go Normal file
View File

@@ -0,0 +1,52 @@
package schema
import (
"encoding/json"
"fmt"
)
// ModerationInput accepts the text-only forms supported by the OpenAI
// moderations API. Multimodal moderation can be added without changing the
// response contract once LocalAI has a moderation-capable vision path.
type ModerationInput []string
func (i *ModerationInput) UnmarshalJSON(data []byte) error {
var single string
if err := json.Unmarshal(data, &single); err == nil {
*i = ModerationInput{single}
return nil
}
var multiple []string
if err := json.Unmarshal(data, &multiple); err == nil {
*i = ModerationInput(multiple)
return nil
}
return fmt.Errorf("input must be a text string or array of text strings")
}
func (i ModerationInput) MarshalJSON() ([]byte, error) {
if len(i) == 1 {
return json.Marshal(i[0])
}
return json.Marshal([]string(i))
}
type ModerationRequest struct {
BasicModelRequest
Input ModerationInput `json:"input"`
}
type ModerationResult struct {
Flagged bool `json:"flagged"`
Categories map[string]bool `json:"categories"`
CategoryScores map[string]float64 `json:"category_scores"`
CategoryAppliedInputTypes map[string][]string `json:"category_applied_input_types"`
}
type ModerationResponse struct {
ID string `json:"id"`
Model string `json:"model"`
Results []ModerationResult `json:"results"`
}

3
docs/content/features/constrained_grammars.md
View File

@@ -69,4 +69,5 @@ For more complex grammars, you can define multi-line BNF rules. The grammar pars
## Related Features
- [OpenAI Functions]({{%relref "features/openai-functions" %}}) - Function calling with structured outputs
- [Text Generation]({{%relref "features/text-generation" %}}) - General text generation capabilities
- [Text Generation]({{%relref "features/text-generation" %}}) - General text generation capabilities
- [Moderation]({{%relref "features/moderation" %}}) - OpenAI-compatible safety classification whose response is constrained to the moderation schema

37
docs/content/features/moderation.md Normal file
View File

@@ -0,0 +1,37 @@
+++
disableToc = false
title = "Moderation"
weight = 65
url = "/features/moderation/"
+++
LocalAI exposes an OpenAI-compatible text moderation endpoint at
`POST /v1/moderations`. It uses a local text-generation model with a constrained
JSON grammar, so no separate moderation service or cloud API is required.
```bash
curl http://localhost:8080/v1/moderations \
-H "Content-Type: application/json" \
-d '{
"model": "your-instruct-model",
"input": "Text to classify"
}'
```
`input` may be one string or an array of strings. The response contains one
result per input with `flagged`, `categories`, `category_scores`, and
`category_applied_input_types` fields. The category names match the OpenAI
moderation API, including harassment, hate, illicit activity, self-harm,
sexual content, and violence categories.
The selected model must support text completion. For consistent results, use
an instruction-tuned model that follows safety-classification prompts well.
LocalAI constrains the output shape, but the model determines the classification
quality and confidence scores.
{{% notice note %}}
This first implementation supports text only. OpenAI-style multimodal input
objects containing images return a validation error.
{{% /notice %}}

1
docs/content/whats-new.md
View File

@@ -12,6 +12,7 @@ You can see the release notes [here](https://github.com/mudler/LocalAI/releases)
## 2026 Highlights
- **August 2026**: [Text moderation](/features/moderation/) - new OpenAI-compatible `POST /v1/moderations` endpoint. It uses any local completion model with a constrained JSON grammar and returns the standard safety categories, scores, and per-input flags.
- **July 2026**: [LongCat video and avatar generation](/features/video-generation/) - dedicated CUDA backend for `LongCat-Video` text/image-to-video and `LongCat-Video-Avatar-1.5` speech-driven avatars. Includes multi-segment continuation, portrait and recorded-audio inputs in Studio, and an SDPA CUDA 13 ARM64 build for DGX Spark.
- **April 2026**: [Audio Transform](/features/audio-transform/) - generic audio-in / audio-out endpoint with optional reference signal. First implementation: [LocalVQE](https://github.com/localai-org/LocalVQE) C++ backend (joint AEC + noise suppression + dereverberation, DeepVQE-style). Both batch (`POST /audio/transformations`) and bidirectional WebSocket streaming (`/audio/transformations/stream`). Studio "Transform" tab with synchronized waveform players for input / reference / output.
- **April 2026**: [Face recognition backend](/features/face-recognition/) - `insightface`-powered 1:1 verification, 1:N identification, face embedding, face detection, and demographic analysis. Ships both a non-commercial `buffalo_l` model and an Apache 2.0 OpenCV Zoo alternative.

157
swagger/docs.go
View File

@@ -1550,6 +1550,34 @@ const docTemplate = `{
}
}
},
"/api/traces/summary": {
"get": {
"description": "Returns request, failure and latency totals over a recent window, plus a bucketed series for sparklines. Exists so callers wanting three numbers do not have to fetch the whole trace list and count it themselves.",
"produces": [
"application/json"
],
"tags": [
"monitoring"
],
"summary": "Summarize recent API traces",
"parameters": [
{
"type": "integer",
"description": "Window in hours (default 24, max 168)",
"name": "hours",
"in": "query"
}
],
"responses": {
"200": {
"description": "Counted trace totals",
"schema": {
"$ref": "#/definitions/middleware.TraceSummary"
}
}
}
}
},
"/api/traces/{id}": {
"get": {
"description": "Returns a single captured API exchange, including the request and response bodies omitted from the list response",
@@ -3271,6 +3299,33 @@ const docTemplate = `{
}
}
},
"/v1/moderations": {
"post": {
"tags": [
"moderation"
],
"summary": "Classify text for potentially harmful content.",
"parameters": [
{
"description": "query params",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/schema.ModerationRequest"
}
}
],
"responses": {
"200": {
"description": "Response",
"schema": {
"$ref": "#/definitions/schema.ModerationResponse"
}
}
}
}
},
"/v1/rerank": {
"post": {
"tags": [
@@ -4357,6 +4412,43 @@ const docTemplate = `{
}
}
},
"middleware.TraceBucket": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"errors": {
"type": "integer"
},
"start": {
"type": "string"
}
}
},
"middleware.TraceSummary": {
"type": "object",
"properties": {
"buckets": {
"type": "array",
"items": {
"$ref": "#/definitions/middleware.TraceBucket"
}
},
"errors": {
"type": "integer"
},
"p95_ms": {
"type": "integer"
},
"total": {
"type": "integer"
},
"window_hours": {
"type": "integer"
}
}
},
"model.BackendLogLine": {
"type": "object",
"properties": {
@@ -5979,6 +6071,67 @@ const docTemplate = `{
}
}
},
"schema.ModerationRequest": {
"type": "object",
"properties": {
"input": {
"type": "array",
"items": {
"type": "string"
}
},
"model": {
"type": "string"
}
}
},
"schema.ModerationResponse": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"model": {
"type": "string"
},
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.ModerationResult"
}
}
}
},
"schema.ModerationResult": {
"type": "object",
"properties": {
"categories": {
"type": "object",
"additionalProperties": {
"type": "boolean"
}
},
"category_applied_input_types": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"category_scores": {
"type": "object",
"additionalProperties": {
"type": "number",
"format": "float64"
}
},
"flagged": {
"type": "boolean"
}
}
},
"schema.MultimediaSourceConfig": {
"type": "object",
"properties": {
@@ -7068,6 +7221,10 @@ const docTemplate = `{
"schema.SysInfoModel": {
"type": "object",
"properties": {
"backend": {
"description": "Backend is the engine serving this model. The loader knows only the ID,\nso it is resolved from the model's config; empty when the model was\nloaded without one (a loose file, or a config since removed).",
"type": "string"
},
"id": {
"type": "string"
}

157
swagger/swagger.json
View File

@@ -1547,6 +1547,34 @@
}
}
},
"/api/traces/summary": {
"get": {
"description": "Returns request, failure and latency totals over a recent window, plus a bucketed series for sparklines. Exists so callers wanting three numbers do not have to fetch the whole trace list and count it themselves.",
"produces": [
"application/json"
],
"tags": [
"monitoring"
],
"summary": "Summarize recent API traces",
"parameters": [
{
"type": "integer",
"description": "Window in hours (default 24, max 168)",
"name": "hours",
"in": "query"
}
],
"responses": {
"200": {
"description": "Counted trace totals",
"schema": {
"$ref": "#/definitions/middleware.TraceSummary"
}
}
}
}
},
"/api/traces/{id}": {
"get": {
"description": "Returns a single captured API exchange, including the request and response bodies omitted from the list response",
@@ -3268,6 +3296,33 @@
}
}
},
"/v1/moderations": {
"post": {
"tags": [
"moderation"
],
"summary": "Classify text for potentially harmful content.",
"parameters": [
{
"description": "query params",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/schema.ModerationRequest"
}
}
],
"responses": {
"200": {
"description": "Response",
"schema": {
"$ref": "#/definitions/schema.ModerationResponse"
}
}
}
}
},
"/v1/rerank": {
"post": {
"tags": [
@@ -4354,6 +4409,43 @@
}
}
},
"middleware.TraceBucket": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"errors": {
"type": "integer"
},
"start": {
"type": "string"
}
}
},
"middleware.TraceSummary": {
"type": "object",
"properties": {
"buckets": {
"type": "array",
"items": {
"$ref": "#/definitions/middleware.TraceBucket"
}
},
"errors": {
"type": "integer"
},
"p95_ms": {
"type": "integer"
},
"total": {
"type": "integer"
},
"window_hours": {
"type": "integer"
}
}
},
"model.BackendLogLine": {
"type": "object",
"properties": {
@@ -5976,6 +6068,67 @@
}
}
},
"schema.ModerationRequest": {
"type": "object",
"properties": {
"input": {
"type": "array",
"items": {
"type": "string"
}
},
"model": {
"type": "string"
}
}
},
"schema.ModerationResponse": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"model": {
"type": "string"
},
"results": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.ModerationResult"
}
}
}
},
"schema.ModerationResult": {
"type": "object",
"properties": {
"categories": {
"type": "object",
"additionalProperties": {
"type": "boolean"
}
},
"category_applied_input_types": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"category_scores": {
"type": "object",
"additionalProperties": {
"type": "number",
"format": "float64"
}
},
"flagged": {
"type": "boolean"
}
}
},
"schema.MultimediaSourceConfig": {
"type": "object",
"properties": {
@@ -7065,6 +7218,10 @@
"schema.SysInfoModel": {
"type": "object",
"properties": {
"backend": {
"description": "Backend is the engine serving this model. The loader knows only the ID,\nso it is resolved from the model's config; empty when the model was\nloaded without one (a loose file, or a config since removed).",
"type": "string"
},
"id": {
"type": "string"
}

107
swagger/swagger.yaml
View File

@@ -438,6 +438,30 @@ definitions:
$ref: '#/definitions/voiceprofile.Profile'
type: array
type: object
middleware.TraceBucket:
properties:
count:
type: integer
errors:
type: integer
start:
type: string
type: object
middleware.TraceSummary:
properties:
buckets:
items:
$ref: '#/definitions/middleware.TraceBucket'
type: array
errors:
type: integer
p95_ms:
type: integer
total:
type: integer
window_hours:
type: integer
type: object
model.BackendLogLine:
properties:
stream:
@@ -1558,6 +1582,46 @@ definitions:
object:
type: string
type: object
schema.ModerationRequest:
properties:
input:
items:
type: string
type: array
model:
type: string
type: object
schema.ModerationResponse:
properties:
id:
type: string
model:
type: string
results:
items:
$ref: '#/definitions/schema.ModerationResult'
type: array
type: object
schema.ModerationResult:
properties:
categories:
additionalProperties:
type: boolean
type: object
category_applied_input_types:
additionalProperties:
items:
type: string
type: array
type: object
category_scores:
additionalProperties:
format: float64
type: number
type: object
flagged:
type: boolean
type: object
schema.MultimediaSourceConfig:
properties:
headers:
@@ -2337,6 +2401,12 @@ definitions:
type: object
schema.SysInfoModel:
properties:
backend:
description: |-
Backend is the engine serving this model. The loader knows only the ID,
so it is resolved from the model's config; empty when the model was
loaded without one (a loose file, or a config since removed).
type: string
id:
type: string
type: object
@@ -3837,6 +3907,26 @@ paths:
summary: Clear API traces
tags:
- monitoring
/api/traces/summary:
get:
description: Returns request, failure and latency totals over a recent window,
plus a bucketed series for sparklines. Exists so callers wanting three numbers
do not have to fetch the whole trace list and count it themselves.
parameters:
- description: Window in hours (default 24, max 168)
in: query
name: hours
type: integer
produces:
- application/json
responses:
"200":
description: Counted trace totals
schema:
$ref: '#/definitions/middleware.TraceSummary'
summary: Summarize recent API traces
tags:
- monitoring
/api/voice-profiles:
get:
description: List saved voice-cloning references without exposing filesystem
@@ -4954,6 +5044,23 @@ paths:
summary: List available models enriched with capabilities and input/output modalities.
tags:
- models
/v1/moderations:
post:
parameters:
- description: query params
in: body
name: request
required: true
schema:
$ref: '#/definitions/schema.ModerationRequest'
responses:
"200":
description: Response
schema:
$ref: '#/definitions/schema.ModerationResponse'
summary: Classify text for potentially harmful content.
tags:
- moderation
/v1/rerank:
post:
parameters: