mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 10:28:43 -04:00
b00422e45f
* feat(backends): add LongCat video and avatar generation Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] [web] * refactor(config): declare model I/O modalities Make model configs declare input and output modalities so capability discovery no longer branches on backend or checkpoint names. Complete the LongCat gallery and user documentation, make the SDPA patch apply to the pinned upstream revision, and stabilize the Agent Jobs race exposed by the required hook. Assisted-by: Codex:GPT-5 [web] --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
109 lines
3.0 KiB
Go
109 lines
3.0 KiB
Go
package backend
|
|
|
|
import (
|
|
"maps"
|
|
"time"
|
|
|
|
"github.com/mudler/LocalAI/core/config"
|
|
"github.com/mudler/LocalAI/core/trace"
|
|
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
|
model "github.com/mudler/LocalAI/pkg/model"
|
|
)
|
|
|
|
// VideoGenerationOptions is the backend-neutral request passed to video generators.
|
|
// Media fields contain staged local paths by the time they reach this layer.
|
|
type VideoGenerationOptions struct {
|
|
Height int32
|
|
Width int32
|
|
Prompt string
|
|
NegativePrompt string
|
|
StartImage string
|
|
EndImage string
|
|
Audio string
|
|
Destination string
|
|
NumFrames int32
|
|
FPS int32
|
|
Seed int32
|
|
CFGScale float32
|
|
Step int32
|
|
Params map[string]string
|
|
}
|
|
|
|
func VideoGeneration(options VideoGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
|
|
opts := ModelOptions(modelConfig, appConfig)
|
|
inferenceModel, err := loader.Load(opts...)
|
|
if err != nil {
|
|
recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
|
|
return nil, err
|
|
}
|
|
|
|
fn := func() error {
|
|
_, err := inferenceModel.GenerateVideo(
|
|
appConfig.Context,
|
|
&proto.GenerateVideoRequest{
|
|
Height: options.Height,
|
|
Width: options.Width,
|
|
Prompt: options.Prompt,
|
|
NegativePrompt: options.NegativePrompt,
|
|
StartImage: options.StartImage,
|
|
EndImage: options.EndImage,
|
|
Audio: options.Audio,
|
|
NumFrames: options.NumFrames,
|
|
Fps: options.FPS,
|
|
Seed: options.Seed,
|
|
CfgScale: options.CFGScale,
|
|
Step: options.Step,
|
|
Dst: options.Destination,
|
|
Params: maps.Clone(options.Params),
|
|
},
|
|
)
|
|
return err
|
|
}
|
|
|
|
if appConfig.EnableTracing {
|
|
trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes)
|
|
|
|
traceData := map[string]any{
|
|
"prompt": options.Prompt,
|
|
"negative_prompt": options.NegativePrompt,
|
|
"height": options.Height,
|
|
"width": options.Width,
|
|
"num_frames": options.NumFrames,
|
|
"fps": options.FPS,
|
|
"seed": options.Seed,
|
|
"cfg_scale": options.CFGScale,
|
|
"step": options.Step,
|
|
"has_start_image": options.StartImage != "",
|
|
"has_end_image": options.EndImage != "",
|
|
"has_audio": options.Audio != "",
|
|
}
|
|
|
|
startTime := time.Now()
|
|
originalFn := fn
|
|
fn = func() error {
|
|
err := originalFn()
|
|
duration := time.Since(startTime)
|
|
|
|
errStr := ""
|
|
if err != nil {
|
|
errStr = err.Error()
|
|
}
|
|
|
|
trace.RecordBackendTrace(trace.BackendTrace{
|
|
Timestamp: startTime,
|
|
Duration: duration,
|
|
Type: trace.BackendTraceVideoGeneration,
|
|
ModelName: modelConfig.Name,
|
|
Backend: modelConfig.Backend,
|
|
Summary: trace.TruncateString(options.Prompt, 200),
|
|
Error: errStr,
|
|
Data: traceData,
|
|
})
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
return fn, nil
|
|
}
|