feat(addons): move connector file bytes by handle

A connector add-on that declares files.transfer no longer carries file
bytes in the connector envelope. The host puts a random handle in the
request of the op media (write) or send_file (read), and the add-on
streams the file with two new host operations, connector.file.put and
connector.file.get, in chunks of at most 256 KiB.

The process that runs the add-on registers the handle when the call
arrives and ends it when the call returns. A handle is bound to the
add-on, so another add-on gets the same answer as for an unknown
handle. An unfinished inbound file is dropped with its partial bytes,
and a handle with no call for two minutes expires. Inbound bytes go
to the blob store of the workspace, outbound bytes come from it, so a
cluster needs the shared object store and no shared disk. The file is
never held in memory. The host overwrites the size and hash of an
inbound file with its own record.

The change is additive: the field files.transfer needs manifest API 2,
a kind without it keeps the inline path and its 1 MiB cap, and the host
never sends it a handle. A call that carries a handle may run up to 15
minutes. The ceiling of a file also stays under the largest file of a
workspace.

Assisted-by: Claude:claude-sonnet-5-5 go test

core/application/addon_gateway.go

@@ -302,7 +302,7 @@ func (a *Application) startAddonGateway() error {
 	if mode == auth.AddonModeLocalOwner {
 		xlog.Warn(localOwnerWarning)
 	}
-	g := &gateway.Gateway{Limits: c.Limits, Manager: a.addonManager, ConnectorAuthority: connectorAuthority{a}, Revalidate: auth.AddonRevalidate(a.authDB, mode)}
+	g := &gateway.Gateway{Transfers: a.transfers(), Limits: c.Limits, Manager: a.addonManager, ConnectorAuthority: connectorAuthority{a}, Revalidate: auth.AddonRevalidate(a.authDB, mode)}
 	if cfg.InstanceID != cfg.OwnerID {
 		g.Forward = carrier
 		a.addonGateway = g

core/application/addon_policy_reader.go

@@ -68,6 +68,16 @@ func (a *Application) studioToolPins(ctx context.Context) (map[string]string, er
 // the owner policy. Without add-ons there is no owner policy and no connector
 // to move a file, so the default stands.
 func (a *Application) studioConnectorFilesCeiling(ctx context.Context) (int64, error) {
+	n, err := a.policyFilesCeiling(ctx)
+	if err != nil {
+		return 0, err
+	}
+	// A file that a sender shares becomes a file of the chat, so the ceiling
+	// never passes what one file of a workspace may be.
+	return min(n, a.studioUploadLimit()), nil
+}
+
+func (a *Application) policyFilesCeiling(ctx context.Context) (int64, error) {
 	if err := ctx.Err(); err != nil {
 		return 0, err
 	}

core/application/application.go

@@ -26,6 +26,7 @@ import (
 	"github.com/mudler/LocalAI/core/services/addons/backendlib"
 	"github.com/mudler/LocalAI/core/services/addons/catalog"
 	"github.com/mudler/LocalAI/core/services/addons/gateway"
+	"github.com/mudler/LocalAI/core/services/addons/hostops"
 	"github.com/mudler/LocalAI/core/services/addons/hostsettings"
 	"github.com/mudler/LocalAI/core/services/addons/packages"
 	"github.com/mudler/LocalAI/core/services/addons/pins"
@@ -113,9 +114,12 @@ type Application struct {
 	backendLibrary  *backendlib.Installer
 	addonManager    *addons.Manager
 	addonHostOps    *addonruntime.HostOps
-	addonDataRoot   string // parent of the private data directory of each add-on; empty when none runs here
-	addonsProbe     addonsProbeState
-	modelsProbe     modelsProbeState
+	// addonTransfers holds the handles of connector file transfers; see transfers.
+	addonTransfers     *hostops.Transfers
+	addonTransfersOnce sync.Once
+	addonDataRoot      string // parent of the private data directory of each add-on; empty when none runs here
+	addonsProbe        addonsProbeState
+	modelsProbe        modelsProbeState
 
 	// libraryStore is the library sources file when no add-on owner runs here.
 	libraryStore   *catalog.Sources

core/application/studio_connectors.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"time"
 
 	"github.com/mudler/xlog"
 
@@ -63,7 +64,16 @@ func (t connectorTransport) Call(ctx context.Context, scope studio.ConnectorCall
 // function as a message the manager pulls itself. A dedicated add-on owner
 // refuses it instead.
 func (a *Application) RegisterAddonConnector() error {
-	if a.addonHostOps == nil || a.studioConnectors == nil {
+	if a.addonHostOps == nil {
+		return nil
+	}
+	// The two file operations serve every connector add-on that declares
+	// files.transfer; the handles come with the connector calls. They are
+	// registered wherever add-ons run, also on a dedicated owner.
+	if err := hostops.RegisterConnectorTransfer(a.addonHostOps.Broker, a.transfers()); err != nil {
+		return err
+	}
+	if a.studioConnectors == nil {
 		return nil
 	}
 	if a.ownsAddonsOnly() {
@@ -130,3 +140,17 @@ func (a *Application) startStudioConnectors(svc *studio.Service) {
 	a.studioConnectors, a.studioConnectorsStop = c, cancel
 	go c.Run(ctx)
 }
+
+// transfers is the table of live connector file handles of this process. The
+// gateway checks and ends them; the broker serves the bytes. Its blob store is
+// the one of the workspace, set when the files start.
+func (a *Application) transfers() *hostops.Transfers {
+	a.addonTransfersOnce.Do(func() {
+		t := hostops.NewTransfers(nil)
+		t.Audit = func(o hostops.TransferOutcome) {
+			xlog.Info("Connector file transfer", "direction", o.Direction, "binding", o.BindingID, "bytes", o.Bytes, "result", o.Result, "duration", o.Duration.Round(time.Millisecond))
+		}
+		a.addonTransfers = t
+	})
+	return a.addonTransfers
+}

core/application/studio_files.go

@@ -108,6 +108,7 @@ func (a *Application) initStudioFilesWithS3(newS3 func(context.Context, storage.
 	if err != nil {
 		return stop("workspace blob store", err, "Files are off: their storage could not be opened.")
 	}
+	a.transfers().UseBlobs(blobs)
 	ws, closer, err := a.openWorkspaceIndex(root, blobs)
 	if err != nil {
 		return stop("workspace index", err, "Files are off: the workspace could not be started.")