feat(addons): move connector file bytes by handle

A connector add-on that declares files.transfer no longer carries file
bytes in the connector envelope. The host puts a random handle in the
request of the op media (write) or send_file (read), and the add-on
streams the file with two new host operations, connector.file.put and
connector.file.get, in chunks of at most 256 KiB.

The process that runs the add-on registers the handle when the call
arrives and ends it when the call returns. A handle is bound to the
add-on, so another add-on gets the same answer as for an unknown
handle. An unfinished inbound file is dropped with its partial bytes,
and a handle with no call for two minutes expires. Inbound bytes go
to the blob store of the workspace, outbound bytes come from it, so a
cluster needs the shared object store and no shared disk. The file is
never held in memory. The host overwrites the size and hash of an
inbound file with its own record.

The change is additive: the field files.transfer needs manifest API 2,
a kind without it keeps the inline path and its 1 MiB cap, and the host
never sends it a handle. A call that carries a handle may run up to 15
minutes. The ceiling of a file also stays under the largest file of a
workspace.

Assisted-by: Claude:claude-sonnet-5-5 go test

pkg/addons/contract/connector.go

@@ -435,6 +435,11 @@ type ConnectorRequest struct {
 	// send_file: the file. Its name is SafeFileName of itself and its data
 	// is 1 to MaxConnectorFileBytes long.
 	File *ConnectorFile `json:"file,omitempty"`
+	// Transfer is the handle for the bytes of a file, set only for a kind
+	// that declares files.transfer. media: the add-on writes the file with
+	// ConnectorFilePutOp. send_file: File is nil and the add-on reads the
+	// file with ConnectorFileGetOp.
+	Transfer *ConnectorTransfer `json:"transfer,omitempty"`
 }
 
 // ConnectorFile is a file that the op media returns or the op send_file
@@ -468,6 +473,9 @@ type ConnectorResponse struct {
 	BotUsername string `json:"botUsername,omitempty"`
 	// media
 	File *ConnectorFile `json:"file,omitempty"`
+	// Received answers the op media of a request with a Transfer: the file
+	// was written by handle. The host adds Size and SHA256 from its record.
+	Received *ConnectorReceived `json:"received,omitempty"`
 	// history: the messages, oldest first, at most the Limit asked for.
 	// Truncated says that older messages exist that are not returned.
 	Messages  []ConnectorHistoryMessage `json:"messages,omitempty"`
@@ -503,6 +511,18 @@ type ConnectorRuntime interface {
 // MaxPollSeconds for the platform.
 const ConnectorTimeout = (MaxPollSeconds + 20) * time.Second
 
+// ConnectorTransferTimeout is how long a call that carries a Transfer may
+// run: it moves a whole file chunk by chunk, and a large file takes minutes.
+const ConnectorTransferTimeout = 15 * time.Minute
+
+// CallTimeout is the time one connector call may run.
+func (r ConnectorRequest) CallTimeout() time.Duration {
+	if r.Transfer != nil {
+		return ConnectorTransferTimeout
+	}
+	return ConnectorTimeout
+}
+
 // SafeFileName returns a file name that every platform takes: the last
 // element of a path, without control characters, quotes or path separators,
 // cut to MaxConnectorFileName bytes on a character boundary and keeping the

pkg/addons/contract/connectorfiles.go

@@ -40,6 +40,11 @@ type ConnectorFiles struct {
 	Receive *FileAbility `json:"receive,omitempty"`
 	// Send is the ability to upload files from the bot.
 	Send *FileAbility `json:"send,omitempty"`
+	// Transfer says that the add-on moves file bytes by a short-lived handle
+	// (ConnectorTransfer, ConnectorFilePutOp, ConnectorFileGetOp) and not in
+	// the connector envelope. Only then may a limit pass MaxConnectorFileBytes.
+	// A kind that declares files without it keeps the inline path.
+	Transfer bool `json:"transfer,omitempty"`
 }
 
 var mediaTypePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9!#$&^_.+-]{0,63}/(\*|[a-z0-9][a-z0-9!#$&^_.+-]{0,63})$`)
@@ -79,3 +84,74 @@ func (f *ConnectorFiles) validate(api VersionRange) error {
 	}
 	return f.Send.validate("send")
 }
+
+// Host operations with which an add-on moves the bytes of a file by handle. A
+// handle is issued by the host with one connector call (ConnectorRequest.Transfer),
+// is single use, bound to that call, that add-on and that binding, and ends
+// when the call ends. The add-on needs the operation prefix "connector." in
+// its host policy.
+const (
+	// ConnectorFilePutOp writes the next chunk of an inbound file.
+	ConnectorFilePutOp = "connector.file.put"
+	// ConnectorFileGetOp reads the next chunk of an outbound file.
+	ConnectorFileGetOp = "connector.file.get"
+	// TransferChunkBytes is the largest chunk of data of one put or get. It
+	// is far below the payload limit of a host operation after base64.
+	TransferChunkBytes = 256 << 10
+	// MaxTransferHandle bounds the length of a handle.
+	MaxTransferHandle = 128
+)
+
+// ConnectorTransfer is the handle that the host gives to the add-on with the
+// ops media (inbound, write only) and send_file (outbound, read only).
+type ConnectorTransfer struct {
+	// Handle is random and means nothing to the add-on.
+	Handle string `json:"handle"`
+	// MaxBytes is the largest inbound file the host takes, the effective
+	// limit. The add-on should refuse a larger file before it downloads it.
+	MaxBytes int64 `json:"maxBytes,omitempty"`
+	// Name, MIME and Size describe the outbound file. Name is already safe.
+	Name string `json:"name,omitempty"`
+	MIME string `json:"mime,omitempty"`
+	Size int64  `json:"size,omitempty"`
+}
+
+// ConnectorFilePut is the argument object of ConnectorFilePutOp. Offset must
+// equal the bytes the host already has; when it does not, the host answers
+// with Received and the add-on resumes there. Final ends the file.
+type ConnectorFilePut struct {
+	Handle string `json:"handle"`
+	Offset int64  `json:"offset"`
+	Data   []byte `json:"data,omitempty"`
+	Final  bool   `json:"final,omitempty"`
+}
+
+// ConnectorFilePutResult is the answer of ConnectorFilePutOp.
+type ConnectorFilePutResult struct {
+	Received int64 `json:"received"`
+}
+
+// ConnectorFileGet is the argument object of ConnectorFileGetOp. Length is at
+// most TransferChunkBytes.
+type ConnectorFileGet struct {
+	Handle string `json:"handle"`
+	Offset int64  `json:"offset"`
+	Length int64  `json:"length"`
+}
+
+// ConnectorFileGetResult is the answer of ConnectorFileGetOp. EOF is true
+// when Data ends the file.
+type ConnectorFileGetResult struct {
+	Data []byte `json:"data,omitempty"`
+	EOF  bool   `json:"eof,omitempty"`
+}
+
+// ConnectorReceived is what the add-on answers to the op media when it wrote
+// the file by handle: Name and MIME. The host that holds the handle overwrites
+// Size and SHA256 with its own record, so the add-on cannot choose them.
+type ConnectorReceived struct {
+	Name   string `json:"name"`
+	MIME   string `json:"mime,omitempty"`
+	Size   int64  `json:"size,omitempty"`
+	SHA256 string `json:"sha256,omitempty"`
+}