mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-11-06 12:33:16 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
3ca176d15db7bfdc280bed40811ade1cb336fcc3
LocalAI/.github/workflows/addons-package.yml
T
Ettore Di Giacinto b15905e90f fix: restore the files that the previous commit removed by mistake 
The previous commit was made on a stale base and so reverted work that
landed on master before it. This commit puts every file back to the
state of its parent's parent. The permission enforcement of the
previous commit stays.

Assisted-by: Claude:claude-sonnet-5-5 go
2026-11-03 14:33:05 +00:00

579 lines
24 KiB
YAML
Raw Blame History

---
name: 'add-on packages'
# Builds, checks, signs and publishes the first-party add-on OCI packages
# (Tables, Voice, Face, Fine-tuning, Quantization, Studio, Talk, Telegram, Slack, Office and
# Sandbox). This workflow is
# independent of the backend image workflows: it has its own registry input,
# its own signer identity and its own jobs.
#
# Event policy (the rules live in scripts/addons/channel.sh):
# pull_request validate and build legs, publish nothing, no secrets
# push to master publish the development channel
# push to other branch publish nothing
# push of a v<semver> tag prepare the immutable version, move no alias
# release published promote latest, for a stable non-draft release only
#
# Every add-on has its own platform file (addons/<id>/package/platforms.yaml)
# and its own package legs. All add-ons publish to one OCI repository, the
# value of ADDON_OCI_REPOSITORY (the add-on registry organization). Each
# add-on is a set of tags of that repository, prefixed with its id:
# <id>-sha-<short sha> for a commit, <id>-<version> for a release and
# <id>-development or <id>-latest for a channel alias. The development alias is
# the tag a development library entry names (mutable, no digest); the install
# resolves it to a digest, as for a development backend, so it moves only in
# the promote job below. Signatures and digests
# are per image, so add-ons never collide. A publishing event fails for an
# add-on until one of its platforms is enabled.
#
# Aliases move only in the promote job, after every platform leg passed, the
# index was pushed by digest, signed and verified.
#
# Catalog. gallery/addons/index.yaml holds the development entries (mutable, one
# for each enabled platform) and is generated by scripts/addons/catalog.sh; the
# validate job fails when the committed file is stale. A v<semver> tag push
# also renders the release index, with entries pinned by digest and by the
# immutable tag and signed by the run of the tag, and keeps it as the artifact
# addons-release-index. When the release is published, the release-index job
# attaches it to the GitHub release as addons-index.yaml. A release build of
# LocalAI reads that file (core/services/addons/catalog/official.go), so it
# installs the packages its own release built, not the moving development ones.
#
# Sandbox guest image. The sandbox-image job (sandbox-images.yml) builds and
# signs the guest image and exposes its reference by digest as the output
# "image" (<repository>@sha256:<digest>). The package job passes it to the
# Sandbox package as SANDBOX_GUEST_IMAGE; a publishing event fails without it.
# An event that publishes nothing pushes no image, so its Sandbox leg is a
# local-check build.
on:
pull_request:
paths:
- 'addons/**'
- 'pkg/addons/**'
- 'gallery/addons/**'
- 'tests/addons/packaging/**'
- 'core/services/addons/**'
- 'core/http/react-ui/package.json'
- 'core/http/react-ui/package-lock.json'
- 'go.mod'
- 'go.sum'
- '.github/workflows/addons-package.yml'
- '.github/workflows/sandbox-images.yml'
- 'Makefile'
- 'scripts/**'
- 'backend/*.proto'
- 'backend/*/*.proto'
- 'docs/content/features/addons.md'
- '.github/actions/**'
push:
branches: [master]
tags: ['v*']
paths:
- 'addons/**'
- 'pkg/addons/**'
- 'gallery/addons/**'
- 'tests/addons/packaging/**'
- 'core/services/addons/**'
- 'core/http/react-ui/package.json'
- 'core/http/react-ui/package-lock.json'
- 'go.mod'
- 'go.sum'
- '.github/workflows/addons-package.yml'
- '.github/workflows/sandbox-images.yml'
- 'Makefile'
- 'scripts/**'
- 'backend/*.proto'
- 'backend/*/*.proto'
- 'docs/content/features/addons.md'
- '.github/actions/**'
release:
types: [published]
concurrency:
group: ci-addons-package-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
jobs:
validate:
name: validate package rules
runs-on: ubuntu-24.04
timeout-minutes: 20
permissions:
contents: read
strategy:
fail-fast: true
matrix:
addon: [tables, voice, face, fine-tuning, quantization, studio, talk, telegram, slack, github, office, sandbox, web, memory]
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- uses: actions/setup-node@v7
with:
node-version: '22'
- name: Install the host UI build tools
# The package checks build the add-on UI with the bundler the host UI
# already installs.
run: npm ci --prefix core/http/react-ui
- name: Generate the gRPC stubs the packaging checks compile
# The generated stubs are not in the tree and the Go packages of the
# host import them. Same pinned tools as the test workflow.
if: matrix.addon == 'tables'
run: |
curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip
sudo unzip -j -d /usr/local/bin protoc.zip bin/protoc
rm protoc.zip
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
PATH="$PATH:$HOME/go/bin" make protogen-go
- name: Build the UI of this add-on
env:
ADDON: ${{ matrix.addon }}
run: node "addons/$ADDON/ui/build.mjs"
- name: Fill the module cache the packaging checks read
# package_test.sh runs offline (GOPROXY=off) against its own cache.
# The suite covers every add-on, so one leg runs it.
if: matrix.addon == 'tables'
run: |
GOPATH="$PWD/.cache/addons/go" GOMODCACHE="$PWD/.cache/addons/go/pkg/mod" \
GOCACHE="$PWD/.cache/addons/go-build" go mod download
- name: Run the packaging checks
if: matrix.addon == 'tables'
run: sh scripts/addons/package_test.sh
- name: Check that the committed catalog is the generated one
# One entry for each enabled platform of each add-on.
if: matrix.addon == 'tables'
run: sh scripts/addons/catalog.sh --channel development --check
sandbox-image:
name: sandbox guest image
# Builds the guest image of the Sandbox add-on. The reusable workflow
# decides from the event whether it pushes and signs, as this workflow
# does, and exposes the reference of the image by digest as "digest".
uses: ./.github/workflows/sandbox-images.yml
secrets: inherit
permissions:
contents: read
packages: write
id-token: write
plan:
name: plan the event
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: read
env:
ADDON_OCI_REPOSITORY: ${{ vars.ADDON_OCI_REPOSITORY }}
outputs:
publish: ${{ steps.plan.outputs.publish }}
immutable: ${{ steps.plan.outputs.immutable }}
aliases: ${{ steps.plan.outputs.aliases }}
channel: ${{ steps.plan.outputs.channel }}
version: ${{ steps.plan.outputs.version }}
matrix: ${{ steps.plan.outputs.matrix }}
addons: ${{ steps.plan.outputs.addons }}
has_platforms: ${{ steps.plan.outputs.has_platforms }}
registry: ${{ steps.plan.outputs.registry }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Decide what this event may publish
id: plan
env:
EVENT: ${{ github.event_name }}
REF: ${{ github.ref }}
SHA: ${{ github.sha }}
DRAFT: ${{ github.event.release.draft || false }}
PRERELEASE: ${{ github.event.release.prerelease || false }}
run: |
set -eu
flags=""
[ "$DRAFT" = "true" ] && flags="$flags --draft"
[ "$PRERELEASE" = "true" ] && flags="$flags --prerelease"
# shellcheck disable=SC2086
sh scripts/addons/channel.sh --event "$EVENT" --ref "$REF" --sha "$SHA" $flags > decision.txt
cat decision.txt
version=$(sed -n 's/^version=//p' decision.txt)
publish=$(sed -n 's/^publish=//p' decision.txt)
[ -n "$version" ] || version="0.0.0-dev.${SHA%"${SHA#????????????}"}"
if [ "$publish" = "true" ]; then
# Fail closed: a publishing event needs the namespace input and a
# non-empty release platform matrix. There is no default registry.
if [ -z "$ADDON_OCI_REPOSITORY" ]; then
echo "ADDON_OCI_REPOSITORY is not configured; refusing to publish" >&2
exit 1
fi
case "$ADDON_OCI_REPOSITORY" in
*/*) ;;
*) echo "ADDON_OCI_REPOSITORY needs a namespace and a name" >&2; exit 1 ;;
esac
fi
# One set of legs for each add-on, from its own platform file. Only
# the "no enabled platform" answer is tolerated, and only on an event
# that publishes nothing: that add-on gets no legs. On a publishing
# event it fails the plan. A missing or malformed platform file, or
# any other error, fails the plan on every event.
include='[]'
for id in tables voice face fine-tuning quantization studio talk telegram slack github office sandbox web memory; do
cfg="addons/$id/package/platforms.yaml"
err="matrix-$id.err"
if rows=$(go run ./scripts/addons/cmd/addonpkg matrix --config "$cfg" 2>"$err"); then
:
elif grep -q 'no enabled platform' "$err"; then
if [ "$publish" = "true" ]; then
echo "add-on $id has no enabled platform; refusing to publish" >&2
exit 1
fi
continue
else
echo "cannot read the platform matrix of add-on $id" >&2
cat "$err" >&2
exit 1
fi
include=$(printf '%s' "$rows" | jq -c --argjson all "$include" --arg addon "$id" '$all + [.include[] | . + {addon: $addon}]')
done
matrix=$(printf '%s' "$include" | jq -c '{include: .}')
addons=$(printf '%s' "$include" | jq -c 'map(.addon) | unique')
has=$(printf '%s' "$include" | jq 'length > 0')
{
cat decision.txt
echo "version=$version"
echo "matrix=$matrix"
echo "addons=$addons"
echo "has_platforms=$has"
echo "registry=${ADDON_OCI_REPOSITORY%%/*}"
} >> "$GITHUB_OUTPUT"
package:
name: package ${{ matrix.addon }} ${{ matrix.platform }}
needs: [plan, validate, sandbox-image]
if: needs.plan.outputs.has_platforms == 'true'
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
permissions:
contents: read
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- uses: actions/setup-node@v7
with:
node-version: '22'
- name: Build the UI
env:
ADDON: ${{ matrix.addon }}
run: |
npm ci --prefix core/http/react-ui
node "addons/$ADDON/ui/build.mjs"
- name: Fetch the runtime files
env:
ADDON: ${{ matrix.addon }}
PLATFORM: ${{ matrix.platform }}
run: |
# An add-on that ships files besides its executable describes where
# they come from; the script checks their checksums.
if [ -f "addons/$ADDON/package/fetch-runtime.sh" ]; then
sh "addons/$ADDON/package/fetch-runtime.sh" "$PLATFORM" "$RUNNER_TEMP/runtime"
fi
- name: Package this platform
env:
ADDON: ${{ matrix.addon }}
VERSION: ${{ needs.plan.outputs.version }}
PLATFORM: ${{ matrix.platform }}
SHA: ${{ github.sha }}
PUBLISH: ${{ needs.plan.outputs.publish }}
# The guest image of the Sandbox add-on, by digest. scripts/addons/
# package.sh stamps it into the executable.
SANDBOX_GUEST_IMAGE: ${{ matrix.addon == 'sandbox' && needs.sandbox-image.outputs.image || '' }}
run: |
set -- --config "addons/$ADDON/package/platforms.yaml" --layout "layout" --version "$VERSION" --platform "$PLATFORM" --revision "$SHA"
if [ "$ADDON" = sandbox ] && [ "$PUBLISH" = true ]; then
# Fail closed: a published Sandbox package without a guest image
# can run the host backend only.
case "$SANDBOX_GUEST_IMAGE" in
*@sha256:????????????????????????????????????????????????????????????????) ;;
*) echo "SANDBOX_GUEST_IMAGE must be <repository>@sha256:<digest>; the sandbox-image job gave '$SANDBOX_GUEST_IMAGE'" >&2; exit 1 ;;
esac
elif [ "$ADDON" = sandbox ] && [ -z "$SANDBOX_GUEST_IMAGE" ]; then
# An event that publishes nothing pushes no guest image, so there
# is no digest to stamp. Build a local-check leg: package.sh then
# uses a stand-in reference and the receipt is not publishable.
set -- "$@" --local-check
fi
if [ -d "$RUNNER_TEMP/runtime" ]; then set -- "$@" --runtime-dir "$RUNNER_TEMP/runtime"; fi
sh scripts/addons/package.sh "$@"
- name: Native runtime check
env:
ADDON: ${{ matrix.addon }}
VERSION: ${{ needs.plan.outputs.version }}
PLATFORM: ${{ matrix.platform }}
run: |
go run ./scripts/addons/cmd/addonpkg native-check --config "addons/$ADDON/package/platforms.yaml" --layout layout --platform "$PLATFORM" --version "$VERSION"
- uses: actions/upload-artifact@v7
with:
name: layout-${{ matrix.addon }}-${{ matrix.slug }}
path: layout
if-no-files-found: error
retention-days: 7
publish:
name: publish and sign ${{ matrix.addon }}
needs: [plan, validate, package]
if: needs.plan.outputs.publish == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 60
concurrency:
group: addons-publish-${{ matrix.addon }}-${{ needs.plan.outputs.immutable }}
cancel-in-progress: false
permissions:
contents: read
id-token: write
strategy:
# A leg that is pushing must finish, so one failure never cancels another.
fail-fast: false
matrix:
addon: ${{ fromJSON(needs.plan.outputs.addons) }}
env:
ADDON_OCI_REPOSITORY: ${{ vars.ADDON_OCI_REPOSITORY }}
ADDON: ${{ matrix.addon }}
# cosign gates --registry-referrers-mode=oci-1-1 behind this flag.
COSIGN_EXPERIMENTAL: '1'
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- uses: actions/download-artifact@v8
with:
pattern: layout-${{ matrix.addon }}-*
path: legs
- name: Join the platform legs into one index
id: merge
env:
VERSION: ${{ needs.plan.outputs.version }}
SHA: ${{ github.sha }}
run: |
set -eu
go run ./scripts/addons/cmd/addonpkg merge --config "addons/$ADDON/package/platforms.yaml" \
--out merged --version "$VERSION" --revision "$SHA" legs/*
- name: Registry login
uses: docker/login-action@v4
with:
registry: ${{ needs.plan.outputs.registry }}
username: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
password: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
- name: Push the platform digests and the index
id: push
env:
IMMUTABLE: ${{ needs.plan.outputs.immutable }}
run: |
set -eu
go run ./scripts/addons/cmd/addonpkg target --repository "$ADDON_OCI_REPOSITORY" --addon "$ADDON" --tag "$IMMUTABLE"
go run ./scripts/addons/cmd/addonpkg publish --config "addons/$ADDON/package/platforms.yaml" \
--layout merged --repository "$ADDON_OCI_REPOSITORY" --tag "$IMMUTABLE" --adopt-existing | tee push.txt
echo "digest=$(sed -n 's/^digest=//p' push.txt)" >> "$GITHUB_OUTPUT"
echo "children=$(sed -n 's/^children=//p' push.txt)" >> "$GITHUB_OUTPUT"
- uses: sigstore/cosign-installer@v3
with:
cosign-release: 'v2.6.5'
- name: Sign by digest
id: sign
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: |
set -eu
cosign sign --yes --recursive \
--new-bundle-format \
--registry-referrers-mode=oci-1-1 \
"${ADDON_OCI_REPOSITORY}@${DIGEST}"
- name: Verify the signature with the exact add-on workflow identity
id: verify
env:
DIGEST: ${{ steps.push.outputs.digest }}
CHILDREN: ${{ steps.push.outputs.children }}
IDENTITY: https://github.com/${{ github.repository }}/.github/workflows/addons-package.yml@${{ github.ref }}
ISSUER: https://token.actions.githubusercontent.com
run: |
set -eu
for d in "$DIGEST" $(echo "$CHILDREN" | tr ',' ' '); do
cosign verify --new-bundle-format \
--certificate-identity "$IDENTITY" \
--certificate-oidc-issuer "$ISSUER" \
"${ADDON_OCI_REPOSITORY}@${d}" > /dev/null
done
- name: Keep the merged layout for the catalog job
# The release index is rendered from the layout that was pushed.
uses: actions/upload-artifact@v7
with:
name: merged-${{ matrix.addon }}
path: merged
if-no-files-found: error
retention-days: 7
- name: Keep the digest for the promote job
# A matrix job cannot hand one output to the next job, so the digest of
# each add-on travels in an artifact of its own.
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: printf '%s\n' "$DIGEST" > digest.txt
- uses: actions/upload-artifact@v7
with:
name: digest-${{ matrix.addon }}
path: digest.txt
if-no-files-found: error
retention-days: 7
promote:
name: promote channel aliases ${{ matrix.addon }}
needs: [plan, validate, publish]
if: needs.plan.outputs.publish == 'true' && needs.plan.outputs.aliases != ''
runs-on: ubuntu-24.04
timeout-minutes: 20
# One promotion at a time per channel, and a queued run never cancels a
# running one. The freshness check below makes a slow, older run a no-op.
concurrency:
group: addons-promote-${{ matrix.addon }}-${{ needs.plan.outputs.channel }}
cancel-in-progress: false
permissions:
contents: read
strategy:
fail-fast: false
matrix:
addon: ${{ fromJSON(needs.plan.outputs.addons) }}
env:
ADDON_OCI_REPOSITORY: ${{ vars.ADDON_OCI_REPOSITORY }}
ADDON: ${{ matrix.addon }}
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- uses: actions/download-artifact@v8
with:
name: digest-${{ matrix.addon }}
path: digest
- name: Registry login
uses: docker/login-action@v4
with:
registry: ${{ needs.plan.outputs.registry }}
username: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
password: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
- name: Move the aliases
env:
ALIASES: ${{ needs.plan.outputs.aliases }}
VERSION: ${{ needs.plan.outputs.version }}
SHA: ${{ github.sha }}
REF: ${{ github.ref }}
EVENT: ${{ github.event_name }}
run: |
set -eu
DIGEST=$(tr -d '[:space:]' < digest/digest.txt)
case "$DIGEST" in
sha256:????????????????????????????????????????????????????????????????) ;;
*) echo "the digest of $ADDON is missing or malformed" >&2; exit 1 ;;
esac
for alias in $(echo "$ALIASES" | tr ',' ' '); do
go run ./scripts/addons/cmd/addonpkg promote \
--repository "$ADDON_OCI_REPOSITORY" --addon "$ADDON" --alias "$alias" \
--digest "$DIGEST" --version "$VERSION" --revision "$SHA" \
--ref "$REF" --event "$EVENT" --git-dir .git
done
catalog:
name: render the release catalog
# Only the push of a v<semver> tag: the entries are pinned to the version
# that run published and signed. A release event rebuilds the packages and
# adopts the published ones, so its own layout is not the pinned content.
needs: [plan, publish]
if: needs.plan.outputs.publish == 'true' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
runs-on: ubuntu-24.04
timeout-minutes: 20
permissions:
contents: read
env:
ADDON_OCI_REPOSITORY: ${{ vars.ADDON_OCI_REPOSITORY }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
- uses: actions/download-artifact@v8
with:
pattern: merged-*
path: downloaded
- name: Render the entries of every add-on
env:
VERSION: ${{ needs.plan.outputs.version }}
REF: ${{ github.ref }}
OWNER_REPO: ${{ github.repository }}
run: |
set -eu
mkdir roots
for d in downloaded/merged-*; do
mv "$d" "roots/${d##*/merged-}"
done
sh scripts/addons/catalog.sh --channel release --repository "$ADDON_OCI_REPOSITORY" \
--version "$VERSION" --signer-ref "$REF" --owner-repo "$OWNER_REPO" \
--merged-root roots --out addons-index.yaml
grep -c '^- id:' addons-index.yaml
- uses: actions/upload-artifact@v7
with:
name: addons-release-index
path: addons-index.yaml
if-no-files-found: error
# The longest the service keeps an artifact. The release must be
# published within this time of the tag.
retention-days: 90
release-index:
name: attach the catalog to the release
# The release is public from here on, so the asset can be attached whether
# or not the release was a draft when the tag was pushed.
needs: [plan]
if: github.event_name == 'release' && !github.event.release.draft
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: write
actions: read
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
steps:
- name: Find the tag run and attach its release index
env:
TAG: ${{ github.event.release.tag_name }}
SHA: ${{ github.sha }}
run: |
set -eu
run=$(gh run list --workflow addons-package.yml --event push --branch "$TAG" --status success \
--json databaseId,headSha --jq "[.[] | select(.headSha == \"$SHA\")][0].databaseId // empty")
[ -n "$run" ] || { echo "no successful add-on package run for the tag $TAG at $SHA" >&2; exit 1; }
gh run download "$run" --name addons-release-index --dir index
test -s index/addons-index.yaml
gh release upload "$TAG" index/addons-index.yaml --clobber
Reference in new issue View Git Blame Copy Permalink