mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 17:18:49 -04:00
mudler-agentandEttore Di Giacinto
3b179ed142
* ci(gallery): have the checksum bot write the model library The checksum bot edited gallery/index.yaml, which is a generated copy of gallery/models/index.yaml. The model library check fails any PR where the copy differs from the generated one, and an edit made to the copy never reaches the source. Point the checker at the library, read the files under .model.files as the library keeps them, and run modelcatalog after the checker so the PR carries both files. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci(gallery): have the other index bots write the model library The gallery agent and the variant proposer still read and wrote the generated gallery copy, so their pull requests failed the model library check and their changes never reached the source. The gallery agent now appends library entries to gallery/models/index.yaml, converting its entries with the gallery package and skipping ids the library already has. Its duplicate check reads the same file. The variant proposer decodes the library shape (the body under model:) and splices variants into the model body through a library layout in galleryedit. The anchor refusal goes: the library has no anchors. Its tests use library entries. Both workflows run modelcatalog after the bot and before the pull request step, and use the Go version from go.mod. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
66 lines
2.4 KiB
Bash
66 lines
2.4 KiB
Bash
#!/bin/bash
|
|
# This scripts needs yq and huggingface_hub to be installed
|
|
# to install hugingface_hub run pip install huggingface_hub
|
|
|
|
# Path to the input YAML file: the model library, gallery/models/index.yaml.
|
|
# Its entries keep the files under .model.files. The caller regenerates the
|
|
# gallery copy, gallery/index.yaml, after this script ran.
|
|
input_yaml=$1
|
|
|
|
# Function to download file and check checksum using Python
|
|
function check_and_update_checksum() {
|
|
model_name="$1"
|
|
file_name="$2"
|
|
uri="$3"
|
|
old_checksum="$4"
|
|
idx="$5"
|
|
|
|
# Download the file and calculate new checksum using Python
|
|
new_checksum=$(python3 ./.github/check_and_update.py $uri)
|
|
result=$?
|
|
|
|
if [[ $result -eq 5 ]]; then
|
|
echo "Contaminated entry detected, deleting entry for $model_name..."
|
|
yq eval -i "del([$idx])" "$input_yaml"
|
|
return
|
|
fi
|
|
|
|
if [[ "$new_checksum" == "" ]]; then
|
|
echo "Error calculating checksum for $file_name. Skipping..."
|
|
return
|
|
fi
|
|
|
|
echo "Checksum for $file_name: $new_checksum"
|
|
|
|
# Compare and update the YAML file if checksums do not match
|
|
|
|
if [[ $result -eq 2 ]]; then
|
|
echo "File not found, deleting entry for $file_name..."
|
|
# yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\"))" "$input_yaml"
|
|
elif [[ "$old_checksum" != "$new_checksum" ]]; then
|
|
echo "Checksum mismatch for $file_name. Updating..."
|
|
yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\").sha256)" "$input_yaml"
|
|
yq eval -i "(.[$idx].model.files[] | select(.filename == \"$file_name\")).sha256 = \"$new_checksum\"" "$input_yaml"
|
|
elif [[ $result -ne 0 ]]; then
|
|
echo "Error downloading file $file_name. Skipping..."
|
|
else
|
|
echo "Checksum match for $file_name. No update needed."
|
|
fi
|
|
}
|
|
|
|
# Read the YAML and process each file
|
|
len=$(yq eval '. | length' "$input_yaml")
|
|
for ((i=0; i<$len; i++))
|
|
do
|
|
name=$(yq eval ".[$i].name" "$input_yaml")
|
|
files_len=$(yq eval ".[$i].model.files | length" "$input_yaml")
|
|
for ((j=0; j<$files_len; j++))
|
|
do
|
|
filename=$(yq eval ".[$i].model.files[$j].filename" "$input_yaml")
|
|
uri=$(yq eval ".[$i].model.files[$j].uri" "$input_yaml")
|
|
checksum=$(yq eval ".[$i].model.files[$j].sha256" "$input_yaml")
|
|
echo "Checking model $name, file $filename. URI = $uri, Checksum = $checksum"
|
|
check_and_update_checksum "$name" "$filename" "$uri" "$checksum" "$i"
|
|
done
|
|
done
|