chore(model): add silero-vad model config

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.bruno/LocalAI Test Requests/model gallery/model delete.bru

@@ -0,0 +1,11 @@
+meta {
+  name: model delete
+  type: http
+  seq: 7
+}
+
+post {
+  url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries
+  body: none
+  auth: none
+}

.bruno/LocalAI Test Requests/transcription/transcribe.bru

@@ -0,0 +1,16 @@
+meta {
+  name: transcribe
+  type: http
+  seq: 1
+}
+
+post {
+  url: {{PROTOCOL}}{{HOST}}:{{PORT}}/v1/audio/transcriptions
+  body: multipartForm
+  auth: none
+}
+
+body:multipart-form {
+  file: @file(transcription/gb1.ogg)
+  model: whisper-1
+}

.gitattributes

@@ -1 +1,2 @@
 *.sh text eol=lf
+backend/cpp/llama/*.hpp linguist-vendored

.github/workflows/checksum_checker.yaml

@@ -23,7 +23,7 @@ jobs:
           sudo pip install --upgrade pip
           pip install huggingface_hub
       - name: 'Setup yq'
-        uses: dcarbone/install-yq-action@v1.1.1
+        uses: dcarbone/install-yq-action@v1.3.1
         with:
           version: 'v4.44.2'
           download-compressed: true

.github/workflows/deploy-explorer.yaml

@@ -33,7 +33,7 @@ jobs:
         run: |
           CGO_ENABLED=0 make build-api
       - name: rm
-        uses: appleboy/ssh-action@v1.1.0
+        uses: appleboy/ssh-action@v1.2.0
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}
@@ -53,7 +53,7 @@ jobs:
             rm: true
             target: ./local-ai
       - name: restarting
-        uses: appleboy/ssh-action@v1.1.0
+        uses: appleboy/ssh-action@v1.2.0
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}

.github/workflows/generate_intel_image.yaml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - base-image: intel/oneapi-basekit:2024.2.0-devel-ubuntu22.04
+          - base-image: intel/oneapi-basekit:2025.0.0-0-devel-ubuntu22.04
             runs-on: 'ubuntu-latest'
             platforms: 'linux/amd64'
     runs-on: ${{matrix.runs-on}}

.github/workflows/test-extra.yml

@@ -123,6 +123,13 @@ jobs:
         run: |
            make --jobs=5 --output-sync=target -C backend/python/parler-tts
            make --jobs=5 --output-sync=target -C backend/python/parler-tts test
+      - name: Setup tmate session if tests fail
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3.19
+        with:
+          detached: true
+          connect-timeout-seconds: 180
+          limit-access-to-actor: true
 
   tests-openvoice:
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -224,7 +224,7 @@ jobs:
       - name: Dependencies
         run: |
           brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm
-          pip install --user --no-cache-dir grpcio-tools==1.64.1
+          pip install --user --no-cache-dir grpcio-tools
       - name: Test
         run: |
           export C_INCLUDE_PATH=/usr/local/include

.gitignore

@@ -12,7 +12,6 @@ prepare-sources
 
 go-ggml-transformers
 go-gpt2
-go-rwkv
 whisper.cpp
 /bloomz
 go-bert

Dockerfile

@@ -85,7 +85,8 @@ WORKDIR /build
 # The requirements-extras target is for any builds with IMAGE_TYPE=extras. It should not be placed in this target unless every IMAGE_TYPE=extras build will use it
 FROM requirements-core AS requirements-extras
 
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+# Install uv as a system package
+RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/bin sh
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

Makefile

@@ -8,15 +8,11 @@ DETECT_LIBS?=true
 # llama.cpp versions
 GOLLAMA_REPO?=https://github.com/go-skynet/go-llama.cpp
 GOLLAMA_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
-CPPLLAMA_VERSION?=0a1c750c80147687df267114c81956757cc14382
-
-# go-rwkv version
-RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
-RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6
+CPPLLAMA_VERSION?=47f931c8f9a26c072d71224bc8013cc66ea9e445
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp
-WHISPER_CPP_VERSION?=0fbaac9c891055796456df7b9122a70c220f9ca1
+WHISPER_CPP_VERSION?=6266a9f9e56a5b925e9892acf650f3eb1245814d
 
 # bert.cpp version
 BERT_REPO?=https://github.com/go-skynet/go-bert.cpp
@@ -24,7 +20,7 @@ BERT_VERSION?=710044b124545415f555e4260d16b146c725a6e4
 
 # go-piper version
 PIPER_REPO?=https://github.com/mudler/go-piper
-PIPER_VERSION?=9d0100873a7dbb0824dfea40e8cec70a1b110759
+PIPER_VERSION?=e10ca041a885d4a8f3871d52924b47792d5e5aa0
 
 # stablediffusion version
 STABLEDIFFUSION_REPO?=https://github.com/mudler/go-stable-diffusion
@@ -34,6 +30,10 @@ STABLEDIFFUSION_VERSION?=4a3cd6aeae6f66ee57eae9a0075f8c58c3a6a38f
 TINYDREAM_REPO?=https://github.com/M0Rf30/go-tiny-dream
 TINYDREAM_VERSION?=c04fa463ace9d9a6464313aa5f9cd0f953b6c057
 
+ONNX_VERSION?=1.20.0
+ONNX_ARCH?=x64
+ONNX_OS?=linux
+
 export BUILD_TYPE?=
 export STABLE_BUILD_TYPE?=$(BUILD_TYPE)
 export CMAKE_ARGS?=
@@ -45,6 +45,7 @@ CGO_LDFLAGS_WHISPER+=-lggml
 CUDA_LIBPATH?=/usr/local/cuda/lib64/
 GO_TAGS?=
 BUILD_ID?=
+NATIVE?=false
 
 TEST_DIR=/tmp/test
 
@@ -83,7 +84,25 @@ ifndef UNAME_S
 UNAME_S := $(shell uname -s)
 endif
 
+# IF native is false, we add -DGGML_NATIVE=OFF to CMAKE_ARGS
+ifeq ($(NATIVE),false)
+	CMAKE_ARGS+=-DGGML_NATIVE=OFF
+endif
+
+# Detect if we are running on arm64
+ifneq (,$(findstring aarch64,$(shell uname -m)))
+	ONNX_ARCH=aarch64
+endif
+
 ifeq ($(OS),Darwin)
+	ONNX_OS=osx
+	ifneq (,$(findstring aarch64,$(shell uname -m)))
+		ONNX_ARCH=arm64
+	else ifneq (,$(findstring arm64,$(shell uname -m)))
+		ONNX_ARCH=arm64
+	else
+		ONNX_ARCH=x86_64
+	endif
 
 	ifeq ($(OSX_SIGNING_IDENTITY),)
 		OSX_SIGNING_IDENTITY := $(shell security find-identity -v -p codesigning | grep '"' | head -n 1 | sed -E 's/.*"(.*)"/\1/')
@@ -138,10 +157,10 @@ ifeq ($(BUILD_TYPE),hipblas)
 	export CC=$(ROCM_HOME)/llvm/bin/clang
 	# llama-ggml has no hipblas support, so override it here.
 	export STABLE_BUILD_TYPE=
-	export GGML_HIPBLAS=1
+	export GGML_HIP=1
 	GPU_TARGETS ?= gfx900,gfx906,gfx908,gfx940,gfx941,gfx942,gfx90a,gfx1030,gfx1031,gfx1100,gfx1101
 	AMDGPU_TARGETS ?= "$(GPU_TARGETS)"
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS="$(AMDGPU_TARGETS)" -DGPU_TARGETS="$(GPU_TARGETS)"
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS="$(AMDGPU_TARGETS)" -DGPU_TARGETS="$(GPU_TARGETS)"
 	CGO_LDFLAGS += -O3 --rtlib=compiler-rt -unwindlib=libgcc -lhipblas -lrocblas --hip-link -L${ROCM_HOME}/lib/llvm/lib
 endif
 
@@ -186,9 +205,9 @@ ALL_GRPC_BACKENDS+=backend-assets/grpc/llama-cpp-fallback
 ALL_GRPC_BACKENDS+=backend-assets/grpc/llama-ggml
 ALL_GRPC_BACKENDS+=backend-assets/grpc/llama-cpp-grpc
 ALL_GRPC_BACKENDS+=backend-assets/util/llama-cpp-rpc-server
-ALL_GRPC_BACKENDS+=backend-assets/grpc/rwkv
 ALL_GRPC_BACKENDS+=backend-assets/grpc/whisper
 ALL_GRPC_BACKENDS+=backend-assets/grpc/local-store
+ALL_GRPC_BACKENDS+=backend-assets/grpc/silero-vad
 ALL_GRPC_BACKENDS+=$(OPTIONAL_GRPC)
 # Use filter-out to remove the specified backends
 ALL_GRPC_BACKENDS := $(filter-out $(SKIP_GRPC_BACKEND),$(ALL_GRPC_BACKENDS))
@@ -248,20 +267,6 @@ sources/go-piper:
 sources/go-piper/libpiper_binding.a: sources/go-piper
 	$(MAKE) -C sources/go-piper libpiper_binding.a example/main piper.o
 
-
-## RWKV
-sources/go-rwkv.cpp:
-	mkdir -p sources/go-rwkv.cpp
-	cd sources/go-rwkv.cpp && \
-	git init && \
-	git remote add origin $(RWKV_REPO) && \
-	git fetch origin && \
-	git checkout $(RWKV_VERSION) && \
-	git submodule update --init --recursive --depth 1 --single-branch
-
-sources/go-rwkv.cpp/librwkv.a: sources/go-rwkv.cpp
-	cd sources/go-rwkv.cpp && cd rwkv.cpp &&	cmake . -DRWKV_BUILD_SHARED_LIBRARY=OFF &&	cmake --build . && 	cp librwkv.a ..
-
 ## stable diffusion
 sources/go-stable-diffusion:
 	mkdir -p sources/go-stable-diffusion
@@ -275,6 +280,20 @@ sources/go-stable-diffusion:
 sources/go-stable-diffusion/libstablediffusion.a: sources/go-stable-diffusion
 	CPATH="$(CPATH):/usr/include/opencv4" $(MAKE) -C sources/go-stable-diffusion libstablediffusion.a
 
+sources/onnxruntime:
+	mkdir -p sources/onnxruntime
+	curl -L https://github.com/microsoft/onnxruntime/releases/download/v$(ONNX_VERSION)/onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz -o sources/onnxruntime/onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz
+	cd sources/onnxruntime && tar -xvf onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz && rm onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz
+	cd sources/onnxruntime && mv onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION)/* ./
+
+backend-assets/lib/libonnxruntime.so.1: backend-assets/lib sources/onnxruntime
+	cp -rfv sources/onnxruntime/lib/* backend-assets/lib/
+ifeq ($(OS),Darwin)
+	mv backend-assets/lib/libonnxruntime.$(ONNX_VERSION).dylib backend-assets/lib/libonnxruntime.dylib
+else
+	mv backend-assets/lib/libonnxruntime.so.$(ONNX_VERSION) backend-assets/lib/libonnxruntime.so.1
+endif
+
 ## tiny-dream
 sources/go-tiny-dream:
 	mkdir -p sources/go-tiny-dream
@@ -301,10 +320,9 @@ sources/whisper.cpp:
 sources/whisper.cpp/libwhisper.a: sources/whisper.cpp
 	cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a
 
-get-sources: sources/go-llama.cpp sources/go-piper sources/go-rwkv.cpp sources/whisper.cpp sources/go-bert.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
+get-sources: sources/go-llama.cpp sources/go-piper sources/whisper.cpp sources/go-bert.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
 
 replace:
-	$(GOCMD) mod edit -replace github.com/donomii/go-rwkv.cpp=$(CURDIR)/sources/go-rwkv.cpp
 	$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(CURDIR)/sources/whisper.cpp
 	$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp/bindings/go=$(CURDIR)/sources/whisper.cpp/bindings/go
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-bert.cpp=$(CURDIR)/sources/go-bert.cpp
@@ -314,7 +332,6 @@ replace:
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(CURDIR)/sources/go-llama.cpp
 
 dropreplace:
-	$(GOCMD) mod edit -dropreplace github.com/donomii/go-rwkv.cpp
 	$(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp
 	$(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp/bindings/go
 	$(GOCMD) mod edit -dropreplace github.com/go-skynet/go-bert.cpp
@@ -330,7 +347,6 @@ prepare-sources: get-sources replace
 rebuild: ## Rebuilds the project
 	$(GOCMD) clean -cache
 	$(MAKE) -C sources/go-llama.cpp clean
-	$(MAKE) -C sources/go-rwkv.cpp clean
 	$(MAKE) -C sources/whisper.cpp clean
 	$(MAKE) -C sources/go-stable-diffusion clean
 	$(MAKE) -C sources/go-bert.cpp clean
@@ -439,8 +455,6 @@ test-models/testmodel.ggml:
 	wget -q https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin -O test-models/whisper-en
 	wget -q https://huggingface.co/mudler/all-MiniLM-L6-v2/resolve/main/ggml-model-q4_0.bin -O test-models/bert
 	wget -q https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav -O test-dir/audio.wav
-	wget -q https://huggingface.co/mudler/rwkv-4-raven-1.5B-ggml/resolve/main/RWKV-4-Raven-1B5-v11-Eng99%2525-Other1%2525-20230425-ctx4096_Q4_0.bin -O test-models/rwkv
-	wget -q https://raw.githubusercontent.com/saharNooby/rwkv.cpp/5eb8f09c146ea8124633ab041d9ea0b1f1db4459/rwkv/20B_tokenizer.json -O test-models/rwkv.tokenizer.json
 	cp tests/models_fixtures/* test-models
 
 prepare-test: grpcs
@@ -761,7 +775,7 @@ backend-assets/grpc/llama-cpp-fallback: backend-assets/grpc backend/cpp/llama/ll
 	cp -rfv backend/cpp/llama-fallback/grpc-server backend-assets/grpc/llama-cpp-fallback
 # TODO: every binary should have its own folder instead, so can have different metal implementations
 ifeq ($(BUILD_TYPE),metal)
-	cp backend/cpp/llama-fallback/llama.cpp/build/bin/default.metallib backend-assets/grpc/
+	cp backend/cpp/llama-fallback/llama.cpp/build/bin/ggml-metal.metal backend-assets/grpc/
 endif
 
 backend-assets/grpc/llama-cpp-cuda: backend-assets/grpc backend/cpp/llama/llama.cpp
@@ -775,7 +789,7 @@ backend-assets/grpc/llama-cpp-hipblas: backend-assets/grpc backend/cpp/llama/lla
 	cp -rf backend/cpp/llama backend/cpp/llama-hipblas
 	$(MAKE) -C backend/cpp/llama-hipblas purge
 	$(info ${GREEN}I llama-cpp build info:hipblas${RESET})
-	BUILD_TYPE="hipblas" $(MAKE) VARIANT="llama-hipblas" build-llama-cpp-grpc-server
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" BUILD_TYPE="hipblas" $(MAKE) VARIANT="llama-hipblas" build-llama-cpp-grpc-server
 	cp -rfv backend/cpp/llama-hipblas/grpc-server backend-assets/grpc/llama-cpp-hipblas
 
 backend-assets/grpc/llama-cpp-sycl_f16: backend-assets/grpc backend/cpp/llama/llama.cpp
@@ -817,13 +831,6 @@ ifneq ($(UPX),)
 	$(UPX) backend-assets/grpc/piper
 endif
 
-backend-assets/grpc/rwkv: sources/go-rwkv.cpp sources/go-rwkv.cpp/librwkv.a backend-assets/grpc
-	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(CURDIR)/sources/go-rwkv.cpp LIBRARY_PATH=$(CURDIR)/sources/go-rwkv.cpp \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./backend/go/llm/rwkv
-ifneq ($(UPX),)
-	$(UPX) backend-assets/grpc/rwkv
-endif
-
 backend-assets/grpc/stablediffusion: sources/go-stable-diffusion sources/go-stable-diffusion/libstablediffusion.a backend-assets/grpc
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" CPATH="$(CPATH):$(CURDIR)/sources/go-stable-diffusion/:/usr/include/opencv4" LIBRARY_PATH=$(CURDIR)/sources/go-stable-diffusion/ \
 	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/stablediffusion
@@ -831,6 +838,13 @@ ifneq ($(UPX),)
 	$(UPX) backend-assets/grpc/stablediffusion
 endif
 
+backend-assets/grpc/silero-vad: backend-assets/grpc backend-assets/lib/libonnxruntime.so.1
+	CGO_LDFLAGS="$(CGO_LDFLAGS)" CPATH="$(CPATH):$(CURDIR)/sources/onnxruntime/include/" LIBRARY_PATH=$(CURDIR)/backend-assets/lib \
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/silero-vad ./backend/go/vad/silero
+ifneq ($(UPX),)
+	$(UPX) backend-assets/grpc/silero-vad
+endif
+
 backend-assets/grpc/tinydream: sources/go-tiny-dream sources/go-tiny-dream/libtinydream.a backend-assets/grpc
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" LIBRARY_PATH=$(CURDIR)/go-tiny-dream \
 	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/tinydream ./backend/go/image/tinydream
@@ -891,7 +905,7 @@ docker-aio-all:
 
 docker-image-intel:
 	docker build \
-		--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.2.0-devel-ubuntu22.04 \
+		--build-arg BASE_IMAGE=intel/oneapi-basekit:2025.0.0-0-devel-ubuntu22.04 \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
 		--build-arg GO_TAGS="none" \
 		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
@@ -899,7 +913,7 @@ docker-image-intel:
 
 docker-image-intel-xpu:
 	docker build \
-		--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.2.0-devel-ubuntu22.04 \
+		--build-arg BASE_IMAGE=intel/oneapi-basekit:2025.0.0-0-devel-ubuntu22.04 \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
 		--build-arg GO_TAGS="none" \
 		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \

README.md

@@ -38,9 +38,13 @@
 </a>
 </p>
 
+<p align="center">
+<a href="https://trendshift.io/repositories/1484" target="_blank"><img src="https://trendshift.io/api/badge/repositories/1484" alt="go-skynet%2FLocalAI | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
+
 > :bulb: Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [:speech_balloon: Discord](https://discord.gg/uJAeKSAGDy) [:book: Documentation website](https://localai.io/)
 >
-> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🥽 Demo](https://demo.localai.io) [🌍 Explorer](https://explorer.localai.io) [🛫 Examples](https://github.com/go-skynet/LocalAI/tree/master/examples/) 
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🥽 Demo](https://demo.localai.io) [🌍 Explorer](https://explorer.localai.io) [🛫 Examples](https://github.com/mudler/LocalAI-examples) 
 
 [![tests](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml)[![Build and Release](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml)[![build container images](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml)[![Bump dependencies](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml)[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/localai)](https://artifacthub.io/packages/search?repo=localai)
 
@@ -56,14 +60,17 @@ curl https://localai.io/install.sh | sh
 
 Or run with docker:
 ```bash
+# CPU only image:
+docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-cpu
+
+# Nvidia GPU:
+docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12
+
+# CPU and GPU image (bigger size):
+docker run -ti --name local-ai -p 8080:8080 localai/localai:latest
+
+# AIO images (it will pre-download a set of models ready for use, see https://localai.io/basics/container/)
 docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-cpu
-# Alternative images:
-# - if you have an Nvidia GPU:
-# docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-12
-# - without preconfigured models
-# docker run -ti --name local-ai -p 8080:8080 localai/localai:latest
-# - without preconfigured models for Nvidia GPUs
-# docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12 
 ```
 
 To load models:
@@ -85,6 +92,8 @@ local-ai run oci://localai/phi-2:latest
 
 ## 📰 Latest project news
 
+- Nov 2024: Voice activity detection models (**VAD**) added to the API: https://github.com/mudler/LocalAI/pull/4204
+- Oct 2024: examples moved to [LocalAI-examples](https://github.com/mudler/LocalAI-examples)
 - Aug 2024:  🆕 FLUX-1, [P2P Explorer](https://explorer.localai.io)
 - July 2024: 🔥🔥 🆕 P2P Dashboard, LocalAI Federated mode and AI Swarms: https://github.com/mudler/LocalAI/pull/2723
 - June 2024: 🆕 You can browse now the model gallery without LocalAI! Check out https://models.localai.io
@@ -155,6 +164,9 @@ Other:
 - Slack bot https://github.com/mudler/LocalAGI/tree/main/examples/slack
 - Shell-Pilot(Interact with LLM using LocalAI models via pure shell scripts on your Linux or MacOS system) https://github.com/reid41/shell-pilot
 - Telegram bot https://github.com/mudler/LocalAI/tree/master/examples/telegram-bot
+- Another Telegram Bot https://github.com/JackBekket/Hellper
+- Auto-documentation https://github.com/JackBekket/Reflexia
+- Github bot which answer on issues, with code and documentation as context https://github.com/JackBekket/GitHelper
 - Github Actions: https://github.com/marketplace/actions/start-localai
 - Examples: https://github.com/mudler/LocalAI/tree/master/examples/
   
@@ -229,7 +241,6 @@ LocalAI couldn't have been built without the help of great software already avai
 - https://github.com/antimatter15/alpaca.cpp
 - https://github.com/EdVince/Stable-Diffusion-NCNN
 - https://github.com/ggerganov/whisper.cpp
-- https://github.com/saharNooby/rwkv.cpp
 - https://github.com/rhasspy/piper
 
 ## 🤗 Contributors

backend/backend.proto

@@ -28,6 +28,8 @@ service Backend {
   rpc Rerank(RerankRequest) returns (RerankResult) {}
 
   rpc GetMetrics(MetricsRequest) returns (MetricsResponse);
+
+  rpc VAD(VADRequest) returns (VADResponse) {}
 }
 
 // Define the empty request
@@ -233,6 +235,11 @@ message ModelOptions {
 
   bool FlashAttention = 56;
   bool NoKVOffload = 57;
+
+  string ModelPath = 59;
+
+  repeated string LoraAdapters = 60;
+  repeated float LoraScales = 61;
 }
 
 message Result {
@@ -288,6 +295,19 @@ message TTSRequest {
   optional string language = 5;
 }
 
+message VADRequest {
+  repeated float audio = 1;
+}
+
+message VADSegment {
+  float start = 1;
+  float end = 2;
+}
+
+message VADResponse {
+  repeated VADSegment segments = 1;
+}
+
 message SoundGenerationRequest {
   string text = 1;
   string model = 2;

backend/cpp/llama/Makefile

@@ -22,7 +22,7 @@ else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 # If it's hipblas we do have also to set CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++ 
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	CMAKE_ARGS+=-DGGML_HIP=ON
 # If it's OSX, DO NOT embed the metal library - -DGGML_METAL_EMBED_LIBRARY=ON requires further investigation
 # But if it's OSX without metal, disable it here
 else ifeq ($(OS),Darwin)

backend/cpp/llama/grpc-server.cpp

@@ -203,7 +203,7 @@ struct llama_client_slot
     std::string stopping_word;
 
     // sampling
-    struct common_sampler_params sparams;
+    struct common_params_sampling sparams;
     common_sampler *ctx_sampling = nullptr;
 
     int32_t ga_i = 0;   // group-attention state
@@ -662,7 +662,7 @@ struct llama_server_context
 
     bool launch_slot_with_data(llama_client_slot* &slot, json data) {
         slot_params default_params;
-        common_sampler_params default_sparams;
+        common_params_sampling default_sparams;
  
         slot->params.stream             = json_value(data, "stream",            false);
         slot->params.cache_prompt       = json_value(data, "cache_prompt",      false);
@@ -670,7 +670,6 @@ struct llama_server_context
         slot->sparams.top_k             = json_value(data, "top_k",             default_sparams.top_k);
         slot->sparams.top_p             = json_value(data, "top_p",             default_sparams.top_p);
         slot->sparams.min_p             = json_value(data, "min_p",             default_sparams.min_p);
-        slot->sparams.tfs_z             = json_value(data, "tfs_z",             default_sparams.tfs_z);
         slot->sparams.typ_p             = json_value(data, "typical_p",         default_sparams.typ_p);
         slot->sparams.temp              = json_value(data, "temperature",       default_sparams.temp);
         slot->sparams.dynatemp_range    = json_value(data, "dynatemp_range",    default_sparams.dynatemp_range);
@@ -1206,7 +1205,6 @@ struct llama_server_context
             {"top_k",             slot.sparams.top_k},
             {"top_p",             slot.sparams.top_p},
             {"min_p",             slot.sparams.min_p},
-            {"tfs_z",             slot.sparams.tfs_z},
             {"typical_p",         slot.sparams.typ_p},
             {"repeat_last_n",     slot.sparams.penalty_last_n},
             {"repeat_penalty",    slot.sparams.penalty_repeat},
@@ -2105,7 +2103,6 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, llama
     //     slot->params.n_predict        = json_value(data, "n_predict",         default_params.n_predict);
     //     slot->sparams.top_k           = json_value(data, "top_k",             default_sparams.top_k);
     //     slot->sparams.top_p           = json_value(data, "top_p",             default_sparams.top_p);
-    //     slot->sparams.tfs_z           = json_value(data, "tfs_z",             default_sparams.tfs_z);
     //     slot->sparams.typical_p       = json_value(data, "typical_p",         default_sparams.typical_p);
     //     slot->sparams.temp            = json_value(data, "temperature",       default_sparams.temp);
     //     slot->sparams.penalty_last_n  = json_value(data, "repeat_last_n",     default_sparams.penalty_last_n);
@@ -2129,7 +2126,6 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, llama
     data["n_predict"] = predict->tokens() == 0 ? -1 : predict->tokens();
     data["top_k"] = predict->topk();
     data["top_p"] = predict->topp();
-    data["tfs_z"] = predict->tailfreesamplingz();
     data["typical_p"] = predict->typicalp();
     data["temperature"] = predict->temperature();
     data["repeat_last_n"] = predict->repeat();
@@ -2176,7 +2172,6 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, llama
 //     llama.params.n_predict = predict->tokens() == 0 ? -1 : predict->tokens();
 //     llama.params.sparams.top_k = predict->topk();
 //     llama.params.sparams.top_p = predict->topp();
-//     llama.params.sparams.tfs_z = predict->tailfreesamplingz();
 //     llama.params.sparams.typical_p = predict->typicalp();
 //     llama.params.sparams.penalty_last_n = predict->repeat();
 //     llama.params.sparams.temp = predict->temperature();
@@ -2304,6 +2299,7 @@ static void params_parse(const backend::ModelOptions* request,
     params.use_mmap = request->mmap();
     params.flash_attn = request->flashattention();
     params.no_kv_offload = request->nokvoffload();
+    params.ctx_shift = false; // We control context-shifting in any case (and we disable it as it could just lead to infinite loops)
 
     params.embedding = request->embeddings();
 

backend/go/llm/rwkv/rwkv.go

@@ -1,95 +0,0 @@
-package main
-
-// This is a wrapper to statisfy the GRPC service interface
-// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
-import (
-	"fmt"
-	"path/filepath"
-
-	"github.com/donomii/go-rwkv.cpp"
-	"github.com/mudler/LocalAI/pkg/grpc/base"
-	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
-)
-
-const tokenizerSuffix = ".tokenizer.json"
-
-type LLM struct {
-	base.SingleThread
-
-	rwkv *rwkv.RwkvState
-}
-
-func (llm *LLM) Load(opts *pb.ModelOptions) error {
-	tokenizerFile := opts.Tokenizer
-	if tokenizerFile == "" {
-		modelFile := filepath.Base(opts.ModelFile)
-		tokenizerFile = modelFile + tokenizerSuffix
-	}
-	modelPath := filepath.Dir(opts.ModelFile)
-	tokenizerPath := filepath.Join(modelPath, tokenizerFile)
-
-	model := rwkv.LoadFiles(opts.ModelFile, tokenizerPath, uint32(opts.GetThreads()))
-
-	if model == nil {
-		return fmt.Errorf("rwkv could not load model")
-	}
-	llm.rwkv = model
-	return nil
-}
-
-func (llm *LLM) Predict(opts *pb.PredictOptions) (string, error) {
-	stopWord := "\n"
-	if len(opts.StopPrompts) > 0 {
-		stopWord = opts.StopPrompts[0]
-	}
-
-	if err := llm.rwkv.ProcessInput(opts.Prompt); err != nil {
-		return "", err
-	}
-
-	response := llm.rwkv.GenerateResponse(int(opts.Tokens), stopWord, float32(opts.Temperature), float32(opts.TopP), nil)
-
-	return response, nil
-}
-
-func (llm *LLM) PredictStream(opts *pb.PredictOptions, results chan string) error {
-	go func() {
-
-		stopWord := "\n"
-		if len(opts.StopPrompts) > 0 {
-			stopWord = opts.StopPrompts[0]
-		}
-
-		if err := llm.rwkv.ProcessInput(opts.Prompt); err != nil {
-			fmt.Println("Error processing input: ", err)
-			return
-		}
-
-		llm.rwkv.GenerateResponse(int(opts.Tokens), stopWord, float32(opts.Temperature), float32(opts.TopP), func(s string) bool {
-			results <- s
-			return true
-		})
-		close(results)
-	}()
-
-	return nil
-}
-
-func (llm *LLM) TokenizeString(opts *pb.PredictOptions) (pb.TokenizationResponse, error) {
-	tokens, err := llm.rwkv.Tokenizer.Encode(opts.Prompt)
-	if err != nil {
-		return pb.TokenizationResponse{}, err
-	}
-
-	l := len(tokens)
-	i32Tokens := make([]int32, l)
-
-	for i, t := range tokens {
-		i32Tokens[i] = int32(t.ID)
-	}
-
-	return pb.TokenizationResponse{
-		Length: int32(l),
-		Tokens: i32Tokens,
-	}, nil
-}

backend/go/vad/silero/main.go

@@ -15,7 +15,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
+	if err := grpc.StartServer(*addr, &VAD{}); err != nil {
 		panic(err)
 	}
 }

backend/go/vad/silero/vad.go

@@ -0,0 +1,54 @@
+package main
+
+// This is a wrapper to statisfy the GRPC service interface
+// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
+import (
+	"fmt"
+
+	"github.com/mudler/LocalAI/pkg/grpc/base"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/streamer45/silero-vad-go/speech"
+)
+
+type VAD struct {
+	base.SingleThread
+	detector *speech.Detector
+}
+
+func (vad *VAD) Load(opts *pb.ModelOptions) error {
+	v, err := speech.NewDetector(speech.DetectorConfig{
+		ModelPath:  opts.ModelFile,
+		SampleRate: 16000,
+		//WindowSize:           1024,
+		Threshold:            0.5,
+		MinSilenceDurationMs: 0,
+		SpeechPadMs:          0,
+	})
+	if err != nil {
+		return fmt.Errorf("create silero detector: %w", err)
+	}
+
+	vad.detector = v
+	return err
+}
+
+func (vad *VAD) VAD(req *pb.VADRequest) (pb.VADResponse, error) {
+	audio := req.Audio
+
+	segments, err := vad.detector.Detect(audio)
+	if err != nil {
+		return pb.VADResponse{}, fmt.Errorf("detect: %w", err)
+	}
+
+	vadSegments := []*pb.VADSegment{}
+	for _, s := range segments {
+		vadSegments = append(vadSegments, &pb.VADSegment{
+			Start: float32(s.SpeechStartAt),
+			End:   float32(s.SpeechEndAt),
+		})
+	}
+
+	return pb.VADResponse{
+		Segments: vadSegments,
+	}, nil
+}

backend/python/autogptq/requirements.txt

@@ -1,6 +1,6 @@
 accelerate
 auto-gptq==0.7.1
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 transformers

backend/python/bark/requirements.txt

@@ -1,4 +1,4 @@
 bark==0.1.5
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi

backend/python/common/template/Makefile

@@ -1,8 +1,9 @@
 .DEFAULT_GOAL := install
 
 .PHONY: install
-install: protogen
+install:
 	bash install.sh
+	$(MAKE) protogen
 
 .PHONY: protogen
 protogen: backend_pb2_grpc.py backend_pb2.py
@@ -12,7 +13,7 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	bash protogen.sh
 
 .PHONY: clean
 clean: protogen-clean

backend/python/common/template/protogen.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/../common/libbackend.sh
+
+python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto

backend/python/common/template/requirements.txt

@@ -1,2 +1,3 @@
-grpcio==1.67.0
-protobuf
+grpcio==1.68.0
+protobuf
+grpcio-tools

backend/python/coqui/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 packaging==24.1

backend/python/diffusers/backend.py

@@ -247,11 +247,16 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                         use_safetensors=True,
                         variant=variant)
             elif request.PipelineType == "FluxPipeline":
+                if fromSingleFile:
+                    self.pipe = FluxPipeline.from_single_file(modelFile,
+                                                              torch_dtype=torchType,
+                                                              use_safetensors=True)
+                else:
                     self.pipe = FluxPipeline.from_pretrained(
                         request.Model,
                         torch_dtype=torch.bfloat16)
-                    if request.LowVRAM:
-                        self.pipe.enable_model_cpu_offload()
+                if request.LowVRAM:
+                    self.pipe.enable_model_cpu_offload()
             elif request.PipelineType == "FluxTransformer2DModel":
                     dtype = torch.bfloat16
                     # specify from environment or default to "ChuckMcSneed/FLUX.1-dev"
@@ -296,22 +301,34 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 self.pipe.controlnet = self.controlnet
             else:
                 self.controlnet = None
-            # Assume directory from request.ModelFile.
-            # Only if request.LoraAdapter it's not an absolute path
-            if request.LoraAdapter and request.ModelFile != "" and not os.path.isabs(request.LoraAdapter) and request.LoraAdapter:
-                # get base path of modelFile
-                modelFileBase = os.path.dirname(request.ModelFile)
+
+            if request.LoraAdapter and not os.path.isabs(request.LoraAdapter):
                 # modify LoraAdapter to be relative to modelFileBase
-                request.LoraAdapter = os.path.join(modelFileBase, request.LoraAdapter)
+                request.LoraAdapter = os.path.join(request.ModelPath, request.LoraAdapter)
+
             device = "cpu" if not request.CUDA else "cuda"
             self.device = device
             if request.LoraAdapter:
                 # Check if its a local file and not a directory ( we load lora differently for a safetensor file )
                 if os.path.exists(request.LoraAdapter) and not os.path.isdir(request.LoraAdapter):
-                    # self.load_lora_weights(request.LoraAdapter, 1, device, torchType)
                     self.pipe.load_lora_weights(request.LoraAdapter)
                 else:
                     self.pipe.unet.load_attn_procs(request.LoraAdapter)
+            if len(request.LoraAdapters) > 0:
+                i = 0
+                adapters_name = []
+                adapters_weights = []
+                for adapter in request.LoraAdapters:
+                    if not os.path.isabs(adapter):
+                        adapter = os.path.join(request.ModelPath, adapter)
+                    self.pipe.load_lora_weights(adapter, adapter_name=f"adapter_{i}")
+                    adapters_name.append(f"adapter_{i}")
+                    i += 1
+
+                for adapters_weight in request.LoraScales:
+                    adapters_weights.append(adapters_weight)
+
+                self.pipe.set_adapters(adapters_name, adapter_weights=adapters_weights)
 
             if request.CUDA:
                 self.pipe.to('cuda')
@@ -392,8 +409,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         # create a dictionary of values for the parameters
         options = {
             "negative_prompt": request.negative_prompt,
-            "width": request.width,
-            "height": request.height,
             "num_inference_steps": steps,
         }
 
@@ -411,13 +426,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         keys = options.keys()
 
         if request.EnableParameters != "":
-            keys = request.EnableParameters.split(",")
+            keys = [key.strip() for key in request.EnableParameters.split(",")]
 
         if request.EnableParameters == "none":
             keys = []
 
         # create a dictionary of parameters by using the keys from EnableParameters and the values from defaults
-        kwargs = {key: options[key] for key in keys}
+        kwargs = {key: options.get(key) for key in keys if key in options}
 
         # Set seed
         if request.seed > 0:
@@ -428,6 +443,12 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         if self.PipelineType == "FluxPipeline":
             kwargs["max_sequence_length"] = 256
 
+        if request.width:
+            kwargs["width"] = request.width
+
+        if request.height:
+            kwargs["height"] = request.height
+
         if self.PipelineType == "FluxTransformer2DModel":
             kwargs["output_type"] = "pil"
             kwargs["generator"] = torch.Generator("cpu").manual_seed(0)
@@ -447,6 +468,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             export_to_video(video_frames, request.dst)
             return backend_pb2.Result(message="Media generated successfully", success=True)
 
+        print(f"Generating image with {kwargs=}", file=sys.stderr)
         image = {}
         if COMPEL:
             conditioning, pooled = self.compel.build_conditioning_tensor(prompt)

backend/python/diffusers/requirements.txt

@@ -1,5 +1,5 @@
 setuptools
-grpcio==1.67.0
+grpcio==1.68.0
 pillow
 protobuf
 certifi

backend/python/exllama2/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 wheel

backend/python/mamba/requirements.txt

@@ -1,3 +1,3 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi

backend/python/openvoice/requirements-intel.txt

@@ -2,7 +2,7 @@
 intel-extension-for-pytorch
 torch
 optimum[openvino]
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 librosa==0.9.1
 faster-whisper==0.9.0

backend/python/openvoice/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 librosa
 faster-whisper

backend/python/parler-tts/Makefile

@@ -12,9 +12,10 @@ export SKIP_CONDA=1
 endif
 
 .PHONY: parler-tts
-parler-tts: protogen
+parler-tts:
 	@echo "Installing $(CONDA_ENV_PATH)..."
 	bash install.sh $(CONDA_ENV_PATH)
+	$(MAKE) protogen
 
 .PHONY: run
 run: protogen
@@ -36,7 +37,7 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	bash protogen.sh
 
 .PHONY: clean
 clean: protogen-clean

backend/python/parler-tts/install.sh

@@ -11,16 +11,18 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then
     EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
 fi
 
+
 installRequirements
 
+
 # https://github.com/descriptinc/audiotools/issues/101
 # incompatible protobuf versions.
-# PYDIR=python3.10
-# pyenv="${MY_DIR}/venv/lib/${PYDIR}/site-packages/google/protobuf/internal/"
+PYDIR=python3.10
+pyenv="${MY_DIR}/venv/lib/${PYDIR}/site-packages/google/protobuf/internal/"
 
-# if [ ! -d ${pyenv} ]; then
-#     echo "(parler-tts/install.sh): Error: ${pyenv} does not exist"
-#     exit 1
-# fi
+if [ ! -d ${pyenv} ]; then
+    echo "(parler-tts/install.sh): Error: ${pyenv} does not exist"
+    exit 1
+fi
 
-# curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o ${pyenv}/builder.py
+curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o ${pyenv}/builder.py

backend/python/parler-tts/protogen.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/../common/libbackend.sh
+
+python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto

backend/python/parler-tts/requirements-after.txt

@@ -1,4 +1,4 @@
 git+https://github.com/huggingface/parler-tts.git@8e465f1b5fcd223478e07175cb40494d19ffbe17
 llvmlite==0.43.0
 numba==0.60.0
-git+https://github.com/descriptinc/audiotools
+grpcio-tools==1.42.0

backend/python/parler-tts/requirements.txt

@@ -1,4 +1,3 @@
-grpcio==1.67.0
-protobuf
+grpcio==1.68.0
 certifi
-llvmlite==0.43.0
+llvmlite==0.43.0

backend/python/rerankers/requirements.txt

@@ -1,3 +1,3 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi

backend/python/sentencetransformers/requirements-cpu.txt

@@ -2,5 +2,5 @@ torch==2.4.1
 accelerate
 transformers
 bitsandbytes
-sentence-transformers==3.2.0
+sentence-transformers==3.3.1
 transformers

backend/python/sentencetransformers/requirements-cublas11.txt

@@ -1,5 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/cu118
 torch==2.4.1+cu118
 accelerate
-sentence-transformers==3.2.0
+sentence-transformers==3.3.1
 transformers

backend/python/sentencetransformers/requirements-cublas12.txt

@@ -1,4 +1,4 @@
 torch==2.4.1
 accelerate
-sentence-transformers==3.2.0
+sentence-transformers==3.3.1
 transformers

backend/python/sentencetransformers/requirements-hipblas.txt

@@ -1,5 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/rocm6.0
 torch==2.4.1+rocm6.0
 accelerate
-sentence-transformers==3.2.0
+sentence-transformers==3.3.1
 transformers

backend/python/sentencetransformers/requirements-intel.txt

@@ -4,5 +4,5 @@ torch
 optimum[openvino]
 setuptools==69.5.1 # https://github.com/mudler/LocalAI/issues/2406
 accelerate
-sentence-transformers==3.2.0
+sentence-transformers==3.3.1
 transformers

backend/python/sentencetransformers/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 datasets

backend/python/transformers-musicgen/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 scipy==1.14.0
 certifi

backend/python/transformers/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 setuptools==69.5.1 # https://github.com/mudler/LocalAI/issues/2406

backend/python/vall-e-x/requirements.txt

@@ -1,3 +1,3 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi

backend/python/vllm/install.sh

@@ -22,7 +22,7 @@ if [ "x${BUILD_TYPE}" == "x" ] && [ "x${FROM_SOURCE}" == "xtrue" ]; then
             git clone https://github.com/vllm-project/vllm
         fi
         pushd vllm
-            uv pip install wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm grpcio==1.67.0 protobuf bitsandbytes
+            uv pip install wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm grpcio==1.68.0 protobuf bitsandbytes
             uv pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
             VLLM_TARGET_DEVICE=cpu python setup.py install
         popd

backend/python/vllm/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.67.0
+grpcio==1.68.0
 protobuf
 certifi
 setuptools

core/backend/embeddings.go

@@ -11,17 +11,9 @@ import (
 
 func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
 
-	var inferenceModel interface{}
-	var err error
+	opts := ModelOptions(backendConfig, appConfig)
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{})
-
-	if backendConfig.Backend == "" {
-		inferenceModel, err = loader.GreedyLoader(opts...)
-	} else {
-		opts = append(opts, model.WithBackendString(backendConfig.Backend))
-		inferenceModel, err = loader.BackendLoader(opts...)
-	}
+	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
 		return nil, err
 	}

core/backend/image.go

@@ -9,9 +9,8 @@ import (
 
 func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) {
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{})
-
-	inferenceModel, err := loader.BackendLoader(
+	opts := ModelOptions(backendConfig, appConfig)
+	inferenceModel, err := loader.Load(
 		opts...,
 	)
 	if err != nil {

core/backend/llm.go

@@ -16,7 +16,6 @@ import (
 	"github.com/mudler/LocalAI/core/schema"
 
 	"github.com/mudler/LocalAI/core/gallery"
-	"github.com/mudler/LocalAI/pkg/grpc"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	model "github.com/mudler/LocalAI/pkg/model"
 	"github.com/mudler/LocalAI/pkg/utils"
@@ -35,15 +34,6 @@ type TokenUsage struct {
 func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c config.BackendConfig, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
 	modelFile := c.Model
 
-	var inferenceModel grpc.Backend
-	var err error
-
-	opts := ModelOptions(c, o, []model.Option{})
-
-	if c.Backend != "" {
-		opts = append(opts, model.WithBackendString(c.Backend))
-	}
-
 	// Check if the modelFile exists, if it doesn't try to load it from the gallery
 	if o.AutoloadGalleries { // experimental
 		if _, err := os.Stat(modelFile); os.IsNotExist(err) {
@@ -56,12 +46,8 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
 		}
 	}
 
-	if c.Backend == "" {
-		inferenceModel, err = loader.GreedyLoader(opts...)
-	} else {
-		inferenceModel, err = loader.BackendLoader(opts...)
-	}
-
+	opts := ModelOptions(c, o)
+	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
 		return nil, err
 	}

core/backend/options.go

@@ -11,7 +11,7 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts []model.Option) []model.Option {
+func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
 	name := c.Name
 	if name == "" {
 		name = c.Model
@@ -125,6 +125,8 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
 		CFGScale:             c.Diffusers.CFGScale,
 		LoraAdapter:          c.LoraAdapter,
 		LoraScale:            c.LoraScale,
+		LoraAdapters:         c.LoraAdapters,
+		LoraScales:           c.LoraScales,
 		F16Memory:            f16,
 		LoraBase:             c.LoraBase,
 		IMG2IMG:              c.Diffusers.IMG2IMG,

core/backend/rerank.go

@@ -11,8 +11,8 @@ import (
 
 func Rerank(modelFile string, request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error) {
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{model.WithModel(modelFile)})
-	rerankModel, err := loader.BackendLoader(opts...)
+	opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
+	rerankModel, err := loader.Load(opts...)
 	if err != nil {
 		return nil, err
 	}

core/backend/soundgeneration.go

@@ -25,9 +25,8 @@ func SoundGeneration(
 	backendConfig config.BackendConfig,
 ) (string, *proto.Result, error) {
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{model.WithModel(modelFile)})
-
-	soundGenModel, err := loader.BackendLoader(opts...)
+	opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
+	soundGenModel, err := loader.Load(opts...)
 	if err != nil {
 		return "", nil, err
 	}

core/backend/stores.go

@@ -8,16 +8,15 @@ import (
 )
 
 func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) (grpc.Backend, error) {
-    if storeName == "" {
-      storeName = "default"
-    }
+	if storeName == "" {
+		storeName = "default"
+	}
 
-    sc := []model.Option{
-      model.WithBackendString(model.LocalStoreBackend),
-      model.WithAssetDir(appConfig.AssetsDestination),
-      model.WithModel(storeName),
-    }
+	sc := []model.Option{
+		model.WithBackendString(model.LocalStoreBackend),
+		model.WithAssetDir(appConfig.AssetsDestination),
+		model.WithModel(storeName),
+	}
 
-    return sl.BackendLoader(sc...)
+	return sl.Load(sc...)
 }
-

core/backend/token_metrics.go

@@ -15,10 +15,8 @@ func TokenMetrics(
 	appConfig *config.ApplicationConfig,
 	backendConfig config.BackendConfig) (*proto.MetricsResponse, error) {
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{
-		model.WithModel(modelFile),
-	})
-	model, err := loader.BackendLoader(opts...)
+	opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
+	model, err := loader.Load(opts...)
 	if err != nil {
 		return nil, err
 	}

core/backend/tokenize.go

@@ -14,15 +14,13 @@ func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.Bac
 	var inferenceModel grpc.Backend
 	var err error
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{
-		model.WithModel(modelFile),
-	})
+	opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
 
 	if backendConfig.Backend == "" {
-		inferenceModel, err = loader.GreedyLoader(opts...)
+		inferenceModel, err = loader.Load(opts...)
 	} else {
 		opts = append(opts, model.WithBackendString(backendConfig.Backend))
-		inferenceModel, err = loader.BackendLoader(opts...)
+		inferenceModel, err = loader.Load(opts...)
 	}
 	if err != nil {
 		return schema.TokenizeResponse{}, err

core/backend/transcript.go

@@ -18,9 +18,9 @@ func ModelTranscription(audio, language string, translate bool, ml *model.ModelL
 		backendConfig.Backend = model.WhisperBackend
 	}
 
-	opts := ModelOptions(backendConfig, appConfig, []model.Option{})
+	opts := ModelOptions(backendConfig, appConfig)
 
-	transcriptionModel, err := ml.BackendLoader(opts...)
+	transcriptionModel, err := ml.Load(opts...)
 	if err != nil {
 		return nil, err
 	}

core/backend/tts.go

@@ -28,11 +28,8 @@ func ModelTTS(
 		bb = model.PiperBackend
 	}
 
-	opts := ModelOptions(config.BackendConfig{}, appConfig, []model.Option{
-		model.WithBackendString(bb),
-		model.WithModel(modelFile),
-	})
-	ttsModel, err := loader.BackendLoader(opts...)
+	opts := ModelOptions(backendConfig, appConfig, model.WithBackendString(bb), model.WithModel(modelFile))
+	ttsModel, err := loader.Load(opts...)
 	if err != nil {
 		return "", nil, err
 	}

core/cli/worker/worker_p2p.go

@@ -76,8 +76,14 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
 					"util",
 					"llama-cpp-rpc-server",
 				)
-				extraArgs := strings.Split(r.ExtraLLamaCPPArgs, " ")
+				var extraArgs []string
+
+				if r.ExtraLLamaCPPArgs != "" {
+					extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ")
+				}
 				args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...)
+				log.Debug().Msgf("Starting llama-cpp-rpc-server on '%s:%d' with args: %+v (%d)", address, port, args, len(args))
+
 				args, grpcProcess = library.LoadLDSO(r.BackendAssetsPath, args, grpcProcess)
 
 				cmd := exec.Command(

core/config/backend_config.go

@@ -134,23 +134,25 @@ type LLMConfig struct {
 	TrimSpace       []string `yaml:"trimspace"`
 	TrimSuffix      []string `yaml:"trimsuffix"`
 
-	ContextSize          *int    `yaml:"context_size"`
-	NUMA                 bool    `yaml:"numa"`
-	LoraAdapter          string  `yaml:"lora_adapter"`
-	LoraBase             string  `yaml:"lora_base"`
-	LoraScale            float32 `yaml:"lora_scale"`
-	NoMulMatQ            bool    `yaml:"no_mulmatq"`
-	DraftModel           string  `yaml:"draft_model"`
-	NDraft               int32   `yaml:"n_draft"`
-	Quantization         string  `yaml:"quantization"`
-	LoadFormat           string  `yaml:"load_format"`
-	GPUMemoryUtilization float32 `yaml:"gpu_memory_utilization"` // vLLM
-	TrustRemoteCode      bool    `yaml:"trust_remote_code"`      // vLLM
-	EnforceEager         bool    `yaml:"enforce_eager"`          // vLLM
-	SwapSpace            int     `yaml:"swap_space"`             // vLLM
-	MaxModelLen          int     `yaml:"max_model_len"`          // vLLM
-	TensorParallelSize   int     `yaml:"tensor_parallel_size"`   // vLLM
-	MMProj               string  `yaml:"mmproj"`
+	ContextSize          *int      `yaml:"context_size"`
+	NUMA                 bool      `yaml:"numa"`
+	LoraAdapter          string    `yaml:"lora_adapter"`
+	LoraBase             string    `yaml:"lora_base"`
+	LoraAdapters         []string  `yaml:"lora_adapters"`
+	LoraScales           []float32 `yaml:"lora_scales"`
+	LoraScale            float32   `yaml:"lora_scale"`
+	NoMulMatQ            bool      `yaml:"no_mulmatq"`
+	DraftModel           string    `yaml:"draft_model"`
+	NDraft               int32     `yaml:"n_draft"`
+	Quantization         string    `yaml:"quantization"`
+	LoadFormat           string    `yaml:"load_format"`
+	GPUMemoryUtilization float32   `yaml:"gpu_memory_utilization"` // vLLM
+	TrustRemoteCode      bool      `yaml:"trust_remote_code"`      // vLLM
+	EnforceEager         bool      `yaml:"enforce_eager"`          // vLLM
+	SwapSpace            int       `yaml:"swap_space"`             // vLLM
+	MaxModelLen          int       `yaml:"max_model_len"`          // vLLM
+	TensorParallelSize   int       `yaml:"tensor_parallel_size"`   // vLLM
+	MMProj               string    `yaml:"mmproj"`
 
 	FlashAttention bool `yaml:"flash_attention"`
 	NoKVOffloading bool `yaml:"no_kv_offloading"`

core/config/backend_config_loader.go

@@ -140,7 +140,7 @@ func (bcl *BackendConfigLoader) LoadBackendConfigFileByName(modelName, modelPath
 		}
 	}
 
-	cfg.SetDefaults(opts...)
+	cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
 
 	return cfg, nil
 }

core/http/app_test.go

@@ -345,7 +345,7 @@ var _ = Describe("API test", func() {
 			It("Should fail if the api key is missing", func() {
 				err, sc := postInvalidRequest("http://127.0.0.1:9090/models/available")
 				Expect(err).ToNot(BeNil())
-				Expect(sc).To(Equal(403))
+				Expect(sc).To(Equal(401))
 			})
 		})
 
@@ -438,7 +438,7 @@ var _ = Describe("API test", func() {
 				Eventually(func() bool {
 					response := getModelStatus("http://127.0.0.1:9090/models/jobs/" + uuid)
 					return response["processed"].(bool)
-				}, "360s", "10s").Should(Equal(true))
+				}, "900s", "10s").Should(Equal(true))
 
 				Eventually(func() []string {
 					models, _ := client.ListModels(context.TODO())
@@ -562,7 +562,7 @@ var _ = Describe("API test", func() {
 				Eventually(func() bool {
 					response := getModelStatus("http://127.0.0.1:9090/models/jobs/" + uuid)
 					return response["processed"].(bool)
-				}, "360s", "10s").Should(Equal(true))
+				}, "900s", "10s").Should(Equal(true))
 
 				By("testing chat")
 				resp, err := client.CreateChatCompletion(context.TODO(), openai.ChatCompletionRequest{Model: modelName, Messages: []openai.ChatCompletionMessage{

core/http/elements/buttons.go

@@ -0,0 +1,97 @@
+package elements
+
+import (
+	"strings"
+
+	"github.com/chasefleming/elem-go"
+	"github.com/chasefleming/elem-go/attrs"
+	"github.com/mudler/LocalAI/core/gallery"
+)
+
+func installButton(galleryName string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-block rounded bg-primary px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/install/model/" + galleryName,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-download pr-2",
+			},
+		),
+		elem.Text("Install"),
+	)
+}
+
+func reInstallButton(galleryName string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-block rounded bg-primary ml-2 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-target":             "#action-div-" + dropBadChars(galleryName),
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/install/model/" + galleryName,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-arrow-rotate-right pr-2",
+			},
+		),
+		elem.Text("Reinstall"),
+	)
+}
+
+func infoButton(m *gallery.GalleryModel) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-left inline-block rounded bg-primary px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"data-modal-target":     modalName(m),
+			"data-modal-toggle":     modalName(m),
+		},
+		elem.P(
+			attrs.Props{
+				"class": "flex items-center",
+			},
+			elem.I(
+				attrs.Props{
+					"class": "fas fa-info-circle pr-2",
+				},
+			),
+			elem.Text("Info"),
+		),
+	)
+}
+
+func deleteButton(galleryID string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"hx-confirm":            "Are you sure you wish to delete the model?",
+			"class":                 "float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-target":             "#action-div-" + dropBadChars(galleryID),
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/delete/model/" + galleryID,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-cancel pr-2",
+			},
+		),
+		elem.Text("Delete"),
+	)
+}
+
+// Javascript/HTMX doesn't like weird IDs
+func dropBadChars(s string) string {
+	return strings.ReplaceAll(s, "@", "__")
+}

core/http/elements/gallery.go

@@ -2,13 +2,11 @@ package elements
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/chasefleming/elem-go"
 	"github.com/chasefleming/elem-go/attrs"
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/mudler/LocalAI/core/gallery"
-	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/services"
 )
 
@@ -16,231 +14,6 @@ const (
 	noImage = "https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"
 )
 
-func renderElements(n []elem.Node) string {
-	render := ""
-	for _, r := range n {
-		render += r.Render()
-	}
-	return render
-}
-
-func DoneProgress(galleryID, text string, showDelete bool) string {
-	var modelName = galleryID
-	// Split by @ and grab the name
-	if strings.Contains(galleryID, "@") {
-		modelName = strings.Split(galleryID, "@")[1]
-	}
-
-	return elem.Div(
-		attrs.Props{
-			"id": "action-div-" + dropBadChars(galleryID),
-		},
-		elem.H3(
-			attrs.Props{
-				"role":      "status",
-				"id":        "pblabel",
-				"tabindex":  "-1",
-				"autofocus": "",
-			},
-			elem.Text(bluemonday.StrictPolicy().Sanitize(text)),
-		),
-		elem.If(showDelete, deleteButton(galleryID, modelName), reInstallButton(galleryID)),
-	).Render()
-}
-
-func ErrorProgress(err, galleryName string) string {
-	return elem.Div(
-		attrs.Props{},
-		elem.H3(
-			attrs.Props{
-				"role":      "status",
-				"id":        "pblabel",
-				"tabindex":  "-1",
-				"autofocus": "",
-			},
-			elem.Text("Error "+bluemonday.StrictPolicy().Sanitize(err)),
-		),
-		installButton(galleryName),
-	).Render()
-}
-
-func ProgressBar(progress string) string {
-	return elem.Div(attrs.Props{
-		"class":           "progress",
-		"role":            "progressbar",
-		"aria-valuemin":   "0",
-		"aria-valuemax":   "100",
-		"aria-valuenow":   "0",
-		"aria-labelledby": "pblabel",
-	},
-		elem.Div(attrs.Props{
-			"id":    "pb",
-			"class": "progress-bar",
-			"style": "width:" + progress + "%",
-		}),
-	).Render()
-}
-
-func P2PNodeStats(nodes []p2p.NodeData) string {
-	/*
-	   <div class="bg-gray-800 p-6 rounded-lg shadow-lg text-left">
-	                       <p class="text-xl font-semibold text-gray-200">Total Workers Detected: {{ len .Nodes }}</p>
-	                       {{ $online := 0 }}
-	                       {{ range .Nodes }}
-	                           {{ if .IsOnline }}
-	                               {{ $online = add $online 1 }}
-	                           {{ end }}
-	                       {{ end }}
-	                       <p class="text-xl font-semibold text-gray-200">Total Online Workers: {{$online}}</p>
-	                   </div>
-	*/
-
-	online := 0
-	for _, n := range nodes {
-		if n.IsOnline() {
-			online++
-		}
-	}
-
-	class := "text-green-500"
-	if online == 0 {
-		class = "text-red-500"
-	}
-	/*
-	   <i class="fas fa-circle animate-pulse text-green-500 ml-2 mr-1"></i>
-	*/
-	circle := elem.I(attrs.Props{
-		"class": "fas fa-circle animate-pulse " + class + " ml-2 mr-1",
-	})
-	nodesElements := []elem.Node{
-		elem.Span(
-			attrs.Props{
-				"class": class,
-			},
-			circle,
-			elem.Text(fmt.Sprintf("%d", online)),
-		),
-		elem.Span(
-			attrs.Props{
-				"class": "text-gray-200",
-			},
-			elem.Text(fmt.Sprintf("/%d", len(nodes))),
-		),
-	}
-
-	return renderElements(nodesElements)
-}
-
-func P2PNodeBoxes(nodes []p2p.NodeData) string {
-	/*
-			<div class="bg-gray-800 p-4 rounded-lg shadow-lg text-left">
-			<div class="flex items-center mb-2">
-				<i class="fas fa-desktop text-gray-400 mr-2"></i>
-				<span class="text-gray-200 font-semibold">{{.ID}}</span>
-			</div>
-			<p class="text-sm text-gray-400 mt-2 flex items-center">
-				Status:
-				<i class="fas fa-circle {{ if .IsOnline }}text-green-500{{ else }}text-red-500{{ end }} ml-2 mr-1"></i>
-				<span class="{{ if .IsOnline }}text-green-400{{ else }}text-red-400{{ end }}">
-					{{ if .IsOnline }}Online{{ else }}Offline{{ end }}
-				</span>
-			</p>
-		</div>
-	*/
-
-	nodesElements := []elem.Node{}
-
-	for _, n := range nodes {
-
-		nodesElements = append(nodesElements,
-			elem.Div(
-				attrs.Props{
-					"class": "bg-gray-700 p-6 rounded-lg shadow-lg text-left",
-				},
-				elem.P(
-					attrs.Props{
-						"class": "text-sm text-gray-400 mt-2 flex",
-					},
-					elem.I(
-						attrs.Props{
-							"class": "fas fa-desktop text-gray-400 mr-2",
-						},
-					),
-					elem.Text("Name: "),
-					elem.Span(
-						attrs.Props{
-							"class": "text-gray-200 font-semibold ml-2 mr-1",
-						},
-						elem.Text(bluemonday.StrictPolicy().Sanitize(n.ID)),
-					),
-					elem.Text("Status: "),
-					elem.If(
-						n.IsOnline(),
-						elem.I(
-							attrs.Props{
-								"class": "fas fa-circle animate-pulse text-green-500 ml-2 mr-1",
-							},
-						),
-						elem.I(
-							attrs.Props{
-								"class": "fas fa-circle animate-pulse text-red-500 ml-2 mr-1",
-							},
-						),
-					),
-					elem.If(
-						n.IsOnline(),
-						elem.Span(
-							attrs.Props{
-								"class": "text-green-400",
-							},
-
-							elem.Text("Online"),
-						),
-						elem.Span(
-							attrs.Props{
-								"class": "text-red-400",
-							},
-							elem.Text("Offline"),
-						),
-					),
-				),
-			))
-	}
-
-	return renderElements(nodesElements)
-}
-
-func StartProgressBar(uid, progress, text string) string {
-	if progress == "" {
-		progress = "0"
-	}
-	return elem.Div(
-		attrs.Props{
-			"hx-trigger": "done",
-			"hx-get":     "/browse/job/" + uid,
-			"hx-swap":    "outerHTML",
-			"hx-target":  "this",
-		},
-		elem.H3(
-			attrs.Props{
-				"role":      "status",
-				"id":        "pblabel",
-				"tabindex":  "-1",
-				"autofocus": "",
-			},
-			elem.Text(bluemonday.StrictPolicy().Sanitize(text)), //Perhaps overly defensive
-			elem.Div(attrs.Props{
-				"hx-get":     "/browse/job/progress/" + uid,
-				"hx-trigger": "every 600ms",
-				"hx-target":  "this",
-				"hx-swap":    "innerHTML",
-			},
-				elem.Raw(ProgressBar(progress)),
-			),
-		),
-	).Render()
-}
-
 func cardSpan(text, icon string) elem.Node {
 	return elem.Span(
 		attrs.Props{
@@ -268,7 +41,6 @@ func searchableElement(text, icon string) elem.Node {
 			attrs.Props{
 				"class": "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2",
 			},
-
 			elem.A(
 				attrs.Props{
 					//	"name":      "search",
@@ -290,7 +62,8 @@ func searchableElement(text, icon string) elem.Node {
 	)
 }
 
-func link(text, url string) elem.Node {
+/*
+func buttonLink(text, url string) elem.Node {
 	return elem.A(
 		attrs.Props{
 			"class":  "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2",
@@ -303,163 +76,255 @@ func link(text, url string) elem.Node {
 		elem.Text(bluemonday.StrictPolicy().Sanitize(text)),
 	)
 }
-func installButton(galleryName string) elem.Node {
-	return elem.Button(
-		attrs.Props{
-			"data-twe-ripple-init":  "",
-			"data-twe-ripple-color": "light",
-			"class":                 "float-right inline-block rounded bg-primary px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
-			"hx-swap":               "outerHTML",
-			// post the Model ID as param
-			"hx-post": "/browse/install/model/" + galleryName,
-		},
-		elem.I(
-			attrs.Props{
-				"class": "fa-solid fa-download pr-2",
-			},
-		),
-		elem.Text("Install"),
-	)
-}
+*/
 
-func reInstallButton(galleryName string) elem.Node {
-	return elem.Button(
+func link(text, url string) elem.Node {
+	return elem.A(
 		attrs.Props{
-			"data-twe-ripple-init":  "",
-			"data-twe-ripple-color": "light",
-			"class":                 "float-right inline-block rounded bg-primary ml-2 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
-			"hx-target":             "#action-div-" + dropBadChars(galleryName),
-			"hx-swap":               "outerHTML",
-			// post the Model ID as param
-			"hx-post": "/browse/install/model/" + galleryName,
+			"class":  "text-base leading-relaxed text-gray-500 dark:text-gray-400",
+			"href":   url,
+			"target": "_blank",
 		},
-		elem.I(
-			attrs.Props{
-				"class": "fa-solid fa-arrow-rotate-right pr-2",
-			},
-		),
-		elem.Text("Reinstall"),
+		elem.I(attrs.Props{
+			"class": "fas fa-link pr-2",
+		}),
+		elem.Text(bluemonday.StrictPolicy().Sanitize(text)),
 	)
 }
 
-func deleteButton(galleryID, modelName string) elem.Node {
-	return elem.Button(
-		attrs.Props{
-			"data-twe-ripple-init":  "",
-			"data-twe-ripple-color": "light",
-			"hx-confirm":            "Are you sure you wish to delete the model?",
-			"class":                 "float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
-			"hx-target":             "#action-div-" + dropBadChars(galleryID),
-			"hx-swap":               "outerHTML",
-			// post the Model ID as param
-			"hx-post": "/browse/delete/model/" + galleryID,
-		},
-		elem.I(
-			attrs.Props{
-				"class": "fa-solid fa-cancel pr-2",
-			},
-		),
-		elem.Text("Delete"),
-	)
-}
-
-// Javascript/HTMX doesn't like weird IDs
-func dropBadChars(s string) string {
-	return strings.ReplaceAll(s, "@", "__")
-}
-
 type ProcessTracker interface {
 	Exists(string) bool
 	Get(string) string
 }
 
-func ListModels(models []*gallery.GalleryModel, processTracker ProcessTracker, galleryService *services.GalleryService) string {
-	modelsElements := []elem.Node{}
-	descriptionDiv := func(m *gallery.GalleryModel) elem.Node {
-		return elem.Div(
-			attrs.Props{
-				"class": "p-6 text-surface dark:text-white",
-			},
-			elem.H5(
-				attrs.Props{
-					"class": "mb-2 text-xl font-bold leading-tight",
-				},
-				elem.Text(bluemonday.StrictPolicy().Sanitize(m.Name)),
-			),
-			elem.P(
-				attrs.Props{
-					"class": "mb-4 text-sm [&:not(:hover)]:truncate text-base",
-				},
-				elem.Text(bluemonday.StrictPolicy().Sanitize(m.Description)),
-			),
+func modalName(m *gallery.GalleryModel) string {
+	return m.Name + "-modal"
+}
+
+func modelDescription(m *gallery.GalleryModel) elem.Node {
+	urls := []elem.Node{}
+	for _, url := range m.URLs {
+		urls = append(urls,
+			elem.Li(attrs.Props{}, link(url, url)),
 		)
 	}
 
-	actionDiv := func(m *gallery.GalleryModel) elem.Node {
-		galleryID := fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
-		currentlyProcessing := processTracker.Exists(galleryID)
-		jobID := ""
-		isDeletionOp := false
-		if currentlyProcessing {
-			status := galleryService.GetStatus(galleryID)
-			if status != nil && status.Deletion {
-				isDeletionOp = true
-			}
-			jobID = processTracker.Get(galleryID)
-			// TODO:
-			// case not handled, if status == nil : "Waiting"
-		}
-
-		nodes := []elem.Node{
-			cardSpan("Repository: "+m.Gallery.Name, "fa-brands fa-git-alt"),
-		}
-
-		if m.License != "" {
-			nodes = append(nodes,
-				cardSpan("License: "+m.License, "fas fa-book"),
-			)
-		}
-
-		tagsNodes := []elem.Node{}
-		for _, tag := range m.Tags {
-			tagsNodes = append(tagsNodes,
-				searchableElement(tag, "fas fa-tag"),
-			)
-		}
-
-		nodes = append(nodes,
-			elem.Div(
-				attrs.Props{
-					"class": "flex flex-row flex-wrap content-center",
-				},
-				tagsNodes...,
-			),
+	tagsNodes := []elem.Node{}
+	for _, tag := range m.Tags {
+		tagsNodes = append(tagsNodes,
+			searchableElement(tag, "fas fa-tag"),
 		)
+	}
 
-		for i, url := range m.URLs {
-			nodes = append(nodes,
-				link("Link #"+fmt.Sprintf("%d", i+1), url),
-			)
-		}
-
-		progressMessage := "Installation"
-		if isDeletionOp {
-			progressMessage = "Deletion"
-		}
-
-		return elem.Div(
+	return elem.Div(
+		attrs.Props{
+			"class": "p-6 text-surface dark:text-white",
+		},
+		elem.H5(
 			attrs.Props{
-				"class": "px-6 pt-4 pb-2",
+				"class": "mb-2 text-xl font-bold leading-tight",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(m.Name)),
+		),
+		elem.Div( // small description
+			attrs.Props{
+				"class": "mb-4 text-sm truncate text-base",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(m.Description)),
+		),
+
+		elem.Div(
+			attrs.Props{
+				"id":          modalName(m),
+				"tabindex":    "-1",
+				"aria-hidden": "true",
+				"class":       "hidden overflow-y-auto overflow-x-hidden fixed top-0 right-0 left-0 z-50 justify-center items-center w-full md:inset-0 h-[calc(100%-1rem)] max-h-full",
 			},
-			elem.P(
-				attrs.Props{
-					"class": "mb-4 text-base",
-				},
-				nodes...,
-			),
 			elem.Div(
 				attrs.Props{
-					"id": "action-div-" + dropBadChars(galleryID),
+					"class": "relative p-4 w-full max-w-2xl max-h-full",
+				},
+				elem.Div(
+					attrs.Props{
+						"class": "relative p-4 w-full max-w-2xl max-h-full bg-white rounded-lg shadow dark:bg-gray-700",
+					},
+					// header
+					elem.Div(
+						attrs.Props{
+							"class": "flex items-center justify-between p-4 md:p-5 border-b rounded-t dark:border-gray-600",
+						},
+						elem.H3(
+							attrs.Props{
+								"class": "text-xl font-semibold text-gray-900 dark:text-white",
+							},
+							elem.Text(bluemonday.StrictPolicy().Sanitize(m.Name)),
+						),
+						elem.Button( // close button
+							attrs.Props{
+								"class":           "text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm w-8 h-8 ms-auto inline-flex justify-center items-center dark:hover:bg-gray-600 dark:hover:text-white",
+								"data-modal-hide": modalName(m),
+							},
+							elem.Raw(
+								`<svg class="w-3 h-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 14">
+									<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="m1 1 6 6m0 0 6 6M7 7l6-6M7 7l-6 6"/>
+								</svg>`,
+							),
+							elem.Span(
+								attrs.Props{
+									"class": "sr-only",
+								},
+								elem.Text("Close modal"),
+							),
+						),
+					),
+					// body
+					elem.Div(
+						attrs.Props{
+							"class": "p-4 md:p-5 space-y-4",
+						},
+						elem.Div(
+							attrs.Props{
+								"class": "flex justify-center items-center",
+							},
+							elem.Img(attrs.Props{
+								//	"class": "rounded-t-lg object-fit object-center h-96",
+								"class":   "lazy rounded-t-lg max-h-48 max-w-96 object-cover mt-3 entered loaded",
+								"src":     m.Icon,
+								"loading": "lazy",
+							}),
+						),
+						elem.P(
+							attrs.Props{
+								"class": "text-base leading-relaxed text-gray-500 dark:text-gray-400",
+							},
+							elem.Text(bluemonday.StrictPolicy().Sanitize(m.Description)),
+						),
+						elem.Hr(
+							attrs.Props{},
+						),
+						elem.P(
+							attrs.Props{
+								"class": "text-sm font-semibold text-gray-900 dark:text-white",
+							},
+							elem.Text("Links"),
+						),
+						elem.Ul(
+							attrs.Props{},
+							urls...,
+						),
+						elem.If(
+							len(m.Tags) > 0,
+							elem.Div(
+								attrs.Props{},
+								elem.P(
+									attrs.Props{
+										"class": "text-sm mb-5 font-semibold text-gray-900 dark:text-white",
+									},
+									elem.Text("Tags"),
+								),
+								elem.Div(
+									attrs.Props{
+										"class": "flex flex-row flex-wrap content-center",
+									},
+									tagsNodes...,
+								),
+							),
+							elem.Div(attrs.Props{}),
+						),
+					),
+					// Footer
+					elem.Div(
+						attrs.Props{
+							"class": "flex items-center p-4 md:p-5 border-t border-gray-200 rounded-b dark:border-gray-600",
+						},
+						elem.Button(
+							attrs.Props{
+								"data-modal-hide": modalName(m),
+								"class":           "py-2.5 px-5 ms-3 text-sm font-medium text-gray-900 focus:outline-none bg-white rounded-lg border border-gray-200 hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-4 focus:ring-gray-100 dark:focus:ring-gray-700 dark:bg-gray-800 dark:text-gray-400 dark:border-gray-600 dark:hover:text-white dark:hover:bg-gray-700",
+							},
+							elem.Text("Close"),
+						),
+					),
+				),
+			),
+		),
+	)
+}
+
+func modelActionItems(m *gallery.GalleryModel, processTracker ProcessTracker, galleryService *services.GalleryService) elem.Node {
+	galleryID := fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
+	currentlyProcessing := processTracker.Exists(galleryID)
+	jobID := ""
+	isDeletionOp := false
+	if currentlyProcessing {
+		status := galleryService.GetStatus(galleryID)
+		if status != nil && status.Deletion {
+			isDeletionOp = true
+		}
+		jobID = processTracker.Get(galleryID)
+		// TODO:
+		// case not handled, if status == nil : "Waiting"
+	}
+
+	nodes := []elem.Node{
+		cardSpan("Repository: "+m.Gallery.Name, "fa-brands fa-git-alt"),
+	}
+
+	if m.License != "" {
+		nodes = append(nodes,
+			cardSpan("License: "+m.License, "fas fa-book"),
+		)
+	}
+	/*
+		tagsNodes := []elem.Node{}
+
+			for _, tag := range m.Tags {
+				tagsNodes = append(tagsNodes,
+					searchableElement(tag, "fas fa-tag"),
+				)
+			}
+
+
+				nodes = append(nodes,
+					elem.Div(
+						attrs.Props{
+							"class": "flex flex-row flex-wrap content-center",
+						},
+						tagsNodes...,
+					),
+				)
+
+				for i, url := range m.URLs {
+					nodes = append(nodes,
+						buttonLink("Link #"+fmt.Sprintf("%d", i+1), url),
+					)
+				}
+	*/
+
+	progressMessage := "Installation"
+	if isDeletionOp {
+		progressMessage = "Deletion"
+	}
+
+	return elem.Div(
+		attrs.Props{
+			"class": "px-6 pt-4 pb-2",
+		},
+		elem.P(
+			attrs.Props{
+				"class": "mb-4 text-base",
+			},
+			nodes...,
+		),
+		elem.Div(
+			attrs.Props{
+				"id":    "action-div-" + dropBadChars(galleryID),
+				"class": "flow-root", // To order buttons left and right
+			},
+			infoButton(m),
+			elem.Div(
+				attrs.Props{
+					"class": "float-right",
 				},
 				elem.If(
 					currentlyProcessing,
@@ -470,14 +335,18 @@ func ListModels(models []*gallery.GalleryModel, processTracker ProcessTracker, g
 						elem.Node(elem.Div(
 							attrs.Props{},
 							reInstallButton(m.ID()),
-							deleteButton(m.ID(), m.Name),
+							deleteButton(m.ID()),
 						)),
 						installButton(m.ID()),
 					),
 				),
 			),
-		)
-	}
+		),
+	)
+}
+
+func ListModels(models []*gallery.GalleryModel, processTracker ProcessTracker, galleryService *services.GalleryService) string {
+	modelsElements := []elem.Node{}
 
 	for _, m := range models {
 		elems := []elem.Node{}
@@ -521,7 +390,10 @@ func ListModels(models []*gallery.GalleryModel, processTracker ProcessTracker, g
 			))
 		}
 
-		elems = append(elems, descriptionDiv(m), actionDiv(m))
+		elems = append(elems,
+			modelDescription(m),
+			modelActionItems(m, processTracker, galleryService),
+		)
 		modelsElements = append(modelsElements,
 			elem.Div(
 				attrs.Props{

core/http/elements/p2p.go

@@ -0,0 +1,147 @@
+package elements
+
+import (
+	"fmt"
+
+	"github.com/chasefleming/elem-go"
+	"github.com/chasefleming/elem-go/attrs"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/mudler/LocalAI/core/p2p"
+)
+
+func renderElements(n []elem.Node) string {
+	render := ""
+	for _, r := range n {
+		render += r.Render()
+	}
+	return render
+}
+
+func P2PNodeStats(nodes []p2p.NodeData) string {
+	/*
+	   <div class="bg-gray-800 p-6 rounded-lg shadow-lg text-left">
+	                       <p class="text-xl font-semibold text-gray-200">Total Workers Detected: {{ len .Nodes }}</p>
+	                       {{ $online := 0 }}
+	                       {{ range .Nodes }}
+	                           {{ if .IsOnline }}
+	                               {{ $online = add $online 1 }}
+	                           {{ end }}
+	                       {{ end }}
+	                       <p class="text-xl font-semibold text-gray-200">Total Online Workers: {{$online}}</p>
+	                   </div>
+	*/
+
+	online := 0
+	for _, n := range nodes {
+		if n.IsOnline() {
+			online++
+		}
+	}
+
+	class := "text-green-500"
+	if online == 0 {
+		class = "text-red-500"
+	}
+	/*
+	   <i class="fas fa-circle animate-pulse text-green-500 ml-2 mr-1"></i>
+	*/
+	circle := elem.I(attrs.Props{
+		"class": "fas fa-circle animate-pulse " + class + " ml-2 mr-1",
+	})
+	nodesElements := []elem.Node{
+		elem.Span(
+			attrs.Props{
+				"class": class,
+			},
+			circle,
+			elem.Text(fmt.Sprintf("%d", online)),
+		),
+		elem.Span(
+			attrs.Props{
+				"class": "text-gray-200",
+			},
+			elem.Text(fmt.Sprintf("/%d", len(nodes))),
+		),
+	}
+
+	return renderElements(nodesElements)
+}
+
+func P2PNodeBoxes(nodes []p2p.NodeData) string {
+	/*
+			<div class="bg-gray-800 p-4 rounded-lg shadow-lg text-left">
+			<div class="flex items-center mb-2">
+				<i class="fas fa-desktop text-gray-400 mr-2"></i>
+				<span class="text-gray-200 font-semibold">{{.ID}}</span>
+			</div>
+			<p class="text-sm text-gray-400 mt-2 flex items-center">
+				Status:
+				<i class="fas fa-circle {{ if .IsOnline }}text-green-500{{ else }}text-red-500{{ end }} ml-2 mr-1"></i>
+				<span class="{{ if .IsOnline }}text-green-400{{ else }}text-red-400{{ end }}">
+					{{ if .IsOnline }}Online{{ else }}Offline{{ end }}
+				</span>
+			</p>
+		</div>
+	*/
+
+	nodesElements := []elem.Node{}
+
+	for _, n := range nodes {
+
+		nodesElements = append(nodesElements,
+			elem.Div(
+				attrs.Props{
+					"class": "bg-gray-700 p-6 rounded-lg shadow-lg text-left",
+				},
+				elem.P(
+					attrs.Props{
+						"class": "text-sm text-gray-400 mt-2 flex",
+					},
+					elem.I(
+						attrs.Props{
+							"class": "fas fa-desktop text-gray-400 mr-2",
+						},
+					),
+					elem.Text("Name: "),
+					elem.Span(
+						attrs.Props{
+							"class": "text-gray-200 font-semibold ml-2 mr-1",
+						},
+						elem.Text(bluemonday.StrictPolicy().Sanitize(n.ID)),
+					),
+					elem.Text("Status: "),
+					elem.If(
+						n.IsOnline(),
+						elem.I(
+							attrs.Props{
+								"class": "fas fa-circle animate-pulse text-green-500 ml-2 mr-1",
+							},
+						),
+						elem.I(
+							attrs.Props{
+								"class": "fas fa-circle animate-pulse text-red-500 ml-2 mr-1",
+							},
+						),
+					),
+					elem.If(
+						n.IsOnline(),
+						elem.Span(
+							attrs.Props{
+								"class": "text-green-400",
+							},
+
+							elem.Text("Online"),
+						),
+						elem.Span(
+							attrs.Props{
+								"class": "text-red-400",
+							},
+							elem.Text("Offline"),
+						),
+					),
+				),
+			))
+	}
+
+	return renderElements(nodesElements)
+}

core/http/elements/progressbar.go

@@ -0,0 +1,89 @@
+package elements
+
+import (
+	"github.com/chasefleming/elem-go"
+	"github.com/chasefleming/elem-go/attrs"
+	"github.com/microcosm-cc/bluemonday"
+)
+
+func DoneProgress(galleryID, text string, showDelete bool) string {
+	return elem.Div(
+		attrs.Props{
+			"id": "action-div-" + dropBadChars(galleryID),
+		},
+		elem.H3(
+			attrs.Props{
+				"role":      "status",
+				"id":        "pblabel",
+				"tabindex":  "-1",
+				"autofocus": "",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(text)),
+		),
+		elem.If(showDelete, deleteButton(galleryID), reInstallButton(galleryID)),
+	).Render()
+}
+
+func ErrorProgress(err, galleryName string) string {
+	return elem.Div(
+		attrs.Props{},
+		elem.H3(
+			attrs.Props{
+				"role":      "status",
+				"id":        "pblabel",
+				"tabindex":  "-1",
+				"autofocus": "",
+			},
+			elem.Text("Error "+bluemonday.StrictPolicy().Sanitize(err)),
+		),
+		installButton(galleryName),
+	).Render()
+}
+
+func ProgressBar(progress string) string {
+	return elem.Div(attrs.Props{
+		"class":           "progress",
+		"role":            "progressbar",
+		"aria-valuemin":   "0",
+		"aria-valuemax":   "100",
+		"aria-valuenow":   "0",
+		"aria-labelledby": "pblabel",
+	},
+		elem.Div(attrs.Props{
+			"id":    "pb",
+			"class": "progress-bar",
+			"style": "width:" + progress + "%",
+		}),
+	).Render()
+}
+
+func StartProgressBar(uid, progress, text string) string {
+	if progress == "" {
+		progress = "0"
+	}
+	return elem.Div(
+		attrs.Props{
+			"hx-trigger": "done",
+			"hx-get":     "/browse/job/" + uid,
+			"hx-swap":    "outerHTML",
+			"hx-target":  "this",
+		},
+		elem.H3(
+			attrs.Props{
+				"role":      "status",
+				"id":        "pblabel",
+				"tabindex":  "-1",
+				"autofocus": "",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(text)), //Perhaps overly defensive
+			elem.Div(attrs.Props{
+				"hx-get":     "/browse/job/progress/" + uid,
+				"hx-trigger": "every 600ms",
+				"hx-target":  "this",
+				"hx-swap":    "innerHTML",
+			},
+				elem.Raw(ProgressBar(progress)),
+			),
+		),
+	).Render()
+}

core/http/endpoints/localai/system.go

@@ -21,10 +21,15 @@ func SystemInformations(ml *model.ModelLoader, appConfig *config.ApplicationConf
 		for b := range appConfig.ExternalGRPCBackends {
 			availableBackends = append(availableBackends, b)
 		}
+
+		sysmodels := []schema.SysInfoModel{}
+		for _, m := range loadedModels {
+			sysmodels = append(sysmodels, schema.SysInfoModel{ID: m.ID})
+		}
 		return c.JSON(
 			schema.SystemInformationResponse{
 				Backends: availableBackends,
-				Models:   loadedModels,
+				Models:   sysmodels,
 			},
 		)
 	}

core/http/endpoints/localai/tts.go

@@ -9,16 +9,19 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/rs/zerolog/log"
+
+	"github.com/mudler/LocalAI/pkg/utils"
 )
 
 // TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
-//	@Summary	Generates audio from the input text.
-//  @Accept json
-//  @Produce audio/x-wav
-//	@Param		request	body		schema.TTSRequest	true	"query params"
-//	@Success	200		{string}	binary				"generated audio/wav file"
-//	@Router		/v1/audio/speech [post]
-//	@Router		/tts [post]
+//
+//		@Summary	Generates audio from the input text.
+//	 	@Accept json
+//	 	@Produce audio/x-wav
+//		@Param		request	body		schema.TTSRequest	true	"query params"
+//		@Success	200		{string}	binary				"generated audio/wav file"
+//		@Router		/v1/audio/speech [post]
+//		@Router		/tts [post]
 func TTSEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 
@@ -67,6 +70,13 @@ func TTSEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfi
 		if err != nil {
 			return err
 		}
+
+		// Convert generated file to target format
+		filePath, err = utils.AudioConvert(filePath, input.Format)
+		if err != nil {
+			return err
+		}
+
 		return c.Download(filePath)
 	}
 }

core/http/endpoints/localai/vad.go

@@ -0,0 +1,68 @@
+package localai
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/mudler/LocalAI/core/backend"
+	"github.com/mudler/LocalAI/core/config"
+	fiberContext "github.com/mudler/LocalAI/core/http/ctx"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/rs/zerolog/log"
+)
+
+// VADEndpoint is Voice-Activation-Detection endpoint
+// @Summary	Detect voice fragments in an audio stream
+// @Accept json
+// @Param		request	body		schema.VADRequest	true	"query params"
+// @Success 200 {object} proto.VADResponse "Response"
+// @Router		/vad [post]
+func VADEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+
+		input := new(schema.VADRequest)
+
+		// Get input data from the request body
+		if err := c.BodyParser(input); err != nil {
+			return err
+		}
+
+		modelFile, err := fiberContext.ModelFromContext(c, cl, ml, input.Model, false)
+		if err != nil {
+			modelFile = input.Model
+			log.Warn().Msgf("Model not found in context: %s", input.Model)
+		}
+
+		cfg, err := cl.LoadBackendConfigFileByName(modelFile, appConfig.ModelPath,
+			config.LoadOptionDebug(appConfig.Debug),
+			config.LoadOptionThreads(appConfig.Threads),
+			config.LoadOptionContextSize(appConfig.ContextSize),
+			config.LoadOptionF16(appConfig.F16),
+		)
+
+		if err != nil {
+			log.Err(err)
+			modelFile = input.Model
+			log.Warn().Msgf("Model not found in context: %s", input.Model)
+		} else {
+			modelFile = cfg.Model
+		}
+		log.Debug().Msgf("Request for model: %s", modelFile)
+
+		opts := backend.ModelOptions(*cfg, appConfig, model.WithBackendString(cfg.Backend), model.WithModel(modelFile))
+
+		vadModel, err := ml.Load(opts...)
+		if err != nil {
+			return err
+		}
+		req := proto.VADRequest{
+			Audio: input.Audio,
+		}
+		resp, err := vadModel.VAD(c.Context(), &req)
+		if err != nil {
+			return err
+		}
+
+		return c.JSON(resp)
+	}
+}

core/http/endpoints/openai/image.go

@@ -136,6 +136,11 @@ func ImageEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appCon
 			config.Backend = model.StableDiffusionBackend
 		}
 
+		if !strings.Contains(input.Size, "x") {
+			input.Size = "512x512"
+			log.Warn().Msgf("Invalid size, using default 512x512")
+		}
+
 		sizeParts := strings.Split(input.Size, "x")
 		if len(sizeParts) != 2 {
 			return fmt.Errorf("invalid value for 'size'")

core/http/endpoints/openai/request.go

@@ -304,7 +304,6 @@ func mergeRequestWithConfig(modelFile string, input *schema.OpenAIRequest, cm *c
 		config.LoadOptionThreads(threads),
 		config.LoadOptionContextSize(ctx),
 		config.LoadOptionF16(f16),
-		config.ModelPath(loader.ModelPath),
 	)
 
 	// Set the parameters for the language model prediction

core/http/middleware/auth.go

@@ -1,95 +1,95 @@
-package middleware
-
-import (
-	"crypto/subtle"
-	"errors"
-
-	"github.com/dave-gray101/v2keyauth"
-	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/fiber/v2/middleware/keyauth"
-	"github.com/microcosm-cc/bluemonday"
-	"github.com/mudler/LocalAI/core/config"
-)
-
-// This file contains the configuration generators and handler functions that are used along with the fiber/keyauth middleware
-// Currently this requires an upstream patch - and feature patches are no longer accepted to v2
-// Therefore `dave-gray101/v2keyauth` contains the v2 backport of the middleware until v3 stabilizes and we migrate.
-
-func GetKeyAuthConfig(applicationConfig *config.ApplicationConfig) (*v2keyauth.Config, error) {
-	customLookup, err := v2keyauth.MultipleKeySourceLookup([]string{"header:Authorization", "header:x-api-key", "header:xi-api-key"}, keyauth.ConfigDefault.AuthScheme)
-	if err != nil {
-		return nil, err
-	}
-
-	return &v2keyauth.Config{
-		CustomKeyLookup: customLookup,
-		Next:            getApiKeyRequiredFilterFunction(applicationConfig),
-		Validator:       getApiKeyValidationFunction(applicationConfig),
-		ErrorHandler:    getApiKeyErrorHandler(applicationConfig),
-		AuthScheme:      "Bearer",
-	}, nil
-}
-
-func getApiKeyErrorHandler(applicationConfig *config.ApplicationConfig) fiber.ErrorHandler {
-	return func(ctx *fiber.Ctx, err error) error {
-		if errors.Is(err, v2keyauth.ErrMissingOrMalformedAPIKey) {
-			if len(applicationConfig.ApiKeys) == 0 {
-				return ctx.Next() // if no keys are set up, any error we get here is not an error.
-			}
-			if applicationConfig.OpaqueErrors {
-				return ctx.SendStatus(403)
-			}
-			return ctx.Status(403).SendString(bluemonday.StrictPolicy().Sanitize(err.Error()))
-		}
-		if applicationConfig.OpaqueErrors {
-			return ctx.SendStatus(500)
-		}
-		return err
-	}
-}
-
-func getApiKeyValidationFunction(applicationConfig *config.ApplicationConfig) func(*fiber.Ctx, string) (bool, error) {
-
-	if applicationConfig.UseSubtleKeyComparison {
-		return func(ctx *fiber.Ctx, apiKey string) (bool, error) {
-			if len(applicationConfig.ApiKeys) == 0 {
-				return true, nil // If no keys are setup, accept everything
-			}
-			for _, validKey := range applicationConfig.ApiKeys {
-				if subtle.ConstantTimeCompare([]byte(apiKey), []byte(validKey)) == 1 {
-					return true, nil
-				}
-			}
-			return false, v2keyauth.ErrMissingOrMalformedAPIKey
-		}
-	}
-
-	return func(ctx *fiber.Ctx, apiKey string) (bool, error) {
-		if len(applicationConfig.ApiKeys) == 0 {
-			return true, nil // If no keys are setup, accept everything
-		}
-		for _, validKey := range applicationConfig.ApiKeys {
-			if apiKey == validKey {
-				return true, nil
-			}
-		}
-		return false, v2keyauth.ErrMissingOrMalformedAPIKey
-	}
-}
-
-func getApiKeyRequiredFilterFunction(applicationConfig *config.ApplicationConfig) func(*fiber.Ctx) bool {
-	if applicationConfig.DisableApiKeyRequirementForHttpGet {
-		return func(c *fiber.Ctx) bool {
-			if c.Method() != "GET" {
-				return false
-			}
-			for _, rx := range applicationConfig.HttpGetExemptedEndpoints {
-				if rx.MatchString(c.Path()) {
-					return true
-				}
-			}
-			return false
-		}
-	}
-	return func(c *fiber.Ctx) bool { return false }
-}
+package middleware
+
+import (
+	"crypto/subtle"
+	"errors"
+
+	"github.com/dave-gray101/v2keyauth"
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/keyauth"
+	"github.com/mudler/LocalAI/core/config"
+)
+
+// This file contains the configuration generators and handler functions that are used along with the fiber/keyauth middleware
+// Currently this requires an upstream patch - and feature patches are no longer accepted to v2
+// Therefore `dave-gray101/v2keyauth` contains the v2 backport of the middleware until v3 stabilizes and we migrate.
+
+func GetKeyAuthConfig(applicationConfig *config.ApplicationConfig) (*v2keyauth.Config, error) {
+	customLookup, err := v2keyauth.MultipleKeySourceLookup([]string{"header:Authorization", "header:x-api-key", "header:xi-api-key", "cookie:token"}, keyauth.ConfigDefault.AuthScheme)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v2keyauth.Config{
+		CustomKeyLookup: customLookup,
+		Next:            getApiKeyRequiredFilterFunction(applicationConfig),
+		Validator:       getApiKeyValidationFunction(applicationConfig),
+		ErrorHandler:    getApiKeyErrorHandler(applicationConfig),
+		AuthScheme:      "Bearer",
+	}, nil
+}
+
+func getApiKeyErrorHandler(applicationConfig *config.ApplicationConfig) fiber.ErrorHandler {
+	return func(ctx *fiber.Ctx, err error) error {
+		if errors.Is(err, v2keyauth.ErrMissingOrMalformedAPIKey) {
+			if len(applicationConfig.ApiKeys) == 0 {
+				return ctx.Next() // if no keys are set up, any error we get here is not an error.
+			}
+			ctx.Set("WWW-Authenticate", "Bearer")
+			if applicationConfig.OpaqueErrors {
+				return ctx.SendStatus(401)
+			}
+			return ctx.Status(401).Render("views/login", nil)
+		}
+		if applicationConfig.OpaqueErrors {
+			return ctx.SendStatus(500)
+		}
+		return err
+	}
+}
+
+func getApiKeyValidationFunction(applicationConfig *config.ApplicationConfig) func(*fiber.Ctx, string) (bool, error) {
+
+	if applicationConfig.UseSubtleKeyComparison {
+		return func(ctx *fiber.Ctx, apiKey string) (bool, error) {
+			if len(applicationConfig.ApiKeys) == 0 {
+				return true, nil // If no keys are setup, accept everything
+			}
+			for _, validKey := range applicationConfig.ApiKeys {
+				if subtle.ConstantTimeCompare([]byte(apiKey), []byte(validKey)) == 1 {
+					return true, nil
+				}
+			}
+			return false, v2keyauth.ErrMissingOrMalformedAPIKey
+		}
+	}
+
+	return func(ctx *fiber.Ctx, apiKey string) (bool, error) {
+		if len(applicationConfig.ApiKeys) == 0 {
+			return true, nil // If no keys are setup, accept everything
+		}
+		for _, validKey := range applicationConfig.ApiKeys {
+			if apiKey == validKey {
+				return true, nil
+			}
+		}
+		return false, v2keyauth.ErrMissingOrMalformedAPIKey
+	}
+}
+
+func getApiKeyRequiredFilterFunction(applicationConfig *config.ApplicationConfig) func(*fiber.Ctx) bool {
+	if applicationConfig.DisableApiKeyRequirementForHttpGet {
+		return func(c *fiber.Ctx) bool {
+			if c.Method() != "GET" {
+				return false
+			}
+			for _, rx := range applicationConfig.HttpGetExemptedEndpoints {
+				if rx.MatchString(c.Path()) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	return func(c *fiber.Ctx) bool { return false }
+}

core/http/routes/localai.go

@@ -34,6 +34,7 @@ func RegisterLocalAIRoutes(app *fiber.App,
 	}
 
 	app.Post("/tts", localai.TTSEndpoint(cl, ml, appConfig))
+	app.Post("/vad", localai.VADEndpoint(cl, ml, appConfig))
 
 	// Stores
 	sl := model.NewModelLoader("")

core/http/views/login.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Open Authenticated Website</title>
+</head>
+<body>
+    <h1>Authorization is required</h1>
+    <input type="text" id="token" placeholder="Token" />
+    <button onclick="login()">Login</button>
+    <script>
+        function login() {
+            const token = document.getElementById('token').value;
+            var date = new Date();
+            date.setTime(date.getTime() + (24*60*60*1000));
+            document.cookie = `token=${token}; expires=${date.toGMTString()}`;
+
+            window.location.reload();
+        }
+    </script>
+</body>
+</html>

core/http/views/partials/footer.html

@@ -1,5 +1,5 @@
 <footer class="text-center py-8">
     LocalAI Version {{.Version}}<br>
-    <a href='https://localai.io' class="text-blue-400 hover:text-blue-600" target="_blank">LocalAI</a> © 2023-2024 <a href='https://mudler.pm' class="text-blue-400 hover:text-blue-600" target="_blank">Ettore Di Giacinto</a>
+    <a href='https://github.com/mudler/LocalAI' class="text-blue-400 hover:text-blue-600" target="_blank">LocalAI</a> © 2023-2024 <a href='https://mudler.pm' class="text-blue-400 hover:text-blue-600" target="_blank">Ettore Di Giacinto</a>
 </footer>
 <script src="/static/assets/tw-elements.js"></script>

core/http/views/partials/head.html

@@ -57,6 +57,7 @@
   <link href="/static/assets/fontawesome/css/fontawesome.css" rel="stylesheet" />
   <link href="/static/assets/fontawesome/css/brands.css" rel="stylesheet" />
   <link href="/static/assets/fontawesome/css/solid.css" rel="stylesheet" />
+  <script src="/static/assets/flowbite.min.js"></script>
   <script src="/static/assets/htmx.js" crossorigin="anonymous"></script>
   <!-- P2P Animation START -->
   <style>
@@ -118,4 +119,11 @@
             100% { transform: rotate(0deg); } /* Return to center */
         }
   </style>
+
+  <!-- https://stackoverflow.com/questions/76051980/flowbite-component-not-working-when-loaded-via-htmx-django-project -->
+  <script>
+      htmx.onLoad(function(content) {
+          initFlowbite();
+      })
+  </script>
 </head>

core/p2p/p2p.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"net"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -22,6 +23,7 @@ import (
 	"github.com/mudler/edgevpn/pkg/services"
 	"github.com/mudler/edgevpn/pkg/types"
 	eutils "github.com/mudler/edgevpn/pkg/utils"
+	"github.com/multiformats/go-multiaddr"
 	"github.com/phayes/freeport"
 	zlog "github.com/rs/zerolog/log"
 
@@ -231,10 +233,14 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
 
 				data := ledger.LastBlock().Storage[servicesID]
 
-				zlog.Debug().Any("data", ledger.LastBlock().Storage).Msg("Ledger data")
+				if logLevel == logLevelDebug {
+					// We want to surface this debugging data only if p2p logging is set to debug
+					// (and not generally the whole application, as this can be really noisy)
+					zlog.Debug().Any("data", ledger.LastBlock().Storage).Msg("Ledger data")
+				}
 
 				for k, v := range data {
-					zlog.Debug().Msgf("New worker found in the ledger data '%s'", k)
+					// New worker found in the ledger data as k (worker id)
 					nd := &NodeData{}
 					if err := v.Unmarshal(nd); err != nil {
 						zlog.Error().Msg("cannot unmarshal node data")
@@ -269,7 +275,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *NodeData, sserv string
 	if ndService, found := service[nd.Name]; !found {
 		if !nd.IsOnline() {
 			// if node is offline and not present, do nothing
-			zlog.Debug().Msgf("Node %s is offline", nd.ID)
+			// Node nd.ID is offline
 			return
 		}
 
@@ -381,22 +387,35 @@ func newNodeOpts(token string) ([]node.Option, error) {
 	noDHT := os.Getenv("LOCALAI_P2P_DISABLE_DHT") == "true"
 	noLimits := os.Getenv("LOCALAI_P2P_ENABLE_LIMITS") == "true"
 
-	loglevel := os.Getenv("LOCALAI_P2P_LOGLEVEL")
-	if loglevel == "" {
-		loglevel = "info"
+	var listenMaddrs []string
+	var bootstrapPeers []string
+
+	laddrs := os.Getenv("LOCALAI_P2P_LISTEN_MADDRS")
+	if laddrs != "" {
+		listenMaddrs = strings.Split(laddrs, ",")
 	}
-	libp2ploglevel := os.Getenv("LOCALAI_LIBP2P_LOGLEVEL")
+
+	bootmaddr := os.Getenv("LOCALAI_P2P_BOOTSTRAP_PEERS_MADDRS")
+	if bootmaddr != "" {
+		bootstrapPeers = strings.Split(bootmaddr, ",")
+	}
+
+	dhtAnnounceMaddrs := stringsToMultiAddr(strings.Split(os.Getenv("LOCALAI_P2P_DHT_ANNOUNCE_MADDRS"), ","))
+
+	libp2ploglevel := os.Getenv("LOCALAI_P2P_LIB_LOGLEVEL")
 	if libp2ploglevel == "" {
 		libp2ploglevel = "fatal"
 	}
 	c := config.Config{
+		ListenMaddrs:      listenMaddrs,
+		DHTAnnounceMaddrs: dhtAnnounceMaddrs,
 		Limit: config.ResourceLimit{
 			Enable:   noLimits,
 			MaxConns: 100,
 		},
 		NetworkToken:   token,
 		LowProfile:     false,
-		LogLevel:       loglevel,
+		LogLevel:       logLevel,
 		Libp2pLogLevel: libp2ploglevel,
 		Ledger: config.Ledger{
 			SyncInterval:     defaultInterval,
@@ -411,9 +430,10 @@ func newNodeOpts(token string) ([]node.Option, error) {
 			RateLimitInterval: defaultInterval,
 		},
 		Discovery: config.Discovery{
-			DHT:      !noDHT,
-			MDNS:     true,
-			Interval: 10 * time.Second,
+			DHT:            !noDHT,
+			MDNS:           true,
+			Interval:       10 * time.Second,
+			BootstrapPeers: bootstrapPeers,
 		},
 		Connection: config.Connection{
 			HolePunch:      true,
@@ -432,6 +452,18 @@ func newNodeOpts(token string) ([]node.Option, error) {
 	return nodeOpts, nil
 }
 
+func stringsToMultiAddr(peers []string) []multiaddr.Multiaddr {
+	res := []multiaddr.Multiaddr{}
+	for _, p := range peers {
+		addr, err := multiaddr.NewMultiaddr(p)
+		if err != nil {
+			continue
+		}
+		res = append(res, addr)
+	}
+	return res
+}
+
 func copyStream(closer chan struct{}, dst io.Writer, src io.Reader) {
 	defer func() { closer <- struct{}{} }() // connection is closed, send signal to stop proxy
 	io.Copy(dst, src)