fix(autogptq): do not use_triton with qwen-vl (#1985)

* Enhance autogptq backend to support VL models

* update dependencies for autogptq

* remove redundant auto-gptq dependency

* Convert base64 to image_url for Qwen-VL model

* implemented model inference for qwen-vl

* remove user prompt from generated answer

* fixed write image error

* fixed use_triton issue when loading Qwen-VL model

---------

Co-authored-by: Binghua Wu <bingwu@estee.com>

.github/labeler.yml

@@ -0,0 +1,19 @@
+enhancements:
+ - head-branch: ['^feature', 'feature']
+
+kind/documentation:
+- any:
+  - changed-files:
+    - any-glob-to-any-file: 'docs/*'
+  - changed-files:
+    - any-glob-to-any-file: '*.md'
+
+examples:
+- any:
+  - changed-files:
+    - any-glob-to-any-file: 'examples/*'
+
+ci:
+- any:
+  - changed-files:
+    - any-glob-to-any-file: '.github/*'

.github/workflows/image-pr.yml

@@ -22,7 +22,7 @@ jobs:
       platforms: ${{ matrix.platforms }}
       runs-on: ${{ matrix.runs-on }}
       base-image: ${{ matrix.base-image }}
-      makeflags: "-j3"
+      makeflags: ${{ matrix.makeflags }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -42,6 +42,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -52,6 +53,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -60,6 +62,7 @@ jobs:
             image-type: 'extras'
             base-image: "rocm/dev-ubuntu-22.04:6.0-complete"
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f16'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -68,6 +71,7 @@ jobs:
             ffmpeg: 'true'
             image-type: 'extras'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
   core-image-build:
     uses: ./.github/workflows/image_build.yml
     with:
@@ -81,7 +85,7 @@ jobs:
       platforms: ${{ matrix.platforms }}
       runs-on: ${{ matrix.runs-on }}
       base-image: ${{ matrix.base-image }}
-      makeflags: "-j3"
+      makeflags: ${{ matrix.makeflags }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -98,6 +102,7 @@ jobs:
             image-type: 'core'
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=5 --output-sync=target"
           - build-type: 'sycl_f16'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -106,6 +111,7 @@ jobs:
             ffmpeg: 'true'
             image-type: 'core'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -115,4 +121,5 @@ jobs:
             ffmpeg: 'true'
             image-type: 'core'
             runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:22.04"
+            base-image: "ubuntu:22.04"
+            makeflags: "--jobs=5 --output-sync=target"

.github/workflows/image.yml

@@ -27,7 +27,9 @@ jobs:
       runs-on: ${{ matrix.runs-on }}
       base-image: ${{ matrix.base-image }}
       aio: ${{ matrix.aio }}
-      makeflags: "-j3"
+      makeflags: ${{ matrix.makeflags }}
+      latest-image: ${{ matrix.latest-image }}
+      latest-image-aio: ${{ matrix.latest-image-aio }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -49,6 +51,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: ''
             platforms: 'linux/amd64'
             tag-latest: 'auto'
@@ -57,6 +60,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "11"
             cuda-minor-version: "7"
@@ -67,6 +71,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -77,6 +82,7 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "11"
             cuda-minor-version: "7"
@@ -88,6 +94,9 @@ jobs:
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
             aio: "-aio-gpu-nvidia-cuda-11"
+            latest-image: 'latest-gpu-nvidia-cuda-11'
+            latest-image-aio: 'latest-aio-gpu-nvidia-cuda-11'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -99,6 +108,9 @@ jobs:
             runs-on: 'arc-runner-set'
             base-image: "ubuntu:22.04"
             aio: "-aio-gpu-nvidia-cuda-12"
+            latest-image: 'latest-gpu-nvidia-cuda-12'
+            latest-image-aio: 'latest-aio-gpu-nvidia-cuda-12'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: ''
             #platforms: 'linux/amd64,linux/arm64'
             platforms: 'linux/amd64'
@@ -108,6 +120,7 @@ jobs:
             image-type: 'extras'
             base-image: "ubuntu:22.04"
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'auto'
@@ -116,7 +129,10 @@ jobs:
             image-type: 'extras'
             aio: "-aio-gpu-hipblas"
             base-image: "rocm/dev-ubuntu-22.04:6.0-complete"
+            latest-image: 'latest-gpu-hipblas'
+            latest-image-aio: 'latest-aio-gpu-hipblas'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -125,6 +141,7 @@ jobs:
             image-type: 'extras'
             base-image: "rocm/dev-ubuntu-22.04:6.0-complete"
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f16'
             platforms: 'linux/amd64'
             tag-latest: 'auto'
@@ -134,6 +151,9 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             aio: "-aio-gpu-intel-f16"
+            latest-image: 'latest-gpu-intel-f16'
+            latest-image-aio: 'latest-aio-gpu-intel-f16'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f32'
             platforms: 'linux/amd64'
             tag-latest: 'auto'
@@ -143,6 +163,9 @@ jobs:
             image-type: 'extras'
             runs-on: 'arc-runner-set'
             aio: "-aio-gpu-intel-f32"
+            latest-image: 'latest-gpu-intel-f32'
+            latest-image-aio: 'latest-aio-gpu-intel-f32'
+            makeflags: "--jobs=3 --output-sync=target"
           # Core images
           - build-type: 'sycl_f16'
             platforms: 'linux/amd64'
@@ -152,6 +175,7 @@ jobs:
             ffmpeg: 'false'
             image-type: 'core'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f32'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -160,6 +184,7 @@ jobs:
             ffmpeg: 'false'
             image-type: 'core'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f16'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -168,6 +193,7 @@ jobs:
             ffmpeg: 'true'
             image-type: 'core'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'sycl_f32'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -176,6 +202,7 @@ jobs:
             ffmpeg: 'true'
             image-type: 'core'
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -184,6 +211,7 @@ jobs:
             image-type: 'core'
             base-image: "rocm/dev-ubuntu-22.04:6.0-complete"
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -192,6 +220,7 @@ jobs:
             image-type: 'core'
             base-image: "rocm/dev-ubuntu-22.04:6.0-complete"
             runs-on: 'arc-runner-set'
+            makeflags: "--jobs=3 --output-sync=target"
   
   core-image-build:
     uses: ./.github/workflows/image_build.yml
@@ -207,7 +236,9 @@ jobs:
       runs-on: ${{ matrix.runs-on }}
       aio: ${{ matrix.aio }}
       base-image: ${{ matrix.base-image }}
-      makeflags: "-j3"
+      makeflags: ${{ matrix.makeflags }}
+      latest-image: ${{ matrix.latest-image }}
+      latest-image-aio: ${{ matrix.latest-image-aio }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -225,6 +256,9 @@ jobs:
             base-image: "ubuntu:22.04"
             runs-on: 'ubuntu-latest'
             aio: "-aio-cpu"
+            latest-image: 'latest-cpu'
+            latest-image-aio: 'latest-aio-cpu'
+            makeflags: "--jobs=5 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "11"
             cuda-minor-version: "7"
@@ -235,6 +269,7 @@ jobs:
             image-type: 'core'
             base-image: "ubuntu:22.04"
             runs-on: 'ubuntu-latest'
+            makeflags: "--jobs=5 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -245,6 +280,7 @@ jobs:
             image-type: 'core'
             base-image: "ubuntu:22.04"
             runs-on: 'ubuntu-latest'
+            makeflags: "--jobs=5 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "11"
             cuda-minor-version: "7"
@@ -255,6 +291,7 @@ jobs:
             image-type: 'core'
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=5 --output-sync=target"
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "1"
@@ -265,3 +302,4 @@ jobs:
             image-type: 'core'
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:22.04"
+            makeflags: "--jobs=5 --output-sync=target"

.github/workflows/image_build.yml

@@ -29,6 +29,14 @@ on:
         description: 'Tag latest'
         default: ''
         type: string
+      latest-image:
+          description: 'Tag latest'
+          default: ''
+          type: string
+      latest-image-aio:
+          description: 'Tag latest'
+          default: ''
+          type: string
       tag-suffix:
         description: 'Tag suffix'
         default: ''
@@ -49,7 +57,7 @@ on:
       makeflags:
         description: 'Make Flags'
         required: false
-        default: ''
+        default: '--jobs=3 --output-sync=target'
         type: string
       aio:
         description: 'AIO Image Name'
@@ -79,6 +87,7 @@ jobs:
           && sudo apt-get install -y git
       - name: Checkout
         uses: actions/checkout@v4
+
       - name: Release space from worker
         if: inputs.runs-on == 'ubuntu-latest'
         run: |
@@ -120,6 +129,7 @@ jobs:
           sudo rm -rf "/usr/local/share/boost" || true
           sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
           df -h
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -134,6 +144,7 @@ jobs:
           flavor: |
             latest=${{ inputs.tag-latest }}
             suffix=${{ inputs.tag-suffix }}
+
       - name: Docker meta AIO (quay.io)
         if: inputs.aio != ''
         id: meta_aio
@@ -147,6 +158,7 @@ jobs:
           flavor: |
             latest=${{ inputs.tag-latest }}
             suffix=${{ inputs.aio }}
+
       - name: Docker meta AIO (dockerhub)
         if: inputs.aio != ''
         id: meta_aio_dockerhub
@@ -160,6 +172,7 @@ jobs:
           flavor: |
             latest=${{ inputs.tag-latest }}
             suffix=${{ inputs.aio }}
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@master
         with:
@@ -184,6 +197,25 @@ jobs:
           username: ${{ secrets.quayUsername }}
           password: ${{ secrets.quayPassword }}
 
+      - name: Cache GRPC
+        uses: docker/build-push-action@v5
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            IMAGE_TYPE=${{ inputs.image-type }}
+            BASE_IMAGE=${{ inputs.base-image }}
+            MAKEFLAGS=${{ inputs.makeflags }}
+            GRPC_VERSION=v1.58.0
+          context: .
+          file: ./Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,ignore-error=true
+          target: grpc
+          platforms: ${{ inputs.platforms }}
+          push: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
@@ -198,18 +230,20 @@ jobs:
             MAKEFLAGS=${{ inputs.makeflags }}
           context: .
           file: ./Dockerfile
+          cache-from: type=gha
           platforms: ${{ inputs.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-      -
-        name: Inspect image
+
+      - name: Inspect image
         if: github.event_name != 'pull_request'
         run: |
           docker pull localai/localai:${{ steps.meta.outputs.version }}
           docker image inspect localai/localai:${{ steps.meta.outputs.version }}
           docker pull quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }}
           docker image inspect quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }}
+
       - name: Build and push AIO image
         if: inputs.aio != ''
         uses: docker/build-push-action@v5
@@ -217,12 +251,14 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           build-args: |
             BASE_IMAGE=quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }}
+            MAKEFLAGS=${{ inputs.makeflags }}
           context: .
           file: ./Dockerfile.aio
           platforms: ${{ inputs.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta_aio.outputs.tags }}
           labels: ${{ steps.meta_aio.outputs.labels }}
+
       - name: Build and push AIO image (dockerhub)
         if: inputs.aio != ''
         uses: docker/build-push-action@v5
@@ -230,15 +266,39 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           build-args: |
             BASE_IMAGE=localai/localai:${{ steps.meta.outputs.version }}
+            MAKEFLAGS=${{ inputs.makeflags }}
           context: .
           file: ./Dockerfile.aio
           platforms: ${{ inputs.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta_aio_dockerhub.outputs.tags }}
           labels: ${{ steps.meta_aio_dockerhub.outputs.labels }}
+
+      - name: Latest tag
+        # run this on branches, when it is a tag and there is a latest-image defined
+        if: github.event_name != 'pull_request' && inputs.latest-image != ''  && github.ref_type == 'tag'
+        run: |
+          docker pull localai/localai:${{ steps.meta.outputs.version }}
+          docker tag localai/localai:${{ steps.meta.outputs.version }} localai/localai:${{ inputs.latest-image }}
+          docker push localai/localai:${{ inputs.latest-image }}
+          docker pull quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }}
+          docker tag quay.io/go-skynet/local-ai:${{ steps.meta.outputs.version }} quay.io/go-skynet/local-ai:${{ inputs.latest-image }}
+          docker push quay.io/go-skynet/local-ai:${{ inputs.latest-image }}
+      - name: Latest AIO tag
+        # run this on branches, when it is a tag and there is a latest-image defined
+        if: github.event_name != 'pull_request' && inputs.latest-image-aio != ''  && github.ref_type == 'tag'
+        run: |
+          docker pull localai/localai:${{ steps.meta_aio_dockerhub.outputs.version }}
+          docker tag localai/localai:${{ steps.meta_aio_dockerhub.outputs.version }} localai/localai:${{ inputs.latest-image-aio }}
+          docker push localai/localai:${{ inputs.latest-image-aio }}
+          docker pull quay.io/go-skynet/local-ai:${{ steps.meta_aio.outputs.version }}
+          docker tag quay.io/go-skynet/local-ai:${{ steps.meta_aio.outputs.version }} quay.io/go-skynet/local-ai:${{ inputs.latest-image-aio }}
+          docker push quay.io/go-skynet/local-ai:${{ inputs.latest-image-aio }}
+  
       - name: job summary
         run: |
           echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY
+
       - name: job summary(AIO)
         if: inputs.aio != ''
         run: |

.github/workflows/labeler.yml

@@ -0,0 +1,12 @@
+name: "Pull Request Labeler"
+on:
+- pull_request_target
+
+jobs:
+  labeler:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v5

.github/workflows/release.yaml

@@ -2,6 +2,9 @@ name: Build and Release
 
 on: push
 
+env:
+  GRPC_VERSION: v1.58.0
+
 permissions:
   contents: write
 
@@ -32,7 +35,8 @@ jobs:
           submodules: true
       - uses: actions/setup-go@v4
         with:
-          go-version: '>=1.21.0'
+          go-version: '1.21.x'
+          cache: false
       - name: Dependencies
         run: |
           sudo apt-get update
@@ -54,17 +58,17 @@ jobs:
         uses: actions/cache@v3
         with:
           path: grpc
-          key: ${{ runner.os }}-grpc
+          key: ${{ runner.os }}-grpc-${{ env.GRPC_VERSION }}
       - name: Build grpc
         if: steps.cache-grpc.outputs.cache-hit != 'true'
         run: |
-          git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+          git clone --recurse-submodules -b ${{ env.GRPC_VERSION }} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
           cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
             -DgRPC_BUILD_TESTS=OFF \
-            ../.. && sudo make -j12
+            ../.. && sudo make --jobs 5 --output-sync=target
       - name: Install gRPC
         run: |
-          cd grpc && cd cmake/build && sudo make -j12 install
+          cd grpc && cd cmake/build && sudo make --jobs 5 --output-sync=target install
       - name: Build
         id: build
         env:
@@ -98,11 +102,11 @@ jobs:
           submodules: true
       - uses: actions/setup-go@v4
         with:
-          go-version: '>=1.21.0'
+          go-version: '1.21.x'
+          cache: false
       - name: Dependencies
         run: |
           sudo apt-get install -y --no-install-recommends libopencv-dev
-          sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
       - name: Build stablediffusion
         run: |
           make backend-assets/grpc/stablediffusion
@@ -136,7 +140,8 @@ jobs:
           submodules: true
       - uses: actions/setup-go@v4
         with:
-          go-version: '>=1.21.0'
+          go-version: '1.21.x'
+          cache: false
       - name: Dependencies
         run: |
           brew install protobuf grpc

.github/workflows/secscan.yaml

@@ -0,0 +1,27 @@
+name: "Security Scan"
+
+# Run workflow each time code is pushed to your repository and on a schedule.
+# The scheduled workflow runs every at 00:00 on Sunday UTC time.
+on:
+  push:
+  schedule:
+  - cron: '0 0 * * 0'
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v3
+      - name: Run Gosec Security Scanner
+        uses: securego/gosec@master
+        with:
+          # we let the report trigger content trigger a failure using the GitHub Security features.
+          args: '-no-fail -fmt sarif -out results.sarif ./...'
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: results.sarif

.github/workflows/test-extra.yml

@@ -33,15 +33,15 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          sudo apt-get install -y libopencv-dev
           
           sudo rm -rfv /usr/bin/conda || true
 
       - name: Test transformers
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/transformers
-           make -C backend/python/transformers test
+           make --jobs=5 --output-sync=target -C backend/python/transformers
+           make --jobs=5 --output-sync=target -C backend/python/transformers test
 
   tests-sentencetransformers:
     runs-on: ubuntu-latest
@@ -62,15 +62,15 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          sudo apt-get install -y libopencv-dev
           
           sudo rm -rfv /usr/bin/conda || true
 
       - name: Test sentencetransformers
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/sentencetransformers
-           make -C backend/python/sentencetransformers test
+           make --jobs=5 --output-sync=target -C backend/python/sentencetransformers
+           make --jobs=5 --output-sync=target -C backend/python/sentencetransformers test
 
   tests-diffusers:
     runs-on: ubuntu-latest
@@ -91,15 +91,15 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          sudo apt-get install -y libopencv-dev
           
           sudo rm -rfv /usr/bin/conda || true
 
       - name: Test diffusers
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/diffusers
-           make -C backend/python/diffusers test
+           make --jobs=5 --output-sync=target -C backend/python/diffusers
+           make --jobs=5 --output-sync=target -C backend/python/diffusers test
 
 
   tests-transformers-musicgen:
@@ -121,46 +121,46 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          sudo apt-get install -y libopencv-dev
           
           sudo rm -rfv /usr/bin/conda || true
 
       - name: Test transformers-musicgen
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/transformers-musicgen
-           make -C backend/python/transformers-musicgen test
+           make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen
+           make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen test
 
 
 
-  tests-petals:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-        with: 
-          submodules: true
-      - name: Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
-          sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+  # tests-petals:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Clone
+  #       uses: actions/checkout@v4
+  #       with: 
+  #         submodules: true
+  #     - name: Dependencies
+  #       run: |
+  #         sudo apt-get update
+  #         sudo apt-get install build-essential ffmpeg
+  #         curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
+  #            sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+  #             gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
+  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
+  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
+  #            sudo apt-get update && \
+  #            sudo apt-get install -y conda
+  #         sudo apt-get install -y ca-certificates cmake curl patch
+  #         sudo apt-get install -y libopencv-dev
           
-          sudo rm -rfv /usr/bin/conda || true
+  #         sudo rm -rfv /usr/bin/conda || true
 
-      - name: Test petals
-        run: |
-           export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/petals
-           make -C backend/python/petals test
+  #     - name: Test petals
+  #       run: |
+  #          export PATH=$PATH:/opt/conda/bin
+  #          make --jobs=5 --output-sync=target -C backend/python/petals
+  #          make --jobs=5 --output-sync=target -C backend/python/petals test
 
            
 
@@ -223,15 +223,15 @@ jobs:
   #            sudo apt-get update && \
   #            sudo apt-get install -y conda
   #         sudo apt-get install -y ca-certificates cmake curl patch
-  #         sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+  #         sudo apt-get install -y libopencv-dev
           
   #         sudo rm -rfv /usr/bin/conda || true
 
   #     - name: Test bark
   #       run: |
   #          export PATH=$PATH:/opt/conda/bin
-  #          make -C backend/python/bark
-  #          make -C backend/python/bark test
+  #          make --jobs=5 --output-sync=target -C backend/python/bark
+  #          make --jobs=5 --output-sync=target -C backend/python/bark test
 
            
   # Below tests needs GPU. Commented out for now
@@ -255,13 +255,13 @@ jobs:
   #            sudo apt-get update && \
   #            sudo apt-get install -y conda
   #         sudo apt-get install -y ca-certificates cmake curl patch
-  #         sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+  #         sudo apt-get install -y libopencv-dev
   #         sudo rm -rfv /usr/bin/conda || true
   #     - name: Test vllm
   #       run: |
   #          export PATH=$PATH:/opt/conda/bin
-  #          make -C backend/python/vllm
-  #          make -C backend/python/vllm test
+  #          make --jobs=5 --output-sync=target -C backend/python/vllm
+  #          make --jobs=5 --output-sync=target -C backend/python/vllm test
   tests-vallex:
     runs-on: ubuntu-latest
     steps:
@@ -281,13 +281,13 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2    
+          sudo apt-get install -y libopencv-dev    
           sudo rm -rfv /usr/bin/conda || true
       - name: Test vall-e-x
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/vall-e-x
-           make -C backend/python/vall-e-x test
+           make --jobs=5 --output-sync=target -C backend/python/vall-e-x
+           make --jobs=5 --output-sync=target -C backend/python/vall-e-x test
 
   tests-coqui:
     runs-on: ubuntu-latest
@@ -313,5 +313,5 @@ jobs:
       - name: Test coqui
         run: |
            export PATH=$PATH:/opt/conda/bin
-           make -C backend/python/coqui
-           make -C backend/python/coqui test
+           make --jobs=5 --output-sync=target -C backend/python/coqui
+           make --jobs=5 --output-sync=target -C backend/python/coqui test

.github/workflows/test.yml

@@ -9,6 +9,9 @@ on:
     tags:
       - '*'
 
+env:
+  GRPC_VERSION: v1.58.0
+
 concurrency:
   group: ci-tests-${{ github.head_ref || github.ref }}-${{ github.repository }}
   cancel-in-progress: true
@@ -60,6 +63,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
+          cache: false
       # You can test your matrix by printing the current Go version
       - name: Display Go version
         run: go version
@@ -75,7 +79,7 @@ jobs:
              sudo apt-get update && \
              sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch
-          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          sudo apt-get install -y libopencv-dev
           
           sudo rm -rfv /usr/bin/conda || true
           PATH=$PATH:/opt/conda/bin make -C backend/python/sentencetransformers
@@ -91,20 +95,20 @@ jobs:
         uses: actions/cache@v3
         with:
           path: grpc
-          key: ${{ runner.os }}-grpc
+          key: ${{ runner.os }}-grpc-${{ env.GRPC_VERSION }}
       - name: Build grpc
         if: steps.cache-grpc.outputs.cache-hit != 'true'
         run: |
-          git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+          git clone --recurse-submodules -b ${{ env.GRPC_VERSION }} --depth 1 --jobs 5 --shallow-submodules https://github.com/grpc/grpc && \
           cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
             -DgRPC_BUILD_TESTS=OFF \
-            ../.. && sudo make -j12
+            ../.. && sudo make --jobs 5
       - name: Install gRPC
         run: |
-          cd grpc && cd cmake/build && sudo make -j12 install
+          cd grpc && cd cmake/build && sudo make --jobs 5 install
       - name: Test
         run: |
-          GO_TAGS="stablediffusion tts" make test
+          GO_TAGS="stablediffusion tts" make --jobs 5 --output-sync=target test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3
@@ -151,7 +155,7 @@ jobs:
           submodules: true
       - name: Build images
         run: |
-          docker build --build-arg FFMPEG=true --build-arg IMAGE_TYPE=core -t local-ai:tests -f Dockerfile .
+          docker build --build-arg FFMPEG=true --build-arg IMAGE_TYPE=core --build-arg MAKEFLAGS="--jobs=5 --output-sync=target" -t local-ai:tests -f Dockerfile .
           BASE_IMAGE=local-ai:tests DOCKER_AIO_IMAGE=local-ai-aio:test make docker-aio
       - name: Test
         run: |
@@ -176,17 +180,20 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
+          cache: false
       # You can test your matrix by printing the current Go version
       - name: Display Go version
         run: go version
       - name: Dependencies
         run: |
-          brew install protobuf grpc
+          brew install protobuf grpc make
       - name: Test
         run: |
           export C_INCLUDE_PATH=/usr/local/include
           export CPLUS_INCLUDE_PATH=/usr/local/include
-          BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DLLAMA_F16C=OFF -DLLAMA_AVX512=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF" make test
+          # Used to run the newer GNUMake version from brew that supports --output-sync
+          export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH"
+          BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DLLAMA_F16C=OFF -DLLAMA_AVX512=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF" make --jobs 4 --output-sync=target test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "golang.go"
+    ]
+}

Dockerfile

@@ -90,11 +90,35 @@ RUN if [ ! -e /usr/bin/python ]; then \
 ###################################
 ###################################
 
+FROM ${BASE_IMAGE} as grpc
+
+ARG MAKEFLAGS
+ARG GRPC_VERSION=v1.58.0
+
+ENV MAKEFLAGS=${MAKEFLAGS}
+
+WORKDIR /build
+
+RUN apt-get update && \
+    apt-get install -y g++ cmake git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone --recurse-submodules --jobs 4 -b ${GRPC_VERSION} --depth 1 --shallow-submodules https://github.com/grpc/grpc
+
+RUN cd grpc && \
+    mkdir -p cmake/build && \
+    cd cmake/build && \
+    cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF ../.. && \
+    make
+
+###################################
+###################################
+
 FROM requirements-${IMAGE_TYPE} as builder
 
 ARG GO_TAGS="stablediffusion tts"
 ARG GRPC_BACKENDS
-ARG BUILD_GRPC=true
 ARG MAKEFLAGS
 
 ENV GRPC_BACKENDS=${GRPC_BACKENDS}
@@ -121,12 +145,9 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ]; then \
 # stablediffusion does not tolerate a newer version of abseil, build it first
 RUN GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
 
-RUN if [ "${BUILD_GRPC}" = "true" ]; then \
-    git clone --recurse-submodules --jobs 4 -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
-    cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
-      -DgRPC_BUILD_TESTS=OFF \
-       ../.. && make install \
-    ; fi
+COPY --from=grpc /build/grpc ./grpc/
+
+RUN cd /build/grpc/cmake/build && make install
 
 # Rebuild with defaults backends
 RUN make build
@@ -179,7 +200,7 @@ WORKDIR /build
 COPY . .
 
 COPY --from=builder /build/sources ./sources/
-COPY --from=builder /build/grpc ./grpc/
+COPY --from=grpc /build/grpc ./grpc/
 
 RUN make prepare-sources && cd /build/grpc/cmake/build && make install && rm -rf grpc
 

Makefile

@@ -5,7 +5,7 @@ BINARY_NAME=local-ai
 
 # llama.cpp versions
 GOLLAMA_STABLE_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
-CPPLLAMA_VERSION?=b06c16ef9f81d84da520232c125d4d8a1d273736
+CPPLLAMA_VERSION?=1b67731e184e27a465b8c5476061294a4af668ea
 
 # gpt4all version
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
@@ -16,7 +16,7 @@ RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
 RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6
 
 # whisper.cpp version
-WHISPER_CPP_VERSION?=1558ec5a16cb2b2a0bf54815df1d41f83dc3815b
+WHISPER_CPP_VERSION?=8f253ef3af1c62c04316ba4afa7145fc4d701a8c
 
 # bert.cpp version
 BERT_VERSION?=6abe312cded14042f6b7c3cd8edf082713334a4d
@@ -28,7 +28,7 @@ PIPER_VERSION?=9d0100873a7dbb0824dfea40e8cec70a1b110759
 STABLEDIFFUSION_VERSION?=362df9da29f882dbf09ade61972d16a1f53c3485
 
 # tinydream version
-TINYDREAM_VERSION?=772a9c0d9aaf768290e63cca3c904fe69faf677a
+TINYDREAM_VERSION?=22a12a4bc0ac5455856f28f3b771331a551a4293
 
 export BUILD_TYPE?=
 export STABLE_BUILD_TYPE?=$(BUILD_TYPE)
@@ -224,7 +224,7 @@ sources/go-stable-diffusion:
 	cd sources/go-stable-diffusion && git checkout -b build $(STABLEDIFFUSION_VERSION) && git submodule update --init --recursive --depth 1
 
 sources/go-stable-diffusion/libstablediffusion.a: sources/go-stable-diffusion
-	$(MAKE) -C sources/go-stable-diffusion libstablediffusion.a
+	CPATH="$(CPATH):/usr/include/opencv4" $(MAKE) -C sources/go-stable-diffusion libstablediffusion.a
 
 ## tiny-dream
 sources/go-tiny-dream:
@@ -263,6 +263,7 @@ dropreplace:
 	$(GOCMD) mod edit -dropreplace github.com/mudler/go-piper
 	$(GOCMD) mod edit -dropreplace github.com/mudler/go-stable-diffusion
 	$(GOCMD) mod edit -dropreplace github.com/nomic-ai/gpt4all/gpt4all-bindings/golang
+	$(GOCMD) mod edit -dropreplace github.com/go-skynet/go-llama.cpp
 
 prepare-sources: get-sources replace
 	$(GOCMD) mod download
@@ -306,6 +307,12 @@ build: prepare backend-assets grpcs ## Build the project
 	$(info ${GREEN}I LD_FLAGS: ${YELLOW}$(LD_FLAGS)${RESET})
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o $(BINARY_NAME) ./
 
+build-minimal:
+	BUILD_GRPC_FOR_BACKEND_LLAMA=true GRPC_BACKENDS=backend-assets/grpc/llama-cpp GO_TAGS=none $(MAKE) build
+
+build-api:
+	BUILD_GRPC_FOR_BACKEND_LLAMA=true BUILD_API_ONLY=true GO_TAGS=none $(MAKE) build
+
 dist: build
 	mkdir -p release
 	cp $(BINARY_NAME) release/$(BINARY_NAME)-$(BUILD_ID)-$(OS)-$(ARCH)
@@ -348,7 +355,7 @@ prepare-e2e:
 	mkdir -p $(TEST_DIR)
 	cp -rfv $(abspath ./tests/e2e-fixtures)/gpu.yaml $(TEST_DIR)/gpu.yaml
 	test -e $(TEST_DIR)/ggllm-test-model.bin || wget -q https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q2_K.gguf -O $(TEST_DIR)/ggllm-test-model.bin
-	docker build --build-arg BUILD_GRPC=true --build-arg GRPC_BACKENDS="$(GRPC_BACKENDS)" --build-arg IMAGE_TYPE=core --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg CUDA_MAJOR_VERSION=11 --build-arg CUDA_MINOR_VERSION=7 --build-arg FFMPEG=true -t localai-tests .
+	docker build --build-arg GRPC_BACKENDS="$(GRPC_BACKENDS)" --build-arg IMAGE_TYPE=core --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg CUDA_MAJOR_VERSION=11 --build-arg CUDA_MINOR_VERSION=7 --build-arg FFMPEG=true -t localai-tests .
 
 run-e2e-image:
 	ls -liah $(abspath ./tests/e2e-fixtures)
@@ -531,7 +538,7 @@ backend-assets/grpc/rwkv: sources/go-rwkv sources/go-rwkv/librwkv.a backend-asse
 	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./backend/go/llm/rwkv
 
 backend-assets/grpc/stablediffusion: sources/go-stable-diffusion sources/go-stable-diffusion/libstablediffusion.a backend-assets/grpc
-	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(CURDIR)/sources/go-stable-diffusion/ LIBRARY_PATH=$(CURDIR)/sources/go-stable-diffusion/ \
+	CGO_LDFLAGS="$(CGO_LDFLAGS)" CPATH="$(CPATH):$(CURDIR)/sources/go-stable-diffusion/:/usr/include/opencv4" LIBRARY_PATH=$(CURDIR)/sources/go-stable-diffusion/ \
 	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/stablediffusion
 
 backend-assets/grpc/tinydream: sources/go-tiny-dream sources/go-tiny-dream/libtinydream.a backend-assets/grpc
@@ -556,7 +563,8 @@ docker:
 	docker build \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
-		--build-arg GO_TAGS=$(GO_TAGS) \
+		--build-arg GO_TAGS="$(GO_TAGS)" \
+		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
 		--build-arg BUILD_TYPE=$(BUILD_TYPE) \
 		-t $(DOCKER_IMAGE) .
 	
@@ -564,6 +572,7 @@ docker-aio:
 	@echo "Building AIO image with base $(BASE_IMAGE) as $(DOCKER_AIO_IMAGE)"
 	docker build \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
+		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
 		-t $(DOCKER_AIO_IMAGE) -f Dockerfile.aio .
 
 docker-aio-all:
@@ -575,6 +584,7 @@ docker-image-intel:
 		--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
 		--build-arg GO_TAGS="none" \
+		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
 		--build-arg BUILD_TYPE=sycl_f32 -t $(DOCKER_IMAGE) .
 
 docker-image-intel-xpu:
@@ -582,4 +592,9 @@ docker-image-intel-xpu:
 		--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 \
 		--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
 		--build-arg GO_TAGS="none" \
+		--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
 		--build-arg BUILD_TYPE=sycl_f32 -t $(DOCKER_IMAGE) .
+
+.PHONY: swagger
+swagger:
+	swag init -g core/http/api.go --output swagger

README.md

@@ -38,6 +38,10 @@
 </a>
 </p>
 
+> :bulb: Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [:speech_balloon: Discord](https://discord.gg/uJAeKSAGDy) [:book: Documentation website](https://localai.io/)
+>
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/) [ 🚀 Roadmap ](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
+
 [![tests](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml)[![Build and Release](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml)[![build container images](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml)[![Bump dependencies](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml)[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/localai)](https://artifacthub.io/packages/search?repo=localai)
 
 **LocalAI** is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that’s compatible with OpenAI (Elevenlabs, Anthropic... ) API specifications for local AI inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families. Does not require GPU.
@@ -46,14 +50,13 @@
 
 [Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
 
+- Landing page: https://github.com/mudler/LocalAI/pull/1922
+- Openvino support: https://github.com/mudler/LocalAI/pull/1892
 - Vector store: https://github.com/mudler/LocalAI/pull/1795
 - All-in-one container image: https://github.com/mudler/LocalAI/issues/1855
-- Parallel function calling: https://github.com/mudler/LocalAI/pull/1726
+- Parallel function calling: https://github.com/mudler/LocalAI/pull/1726 / Tools API support: https://github.com/mudler/LocalAI/pull/1715
 - Upload file API: https://github.com/mudler/LocalAI/pull/1703
-- Tools API support: https://github.com/mudler/LocalAI/pull/1715
-- LLaVa 1.6: https://github.com/mudler/LocalAI/pull/1714
-- ROCm container images: https://github.com/mudler/LocalAI/pull/1595
-- Intel GPU support (sycl, transformers, diffusers): https://github.com/mudler/LocalAI/issues/1653
+- ROCm container images: https://github.com/mudler/LocalAI/pull/1595 / Intel GPU support (sycl, transformers, diffusers): https://github.com/mudler/LocalAI/issues/1653
 - Mamba support: https://github.com/mudler/LocalAI/pull/1589
 - Start and share models with config file: https://github.com/mudler/LocalAI/pull/1522
 - 🐸 Coqui: https://github.com/mudler/LocalAI/pull/1489
@@ -77,7 +80,7 @@ For those in a hurry, here's a straightforward one-liner to launch a LocalAI AIO
 ```bash
 docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-cpu
 # or, if you have an Nvidia GPU:
-# docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-cuda12
+# docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-12
 ```
 
 ## 🚀 [Features](https://localai.io/features/)

aio/cpu/embeddings.yaml

@@ -1,11 +1,5 @@
-backend: bert-embeddings
-embeddings: true
-f16: true
-
-gpu_layers: 90
-mmap: true
 name: text-embedding-ada-002
-
+backend: bert-embeddings
 parameters:
   model: huggingface://mudler/all-MiniLM-L6-v2/ggml-model-q4_0.bin
 

aio/cpu/image-gen.yaml

@@ -50,4 +50,13 @@ download_files:
   uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/releases/download/naifu/UNetModel-MHA-fp16.bin"
 - filename: "stablediffusion_assets/vocab.txt"
   sha256: "e30e57b6f1e47616982ef898d8922be24e535b4fa3d0110477b3a6f02ebbae7d"
-  uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/vocab.txt"
+  uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/vocab.txt"
+
+usage: |
+        curl http://localhost:8080/v1/images/generations \
+          -H "Content-Type: application/json" \
+          -d '{
+            "prompt": "<positive prompt>|<negative prompt>",
+            "step": 25,
+            "size": "512x512"
+          }'

aio/cpu/text-to-text.yaml

@@ -1,25 +1,53 @@
 name: gpt-4
 mmap: true
 parameters:
-  model: huggingface://l3utterfly/phi-2-layla-v1-chatml-gguf/phi-2-layla-v1-chatml-Q8_0.gguf
+  model: huggingface://NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/Hermes-2-Pro-Mistral-7B.Q2_K.gguf
 
 template:
   chat_message: |
-    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
-    {{if .Content}}{{.Content}}{{end}}
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}
+    {{- if .FunctionCall }}<tool_call>{{end}}
+    {{- if eq .RoleName "tool" }}<tool_result>{{end }}
+    {{- if .Content}}
+    {{.Content}}
+    {{- end }}
+    {{- if .FunctionCall}}{{toJson .FunctionCall}}{{end }}
+    {{- if .FunctionCall }}</tool_call>{{end }}
+    {{- if eq .RoleName "tool" }}</tool_result>{{end }}
     <|im_end|>
+  # https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF#prompt-format-for-function-calling
+  function: |
+    <|im_start|>system
+    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
+    <tools>
+    {{range .Functions}}
+    {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
+    {{end}}
+    </tools>
+    Use the following pydantic model json schema for each tool call you will make:
+    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}
+    For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+    <tool_call>
+    {'arguments': <args-dict>, 'name': <function-name>}
+    </tool_call>
+    <|im_end|>
+    {{.Input -}}
+    <|im_start|>assistant
+    <tool_call>
   chat: |
-    {{.Input}}
+    {{.Input -}}
     <|im_start|>assistant
   completion: |
     {{.Input}}
-context_size: 2048
+context_size: 4096
 f16: true
 stopwords:
 - <|im_end|>
 - <dummy32000>
+- "\n</tool_call>"
+- "\n\n\n"
 usage: |
       curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
-          "model": "phi-2-chat",
+          "model": "gpt-4",
           "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}]
       }'

aio/cpu/vision.yaml

@@ -1,8 +1,6 @@
 backend: llama-cpp
 context_size: 4096
 f16: true
-
-gpu_layers: 90
 mmap: true
 name: gpt-4-vision-preview
 
@@ -14,13 +12,6 @@ roles:
 mmproj: bakllava-mmproj.gguf
 parameters:
   model: bakllava.gguf
-  temperature: 0.2
-  top_k: 40
-  top_p: 0.95
-  seed: -1
-mirostat: 2
-mirostat_eta: 1.0
-mirostat_tau: 1.0
 
 template:
   chat: |

aio/entrypoint.sh

@@ -5,53 +5,77 @@ echo "===> LocalAI All-in-One (AIO) container starting..."
 GPU_ACCELERATION=false
 GPU_VENDOR=""
 
+function check_intel() {
+    if lspci | grep -E 'VGA|3D' | grep -iq intel; then
+        echo "Intel GPU detected"
+        if [ -d /opt/intel ]; then
+            GPU_ACCELERATION=true
+            GPU_VENDOR=intel
+        else
+            echo "Intel GPU detected, but Intel GPU drivers are not installed. GPU acceleration will not be available."
+        fi
+    fi
+}
+
+function check_nvidia_wsl() {
+    if lspci | grep -E 'VGA|3D' | grep -iq "Microsoft Corporation Device 008e"; then
+        # We make the assumption this WSL2 cars is NVIDIA, then check for nvidia-smi
+        # Make sure the container was run with `--gpus all` as the only required parameter
+        echo "NVIDIA GPU detected via WSL2"
+        # nvidia-smi should be installed in the container
+        if nvidia-smi; then
+            GPU_ACCELERATION=true
+            GPU_VENDOR=nvidia
+        else
+            echo "NVIDIA GPU detected via WSL2, but nvidia-smi is not installed. GPU acceleration will not be available."
+        fi
+    fi
+}
+
+function check_amd() {
+    if lspci | grep -E 'VGA|3D' | grep -iq amd; then
+        echo "AMD GPU detected"
+        # Check if ROCm is installed
+        if [ -d /opt/rocm ]; then
+            GPU_ACCELERATION=true
+            GPU_VENDOR=amd
+        else
+            echo "AMD GPU detected, but ROCm is not installed. GPU acceleration will not be available."
+        fi
+    fi
+}
+
+function check_nvidia() {
+    if lspci | grep -E 'VGA|3D' | grep -iq nvidia; then
+        echo "NVIDIA GPU detected"
+        # nvidia-smi should be installed in the container
+        if nvidia-smi; then
+            GPU_ACCELERATION=true
+            GPU_VENDOR=nvidia
+        else
+            echo "NVIDIA GPU detected, but nvidia-smi is not installed. GPU acceleration will not be available."
+        fi
+    fi
+}
+
+function check_metal() {
+    if system_profiler SPDisplaysDataType | grep -iq 'Metal'; then
+        echo "Apple Metal supported GPU detected"
+        GPU_ACCELERATION=true
+        GPU_VENDOR=apple
+    fi
+}
+
 function detect_gpu() {
     case "$(uname -s)" in
         Linux)
-            if lspci | grep -E 'VGA|3D' | grep -iq nvidia; then
-                echo "NVIDIA GPU detected"
-                # nvidia-smi should be installed in the container
-                if nvidia-smi; then
-                    GPU_ACCELERATION=true
-                    GPU_VENDOR=nvidia
-                else
-                    echo "NVIDIA GPU detected, but nvidia-smi is not installed. GPU acceleration will not be available."
-                fi
-            elif lspci | grep -E 'VGA|3D' | grep -iq amd; then
-                echo "AMD GPU detected"
-                # Check if ROCm is installed
-                if [ -d /opt/rocm ]; then
-                    GPU_ACCELERATION=true
-                    GPU_VENDOR=amd
-                else
-                    echo "AMD GPU detected, but ROCm is not installed. GPU acceleration will not be available."
-                fi
-            elif lspci | grep -E 'VGA|3D' | grep -iq intel; then
-                echo "Intel GPU detected"
-                if [ -d /opt/intel ]; then
-                    GPU_ACCELERATION=true
-                else
-                    echo "Intel GPU detected, but Intel GPU drivers are not installed. GPU acceleration will not be available."
-                fi
-            elif lspci | grep -E 'VGA|3D' | grep -iq "Microsoft Corporation Device 008e"; then
-                # We make the assumption this WSL2 cars is NVIDIA, then check for nvidia-smi
-                # Make sure the container was run with `--gpus all` as the only required parameter
-                echo "NVIDIA GPU detected via WSL2"
-                # nvidia-smi should be installed in the container
-                if nvidia-smi; then
-                    GPU_ACCELERATION=true
-                    GPU_VENDOR=nvidia
-                else
-                    echo "NVIDIA GPU detected via WSL2, but nvidia-smi is not installed. GPU acceleration will not be available."
-                fi
-            fi
+            check_nvidia
+            check_amd
+            check_intel
+            check_nvidia_wsl
             ;;
         Darwin)
-            if system_profiler SPDisplaysDataType | grep -iq 'Metal'; then
-                echo "Apple Metal supported GPU detected"
-                GPU_ACCELERATION=true
-                GPU_VENDOR=apple
-            fi
+            check_metal
             ;;
     esac
 }
@@ -75,7 +99,8 @@ function detect_gpu_size() {
             echo "Unable to determine NVIDIA GPU memory size. Falling back to CPU."
             GPU_SIZE=gpu-8g
         fi
-
+    elif [ "$GPU_ACCELERATION" = true ] && [ "$GPU_VENDOR" = "intel" ]; then
+        GPU_SIZE=intel
     # Default to a generic GPU size until we implement GPU size detection for non NVIDIA GPUs
     elif [ "$GPU_ACCELERATION" = true ]; then
         echo "Non-NVIDIA GPU detected. Specific GPU memory size detection is not implemented."
@@ -94,8 +119,8 @@ function check_vars() {
         exit 1
     fi
 
-    if [ -z "$SIZE" ]; then
-        echo "SIZE environment variable is not set. Please set it to one of the following: cpu, gpu-8g, gpu-16g, apple"
+    if [ -z "$PROFILE" ]; then
+        echo "PROFILE environment variable is not set. Please set it to one of the following: cpu, gpu-8g, gpu-16g, apple"
         exit 1
     fi
 }
@@ -103,11 +128,11 @@ function check_vars() {
 detect_gpu
 detect_gpu_size
 
-SIZE="${SIZE:-$GPU_SIZE}" # default to cpu
-export MODELS="${MODELS:-/aio/${SIZE}/embeddings.yaml,/aio/${SIZE}/text-to-speech.yaml,/aio/${SIZE}/image-gen.yaml,/aio/${SIZE}/text-to-text.yaml,/aio/${SIZE}/speech-to-text.yaml,/aio/${SIZE}/vision.yaml}"
+PROFILE="${PROFILE:-$GPU_SIZE}" # default to cpu
+export MODELS="${MODELS:-/aio/${PROFILE}/embeddings.yaml,/aio/${PROFILE}/text-to-speech.yaml,/aio/${PROFILE}/image-gen.yaml,/aio/${PROFILE}/text-to-text.yaml,/aio/${PROFILE}/speech-to-text.yaml,/aio/${PROFILE}/vision.yaml}"
 
 check_vars
 
-echo "Starting LocalAI with the following models: $MODELS"
+echo "===> Starting LocalAI[$PROFILE] with the following models: $MODELS"
 
-/build/entrypoint.sh "$@"
+exec /build/entrypoint.sh "$@"

aio/gpu-8g/embeddings.yaml

@@ -1,6 +1,5 @@
 name: text-embedding-ada-002
 backend: sentencetransformers
-embeddings: true
 parameters:
   model: all-MiniLM-L6-v2
 

aio/gpu-8g/image-gen.yaml

@@ -20,7 +20,6 @@ usage: |
           -H "Content-Type: application/json" \
           -d '{
             "prompt": "<positive prompt>|<negative prompt>",
-            "model": "dreamshaper",
             "step": 25,
             "size": "512x512"
           }'

aio/gpu-8g/text-to-text.yaml

@@ -3,39 +3,39 @@ mmap: true
 parameters:
   model: huggingface://NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/Hermes-2-Pro-Mistral-7B.Q6_K.gguf
 
-roles:
-  assistant_function_call: assistant
-  function: tool
 template:
   chat_message: |
-    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "function"}}{{.Role}}{{else if eq .RoleName "user"}}user{{end}}
-    {{ if eq .RoleName "assistant_function_call" }}<tool_call>{{end}}
-    {{ if eq .RoleName "function" }}<tool_result>{{end}}
-    {{if .Content}}{{.Content}}{{end}}
-    {{if .FunctionCall}}{{toJson .FunctionCall}}{{end}}
-    {{ if eq .RoleName "assistant_function_call" }}</tool_call>{{end}}
-    {{ if eq .RoleName "function" }}</tool_result>{{end}}
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}
+    {{- if .FunctionCall }}<tool_call>{{end}}
+    {{- if eq .RoleName "tool" }}<tool_result>{{end }}
+    {{- if .Content}}
+    {{.Content}}
+    {{- end }}
+    {{- if .FunctionCall}}{{toJson .FunctionCall}}{{end }}
+    {{- if .FunctionCall }}</tool_call>{{end }}
+    {{- if eq .RoleName "tool" }}</tool_result>{{end }}
     <|im_end|>
   # https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF#prompt-format-for-function-calling
   function: |
     <|im_start|>system
-    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: 
+    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
     <tools>
     {{range .Functions}}
     {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
     {{end}}
-    </tools> 
-    Use the following pydantic model json schema for each tool call you will make: 
-    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']} 
+    </tools>
+    Use the following pydantic model json schema for each tool call you will make:
+    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}
     For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
     <tool_call>
     {'arguments': <args-dict>, 'name': <function-name>}
-    </tool_call><|im_end|>
-    {{.Input}}
+    </tool_call>
+    <|im_end|>
+    {{.Input -}}
     <|im_start|>assistant
     <tool_call>
   chat: |
-    {{.Input}}
+    {{.Input -}}
     <|im_start|>assistant
   completion: |
     {{.Input}}
@@ -44,6 +44,8 @@ f16: true
 stopwords:
 - <|im_end|>
 - <dummy32000>
+- "\n</tool_call>"
+- "\n\n\n"
 usage: |
       curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
           "model": "gpt-4",

aio/gpu-8g/vision.yaml

@@ -1,8 +1,6 @@
 backend: llama-cpp
 context_size: 4096
 f16: true
-
-gpu_layers: 90
 mmap: true
 name: gpt-4-vision-preview
 

aio/intel/embeddings.yaml

@@ -0,0 +1,12 @@
+name: text-embedding-ada-002
+backend: sentencetransformers
+parameters:
+  model: all-MiniLM-L6-v2
+
+usage: |
+    You can test this model with curl like this:
+
+    curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json" -d '{
+      "input": "Your text string goes here",
+      "model": "text-embedding-ada-002"
+    }'

aio/intel/image-gen.yaml

@@ -0,0 +1,20 @@
+name: stablediffusion
+parameters:
+  model: runwayml/stable-diffusion-v1-5
+backend: diffusers
+step: 25
+f16: true
+diffusers:
+  pipeline_type: StableDiffusionPipeline
+  cuda: true
+  enable_parameters: "negative_prompt,num_inference_steps"
+  scheduler_type: "k_dpmpp_2m"
+
+usage: |
+        curl http://localhost:8080/v1/images/generations \
+          -H "Content-Type: application/json" \
+          -d '{
+            "prompt": "<positive prompt>|<negative prompt>",
+            "step": 25,
+            "size": "512x512"
+          }'

aio/intel/speech-to-text.yaml

@@ -0,0 +1,18 @@
+name: whisper-1
+backend: whisper
+parameters:
+  model: ggml-whisper-base.bin
+
+usage: |
+    ## example audio file
+    wget --quiet --show-progress -O gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg
+
+    ## Send the example audio file to the transcriptions endpoint
+    curl http://localhost:8080/v1/audio/transcriptions \
+         -H "Content-Type: multipart/form-data" \
+         -F file="@$PWD/gb1.ogg" -F model="whisper-1"
+
+download_files:
+- filename: "ggml-whisper-base.bin"
+  sha256: "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe"
+  uri: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin"

aio/intel/text-to-speech.yaml

@@ -0,0 +1,15 @@
+name: tts-1
+download_files:
+  - filename: voice-en-us-amy-low.tar.gz
+    uri: https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-en-us-amy-low.tar.gz
+
+parameters:
+  model: en-us-amy-low.onnx
+
+usage: |
+    To test if this model works as expected, you can use the following curl command:
+
+    curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
+      "model":"tts-1",
+      "input": "Hi, this is a test."
+    }'

aio/intel/text-to-text.yaml

@@ -0,0 +1,53 @@
+name: gpt-4
+mmap: false
+f16: false
+parameters:
+  model: huggingface://NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/Hermes-2-Pro-Mistral-7B.Q6_K.gguf
+
+template:
+  chat_message: |
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}
+    {{- if .FunctionCall }}<tool_call>{{end}}
+    {{- if eq .RoleName "tool" }}<tool_result>{{end }}
+    {{- if .Content}}
+    {{.Content}}
+    {{- end }}
+    {{- if .FunctionCall}}{{toJson .FunctionCall}}{{end }}
+    {{- if .FunctionCall }}</tool_call>{{end }}
+    {{- if eq .RoleName "tool" }}</tool_result>{{end }}
+    <|im_end|>
+  # https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF#prompt-format-for-function-calling
+  function: |
+    <|im_start|>system
+    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
+    <tools>
+    {{range .Functions}}
+    {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
+    {{end}}
+    </tools>
+    Use the following pydantic model json schema for each tool call you will make:
+    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}
+    For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+    <tool_call>
+    {'arguments': <args-dict>, 'name': <function-name>}
+    </tool_call>
+    <|im_end|>
+    {{.Input -}}
+    <|im_start|>assistant
+    <tool_call>
+  chat: |
+    {{.Input -}}
+    <|im_start|>assistant
+  completion: |
+    {{.Input}}
+context_size: 4096
+stopwords:
+- <|im_end|>
+- "\n</tool_call>"
+- <dummy32000>
+- "\n\n\n"
+usage: |
+      curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+          "model": "gpt-4",
+          "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}]
+      }'

aio/intel/vision.yaml

@@ -0,0 +1,35 @@
+backend: llama-cpp
+context_size: 4096
+mmap: false
+f16: false
+name: gpt-4-vision-preview
+
+roles:
+  user: "USER:"
+  assistant: "ASSISTANT:"
+  system: "SYSTEM:"
+
+mmproj: llava-v1.6-7b-mmproj-f16.gguf
+parameters:
+  model: llava-v1.6-mistral-7b.Q5_K_M.gguf
+  temperature: 0.2
+  top_k: 40
+  top_p: 0.95
+  seed: -1
+
+template:
+  chat: |
+    A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.
+    {{.Input}}
+    ASSISTANT:
+
+download_files:
+- filename: llava-v1.6-mistral-7b.Q5_K_M.gguf
+  uri: huggingface://cjpais/llava-1.6-mistral-7b-gguf/llava-v1.6-mistral-7b.Q5_K_M.gguf
+- filename: llava-v1.6-7b-mmproj-f16.gguf
+  uri: huggingface://cjpais/llava-1.6-mistral-7b-gguf/mmproj-model-f16.gguf
+
+usage: |
+    curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+        "model": "gpt-4-vision-preview",
+        "messages": [{"role": "user", "content": [{"type":"text", "text": "What is in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}'

backend/python/autogptq/autogptq.py

@@ -5,12 +5,14 @@ import signal
 import sys
 import os
 import time
+import base64
 
 import grpc
 import backend_pb2
 import backend_pb2_grpc
+
 from auto_gptq import AutoGPTQForCausalLM
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from transformers import TextGenerationPipeline
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
@@ -28,9 +30,18 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             if request.Device != "":
                 device = request.Device
 
-            tokenizer = AutoTokenizer.from_pretrained(request.Model, use_fast=request.UseFastTokenizer)
+            # support loading local model files
+            model_path = os.path.join(os.environ.get('MODELS_PATH', './'), request.Model)
+            tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=request.TrustRemoteCode)
 
-            model = AutoGPTQForCausalLM.from_quantized(request.Model,
+            # support model `Qwen/Qwen-VL-Chat-Int4`
+            if "qwen-vl" in request.Model.lower():
+                self.model_name = "Qwen-VL-Chat"
+                model = AutoModelForCausalLM.from_pretrained(model_path, 
+                    trust_remote_code=request.TrustRemoteCode,
+                    device_map="auto").eval()
+            else:
+                model = AutoGPTQForCausalLM.from_quantized(model_path,
                     model_basename=request.ModelBaseName,
                     use_safetensors=True,
                     trust_remote_code=request.TrustRemoteCode,
@@ -55,6 +66,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         if request.TopP != 0.0:
             top_p = request.TopP
 
+        
+        prompt_images = self.recompile_vl_prompt(request)
+        compiled_prompt = prompt_images[0]
+        print(f"Prompt: {compiled_prompt}", file=sys.stderr)
+
         # Implement Predict RPC
         pipeline = TextGenerationPipeline(
             model=self.model, 
@@ -64,10 +80,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             top_p=top_p,
             repetition_penalty=penalty,
             )
-        t = pipeline(request.Prompt)[0]["generated_text"]
-        # Remove prompt from response if present
-        if request.Prompt in t:
-            t = t.replace(request.Prompt, "")
+        t = pipeline(compiled_prompt)[0]["generated_text"]
+        print(f"generated_text: {t}", file=sys.stderr)
+        
+        if compiled_prompt in t:
+            t = t.replace(compiled_prompt, "")
+        # house keeping. Remove the image files from /tmp folder
+        for img_path in prompt_images[1]:
+            try:
+                os.remove(img_path)
+            except Exception as e:
+                print(f"Error removing image file: {img_path}, {e}", file=sys.stderr)
 
         return backend_pb2.Result(message=bytes(t, encoding='utf-8'))
 
@@ -78,6 +101,24 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         # Not implemented yet
         return self.Predict(request, context)
 
+    def recompile_vl_prompt(self, request):
+        prompt = request.Prompt
+        image_paths = []
+
+        if "qwen-vl" in self.model_name.lower():
+            # request.Images is an array which contains base64 encoded images. Iterate the request.Images array, decode and save each image to /tmp folder with a random filename.
+            # Then, save the image file paths to an array "image_paths".
+            # read "request.Prompt", replace "[img-%d]" with the image file paths in the order they appear in "image_paths". Save the new prompt to "prompt".
+            for i, img in enumerate(request.Images):
+                timestamp = str(int(time.time() * 1000))  # Generate timestamp
+                img_path = f"/tmp/vl-{timestamp}.jpg"  # Use timestamp in filename
+                with open(img_path, "wb") as f:
+                    f.write(base64.b64decode(img))
+                image_paths.append(img_path)
+                prompt = prompt.replace(f"[img-{i}]", "<img>" + img_path + "</img>,")
+        else:
+            prompt = request.Prompt
+        return (prompt, image_paths)
 
 def serve(address):
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS))

backend/python/autogptq/autogptq.yml

@@ -1,3 +1,7 @@
+####
+# Attention! This file is abandoned. 
+# Please use the ../common-env/transformers/transformers.yml file to manage dependencies.
+###
 name: autogptq
 channels:
   - defaults
@@ -24,12 +28,12 @@ dependencies:
   - xz=5.4.2=h5eee18b_0
   - zlib=1.2.13=h5eee18b_0
   - pip:
-      - accelerate==0.23.0
+      - accelerate==0.27.0
       - aiohttp==3.8.5
       - aiosignal==1.3.1
       - async-timeout==4.0.3
       - attrs==23.1.0
-      - auto-gptq==0.4.2
+      - auto-gptq==0.7.1
       - certifi==2023.7.22
       - charset-normalizer==3.3.0
       - datasets==2.14.5
@@ -59,6 +63,7 @@ dependencies:
       - nvidia-nccl-cu12==2.18.1
       - nvidia-nvjitlink-cu12==12.2.140
       - nvidia-nvtx-cu12==12.1.105
+      - optimum==1.17.1
       - packaging==23.2
       - pandas==2.1.1
       - peft==0.5.0
@@ -75,9 +80,11 @@ dependencies:
       - six==1.16.0
       - sympy==1.12
       - tokenizers==0.14.0
-      - torch==2.1.0
       - tqdm==4.66.1
+      - torch==2.2.1
+      - torchvision==0.17.1
       - transformers==4.34.0
+      - transformers_stream_generator==0.0.5
       - triton==2.1.0
       - typing-extensions==4.8.0
       - tzdata==2023.3

backend/python/common-env/transformers/install.sh

@@ -25,7 +25,7 @@ if [ -d "/opt/intel" ]; then
     # Intel GPU: If the directory exists, we assume we are using the intel image
     # (no conda env)
     # https://github.com/intel/intel-extension-for-pytorch/issues/538
-    pip install intel-extension-for-transformers datasets sentencepiece tiktoken neural_speed
+    pip install intel-extension-for-transformers datasets sentencepiece tiktoken neural_speed optimum[openvino]
 fi
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then

backend/python/common-env/transformers/transformers-nvidia.yml

@@ -24,10 +24,11 @@ dependencies:
   - xz=5.4.2=h5eee18b_0
   - zlib=1.2.13=h5eee18b_0
   - pip:
-      - accelerate==0.23.0
+      - accelerate==0.27.0
       - aiohttp==3.8.5
       - aiosignal==1.3.1
       - async-timeout==4.0.3
+      - auto-gptq==0.7.1
       - attrs==23.1.0
       - bark==0.1.5
       - bitsandbytes==0.43.0
@@ -69,6 +70,7 @@ dependencies:
       - nvidia-nccl-cu12==2.18.1
       - nvidia-nvjitlink-cu12==12.2.140
       - nvidia-nvtx-cu12==12.1.105
+      - optimum==1.17.1
       - packaging==23.2
       - pandas
       - peft==0.5.0
@@ -88,6 +90,7 @@ dependencies:
       - sympy==1.12
       - tokenizers
       - torch==2.1.2
+      - torchvision==0.16.2
       - torchaudio==2.1.2
       - tqdm==4.66.1
       - triton==2.1.0
@@ -95,7 +98,6 @@ dependencies:
       - tzdata==2023.3
       - urllib3==1.26.17
       - xxhash==3.4.1
-      - auto-gptq==0.6.0
       - yarl==1.9.2
       - soundfile
       - langid
@@ -116,5 +118,6 @@ dependencies:
       - vocos
       - vllm==0.3.2
       - transformers>=4.38.2  # Updated Version
+      - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1  
 prefix: /opt/conda/envs/transformers

backend/python/common-env/transformers/transformers-rocm.yml

@@ -26,7 +26,8 @@ dependencies:
   - pip:
       - --pre
       - --extra-index-url https://download.pytorch.org/whl/nightly/
-      - accelerate==0.23.0
+      - accelerate==0.27.0
+      - auto-gptq==0.7.1
       - aiohttp==3.8.5
       - aiosignal==1.3.1
       - async-timeout==4.0.3
@@ -82,7 +83,6 @@ dependencies:
       - triton==2.1.0
       - typing-extensions==4.8.0
       - tzdata==2023.3
-      - auto-gptq==0.6.0
       - urllib3==1.26.17
       - xxhash==3.4.1
       - yarl==1.9.2
@@ -90,6 +90,7 @@ dependencies:
       - langid
       - wget
       - unidecode
+      - optimum==1.17.1
       - pyopenjtalk-prebuilt
       - pypinyin
       - inflect
@@ -105,5 +106,6 @@ dependencies:
       - vocos
       - vllm==0.3.2
       - transformers>=4.38.2  # Updated Version
+      - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1
 prefix: /opt/conda/envs/transformers

backend/python/common-env/transformers/transformers.yml

@@ -24,15 +24,17 @@ dependencies:
   - xz=5.4.2=h5eee18b_0
   - zlib=1.2.13=h5eee18b_0
   - pip:
-      - accelerate==0.23.0
+      - accelerate==0.27.0
       - aiohttp==3.8.5
       - aiosignal==1.3.1
+      - auto-gptq==0.7.1
       - async-timeout==4.0.3
       - attrs==23.1.0
       - bark==0.1.5
       - boto3==1.28.61
       - botocore==1.31.61
       - certifi==2023.7.22
+      - coloredlogs==15.0.1
       - TTS==0.22.0
       - charset-normalizer==3.3.0
       - datasets==2.14.5
@@ -47,6 +49,7 @@ dependencies:
       - funcy==2.0
       - grpcio==1.59.0
       - huggingface-hub
+      - humanfriendly==10.0
       - idna==3.4
       - jinja2==3.1.2
       - jmespath==1.0.1
@@ -56,6 +59,10 @@ dependencies:
       - multiprocess==0.70.15
       - networkx
       - numpy==1.26.0
+      - onnx==1.15.0
+      - openvino==2024.0.0
+      - openvino-telemetry==2023.2.1
+      - optimum[openvino]==1.17.1
       - packaging==23.2
       - pandas
       - peft==0.5.0
@@ -75,12 +82,12 @@ dependencies:
       - sympy==1.12
       - tokenizers
       - torch==2.1.2
+      - torchvision==0.16.2
       - torchaudio==2.1.2
       - tqdm==4.66.1
       - triton==2.1.0
       - typing-extensions==4.8.0
       - tzdata==2023.3
-      - auto-gptq==0.6.0
       - urllib3==1.26.17
       - xxhash==3.4.1
       - yarl==1.9.2
@@ -103,5 +110,6 @@ dependencies:
       - vocos
       - vllm==0.3.2
       - transformers>=4.38.2  # Updated Version
+      - transformers_stream_generator==0.0.5
       - xformers==0.0.23.post1  
 prefix: /opt/conda/envs/transformers

backend/python/transformers/transformers_server.py

@@ -8,6 +8,8 @@ import argparse
 import signal
 import sys
 import os
+from threading import Thread
+import asyncio
 
 import time
 import backend_pb2
@@ -17,13 +19,12 @@ import grpc
 import torch
 import torch.cuda
 
+
 XPU=os.environ.get("XPU", "0") == "1"
 if XPU:
-    import intel_extension_for_pytorch as ipex
-    from intel_extension_for_transformers.transformers.modeling import AutoModelForCausalLM
-    from transformers import AutoTokenizer, AutoModel, set_seed
+    from transformers import AutoTokenizer, AutoModel, set_seed, TextIteratorStreamer
 else:
-    from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM, set_seed, BitsAndBytesConfig
+    from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM, set_seed, BitsAndBytesConfig, TextIteratorStreamer
 
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
@@ -81,6 +82,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             compute=torch.bfloat16
 
         self.CUDA = request.CUDA
+        self.OV=False
 
         device_map="cpu"
 
@@ -105,23 +107,61 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     bnb_4bit_compute_dtype = None,
                     load_in_8bit=True,                                   
                 )
-                                                   
-    
+                                               
         try:
             if request.Type == "AutoModelForCausalLM":
                 if XPU:
-                    if quantization == "xpu_4bit":
+                    import intel_extension_for_pytorch as ipex
+                    from intel_extension_for_transformers.transformers.modeling import AutoModelForCausalLM
+
+                    device_map="xpu"
+                    compute=torch.float16
+                    if request.Quantization == "xpu_4bit":
                         xpu_4bit = True
-                    self.model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=request.TrustRemoteCode,
-                                              device_map="xpu", load_in_4bit=xpu_4bit)
+                        xpu_8bit = False
+                    elif request.Quantization == "xpu_8bit":
+                        xpu_4bit = False
+                        xpu_8bit = True
+                    else:
+                        xpu_4bit = False
+                        xpu_8bit = False
+                    self.model = AutoModelForCausalLM.from_pretrained(model_name, 
+                                                                      trust_remote_code=request.TrustRemoteCode, 
+                                                                      use_safetensors=True,
+                                                                      device_map=device_map, 
+                                                                      load_in_4bit=xpu_4bit, 
+                                                                      load_in_8bit=xpu_8bit, 
+                                                                      torch_dtype=compute)
                 else:
-                    self.model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=request.TrustRemoteCode, use_safetensors=True, quantization_config=quantization, device_map=device_map, torch_dtype=compute)
+                    self.model = AutoModelForCausalLM.from_pretrained(model_name, 
+                                                                      trust_remote_code=request.TrustRemoteCode, 
+                                                                      use_safetensors=True, 
+                                                                      quantization_config=quantization, 
+                                                                      device_map=device_map, 
+                                                                      torch_dtype=compute)
+            elif request.Type == "OVModelForCausalLM":
+                from optimum.intel.openvino import OVModelForCausalLM
+                from openvino.runtime import Core
+
+                if "GPU" in Core().available_devices:
+                    device_map="GPU"
+                else:
+                    device_map="CPU"
+                self.model = OVModelForCausalLM.from_pretrained(model_name, 
+                                                                compile=True, 
+                                                                device=device_map)
+                self.OV = True
             else:
-                self.model = AutoModel.from_pretrained(model_name, trust_remote_code=request.TrustRemoteCode,  use_safetensors=True,  quantization_config=quantization, device_map=device_map, torch_dtype=compute)
+                self.model = AutoModel.from_pretrained(model_name, 
+                                                       trust_remote_code=request.TrustRemoteCode,  
+                                                       use_safetensors=True,  
+                                                       quantization_config=quantization, 
+                                                       device_map=device_map, 
+                                                       torch_dtype=compute)
             self.tokenizer = AutoTokenizer.from_pretrained(model_name, use_safetensors=True)
             self.XPU = False
 
-            if XPU:
+            if XPU and self.OV == False:
                 self.XPU = True
                 try:
                     print("Optimizing model", model_name, "to XPU.", file=sys.stderr)
@@ -130,6 +170,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     print("Not using XPU:", err, file=sys.stderr)
 
         except Exception as err:
+            print("Error:", err, file=sys.stderr)
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         # Implement your logic here for the LoadModel service
         # Replace this with your desired response
@@ -167,7 +208,72 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         print("Embeddings:", sentence_embeddings, file=sys.stderr)
         return backend_pb2.EmbeddingResult(embeddings=sentence_embeddings[0])
 
-    def Predict(self, request, context):
+    async def _predict(self, request, context, streaming=False): 
+        set_seed(request.Seed)
+        if request.TopP == 0:
+            request.TopP = 0.9
+
+        max_tokens = 200
+        if request.Tokens > 0:
+            max_tokens = request.Tokens
+
+        inputs = self.tokenizer(request.Prompt, return_tensors="pt")
+        if self.CUDA:
+            inputs = inputs.to("cuda")
+        if XPU and self.OV == False:
+            inputs = inputs.to("xpu")
+            streaming = False
+
+        if streaming:
+            streamer=TextIteratorStreamer(self.tokenizer,
+                                        skip_prompt=True,
+                                        skip_special_tokens=True)
+            config=dict(inputs,
+                        max_new_tokens=max_tokens, 
+                        temperature=request.Temperature, 
+                        top_p=request.TopP,
+                        top_k=request.TopK, 
+                        do_sample=True,
+                        attention_mask=inputs["attention_mask"],
+                        eos_token_id=self.tokenizer.eos_token_id,
+                        pad_token_id=self.tokenizer.eos_token_id,
+                        streamer=streamer)
+            thread=Thread(target=self.model.generate, kwargs=config)
+            thread.start()
+            generated_text = ""
+            try:
+                for new_text in streamer:
+                    generated_text += new_text
+                    yield backend_pb2.Reply(message=bytes(new_text, encoding='utf-8'))
+            finally:
+                thread.join()
+        else:
+            if XPU and self.OV == False:
+                outputs = self.model.generate(inputs["input_ids"],
+                                    max_new_tokens=max_tokens, 
+                                    temperature=request.Temperature, 
+                                    top_p=request.TopP,
+                                    top_k=request.TopK, 
+                                    do_sample=True,
+                                    pad_token=self.tokenizer.eos_token_id)
+            else:
+                outputs = self.model.generate(inputs["input_ids"],
+                        max_new_tokens=max_tokens, 
+                        temperature=request.Temperature, 
+                        top_p=request.TopP,
+                        top_k=request.TopK, 
+                        do_sample=True,
+                        attention_mask=inputs["attention_mask"],
+                        eos_token_id=self.tokenizer.eos_token_id,
+                        pad_token_id=self.tokenizer.eos_token_id)
+            generated_text = self.tokenizer.batch_decode(outputs[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)[0]
+
+        if streaming:
+            return
+
+        yield backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
+
+    async def Predict(self, request, context):
         """
         Generates text based on the given prompt and sampling parameters.
 
@@ -178,26 +284,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         Returns:
             backend_pb2.Reply: The predict result.
         """
-        set_seed(request.Seed)
-        if request.TopP == 0:
-            request.TopP = 0.9
+        gen = self._predict(request, context, streaming=False)
+        res = await gen.__anext__()
+        return res
 
-        max_tokens = 200
-        if request.Tokens > 0:
-            max_tokens = request.Tokens
-
-        inputs = self.tokenizer(request.Prompt, return_tensors="pt").input_ids
-        if self.CUDA:
-            inputs = inputs.to("cuda")
-        if XPU:
-            inputs = inputs.to("xpu")
-
-        outputs = self.model.generate(inputs,max_new_tokens=max_tokens, temperature=request.Temperature, top_p=request.TopP, do_sample=True, pad_token_id=self.tokenizer.eos_token_id)
-        generated_text = self.tokenizer.batch_decode(outputs[:, inputs.shape[1]:], skip_special_tokens=True)[0]
-
-        return backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
-
-    def PredictStream(self, request, context):
+    async def PredictStream(self, request, context):
         """
         Generates text based on the given prompt and sampling parameters, and streams the results.
 
@@ -208,31 +299,33 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         Returns:
             backend_pb2.Result: The predict stream result.
         """
-        yield self.Predict(request, context)
+        iterations = self._predict(request, context, streaming=True)
+        try:
+            async for iteration in iterations:
+                yield iteration
+        finally:
+            await iterations.aclose()
 
-
-def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS))
+async def serve(address):
+    # Start asyncio gRPC server
+    server = grpc.aio.server(migration_thread_pool=futures.ThreadPoolExecutor(max_workers=MAX_WORKERS))
+    # Add the servicer to the server
     backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    # Bind the server to the address
     server.add_insecure_port(address)
-    server.start()
+
+    # Gracefully shutdown the server on SIGTERM or SIGINT
+    loop = asyncio.get_event_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(
+            sig, lambda: asyncio.ensure_future(server.stop(5))
+        )
+
+    # Start the server
+    await server.start()
     print("Server started. Listening on: " + address, file=sys.stderr)
-
-    # Define the signal handler function
-    def signal_handler(sig, frame):
-        print("Received termination signal. Shutting down...")
-        server.stop(0)
-        sys.exit(0)
-
-    # Set the signal handlers for SIGINT and SIGTERM
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
+    # Wait for the server to be terminated
+    await server.wait_for_termination()
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run the gRPC server.")
@@ -241,4 +334,4 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
-    serve(args.addr)
+    asyncio.run(serve(args.addr))

core/backend/options.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"math/rand"
 	"os"
 	"path/filepath"
 
@@ -33,12 +34,20 @@ func modelOpts(c config.BackendConfig, so *config.ApplicationConfig, opts []mode
 	return opts
 }
 
+func getSeed(c config.BackendConfig) int32 {
+	seed := int32(*c.Seed)
+	if seed == config.RAND_SEED {
+		seed = rand.Int31()
+	}
+
+	return seed
+}
+
 func gRPCModelOpts(c config.BackendConfig) *pb.ModelOptions {
 	b := 512
 	if c.Batch != 0 {
 		b = c.Batch
 	}
-
 	return &pb.ModelOptions{
 		CUDA:                 c.CUDA || c.Diffusers.CUDA,
 		SchedulerType:        c.Diffusers.SchedulerType,
@@ -54,7 +63,7 @@ func gRPCModelOpts(c config.BackendConfig) *pb.ModelOptions {
 		CLIPSkip:             int32(c.Diffusers.ClipSkip),
 		ControlNet:           c.Diffusers.ControlNet,
 		ContextSize:          int32(*c.ContextSize),
-		Seed:                 int32(*c.Seed),
+		Seed:                 getSeed(c),
 		NBatch:               int32(b),
 		NoMulMatQ:            c.NoMulMatQ,
 		DraftModel:           c.DraftModel,
@@ -129,13 +138,13 @@ func gRPCPredictOpts(c config.BackendConfig, modelPath string) *pb.PredictOption
 		NKeep:               int32(c.Keep),
 		Batch:               int32(c.Batch),
 		IgnoreEOS:           c.IgnoreEOS,
-		Seed:                int32(*c.Seed),
+		Seed:                getSeed(c),
 		FrequencyPenalty:    float32(c.FrequencyPenalty),
 		MLock:               *c.MMlock,
 		MMap:                *c.MMap,
 		MainGPU:             c.MainGPU,
 		TensorSplit:         c.TensorSplit,
-		TailFreeSamplingZ:   float32(c.TFZ),
-		TypicalP:            float32(c.TypicalP),
+		TailFreeSamplingZ:   float32(*c.TFZ),
+		TypicalP:            float32(*c.TypicalP),
 	}
 }

core/config/application_config.go

@@ -15,11 +15,13 @@ type ApplicationConfig struct {
 	ConfigFile                          string
 	ModelPath                           string
 	UploadLimitMB, Threads, ContextSize int
+	DisableWelcomePage                  bool
 	F16                                 bool
 	Debug, DisableMessage               bool
 	ImageDir                            string
 	AudioDir                            string
 	UploadDir                           string
+	ConfigsDir                          string
 	CORS                                bool
 	PreloadJSONModels                   string
 	PreloadModelsFromPath               string
@@ -104,6 +106,10 @@ var EnableWatchDogBusyCheck = func(o *ApplicationConfig) {
 	o.WatchDogBusy = true
 }
 
+var DisableWelcomePage = func(o *ApplicationConfig) {
+	o.DisableWelcomePage = true
+}
+
 func SetWatchDogBusyTimeout(t time.Duration) AppOption {
 	return func(o *ApplicationConfig) {
 		o.WatchDogBusyTimeout = t
@@ -163,7 +169,7 @@ func WithStringGalleries(galls string) AppOption {
 		}
 		var galleries []gallery.Gallery
 		if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
-			log.Error().Msgf("failed loading galleries: %s", err.Error())
+			log.Error().Err(err).Msg("failed loading galleries")
 		}
 		o.Galleries = append(o.Galleries, galleries...)
 	}
@@ -252,6 +258,12 @@ func WithUploadDir(uploadDir string) AppOption {
 	}
 }
 
+func WithConfigsDir(configsDir string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.ConfigsDir = configsDir
+	}
+}
+
 func WithApiKeys(apiKeys []string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.ApiKeys = apiKeys

core/config/backend_config.go

@@ -4,9 +4,9 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"math/rand"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 
@@ -19,6 +19,10 @@ import (
 	"github.com/charmbracelet/glamour"
 )
 
+const (
+	RAND_SEED = -1
+)
+
 type BackendConfig struct {
 	schema.PredictionOptions `yaml:"parameters"`
 	Name                     string `yaml:"name"`
@@ -185,7 +189,12 @@ func (c *BackendConfig) ShouldCallSpecificFunction() bool {
 }
 
 func (c *BackendConfig) FunctionToCall() string {
-	return c.functionCallNameString
+	if c.functionCallNameString != "" &&
+		c.functionCallNameString != "none" && c.functionCallNameString != "auto" {
+		return c.functionCallNameString
+	}
+
+	return c.functionCallString
 }
 
 func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
@@ -196,13 +205,16 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 	threads := lo.threads
 	f16 := lo.f16
 	debug := lo.debug
-	defaultTopP := 0.7
-	defaultTopK := 80
+	// https://github.com/ggerganov/llama.cpp/blob/75cd4c77292034ecec587ecb401366f57338f7c0/common/sampling.h#L22
+	defaultTopP := 0.95
+	defaultTopK := 40
 	defaultTemp := 0.9
 	defaultMaxTokens := 2048
 	defaultMirostat := 2
 	defaultMirostatTAU := 5.0
 	defaultMirostatETA := 0.1
+	defaultTypicalP := 1.0
+	defaultTFZ := 1.0
 
 	// Try to offload all GPU layers (if GPU is found)
 	defaultNGPULayers := 99999999
@@ -212,7 +224,7 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 
 	if cfg.Seed == nil {
 		//  random number generator seed
-		defaultSeed := int(rand.Int31())
+		defaultSeed := RAND_SEED
 		cfg.Seed = &defaultSeed
 	}
 
@@ -220,6 +232,14 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 		cfg.TopK = &defaultTopK
 	}
 
+	if cfg.TypicalP == nil {
+		cfg.TypicalP = &defaultTypicalP
+	}
+
+	if cfg.TFZ == nil {
+		cfg.TFZ = &defaultTFZ
+	}
+
 	if cfg.MMap == nil {
 		// MMap is enabled by default
 		cfg.MMap = &trueV
@@ -450,6 +470,11 @@ func (cl *BackendConfigLoader) GetAllBackendConfigs() []BackendConfig {
 	for _, v := range cl.configs {
 		res = append(res, v)
 	}
+
+	sort.SliceStable(res, func(i, j int) bool {
+		return res[i].Name < res[j].Name
+	})
+
 	return res
 }
 

core/http/api.go

@@ -6,6 +6,9 @@ import (
 	"os"
 	"strings"
 
+	"github.com/go-skynet/LocalAI/pkg/utils"
+	"github.com/gofiber/swagger" // swagger handler
+
 	"github.com/go-skynet/LocalAI/core/http/endpoints/elevenlabs"
 	"github.com/go-skynet/LocalAI/core/http/endpoints/localai"
 	"github.com/go-skynet/LocalAI/core/http/endpoints/openai"
@@ -40,9 +43,23 @@ func readAuthHeader(c *fiber.Ctx) string {
 	return authHeader
 }
 
+// @title LocalAI API
+// @version 2.0.0
+// @description The LocalAI Rest API.
+// @termsOfService
+// @contact.name LocalAI
+// @contact.url https://localai.io
+// @license.name MIT
+// @license.url https://raw.githubusercontent.com/mudler/LocalAI/master/LICENSE
+// @BasePath /
+// @securityDefinitions.apikey BearerAuth
+// @in header
+// @name Authorization
+
 func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) (*fiber.App, error) {
 	// Return errors as JSON responses
 	app := fiber.New(fiber.Config{
+		Views:                 renderEngine(),
 		BodyLimit:             appConfig.UploadLimitMB * 1024 * 1024, // this is the default limit of 4MB
 		DisableStartupMessage: appConfig.DisableMessage,
 		// Override default error handler
@@ -155,8 +172,27 @@ func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *confi
 		}{Version: internal.PrintableVersion()})
 	})
 
-	// Load upload json
-	openai.LoadUploadConfig(appConfig.UploadDir)
+	// Make sure directories exists
+	os.MkdirAll(appConfig.ImageDir, 0755)
+	os.MkdirAll(appConfig.AudioDir, 0755)
+	os.MkdirAll(appConfig.UploadDir, 0755)
+	os.MkdirAll(appConfig.ConfigsDir, 0755)
+	os.MkdirAll(appConfig.ModelPath, 0755)
+
+	// Load config jsons
+	utils.LoadConfig(appConfig.UploadDir, openai.UploadedFilesFile, &openai.UploadedFiles)
+	utils.LoadConfig(appConfig.ConfigsDir, openai.AssistantsConfigFile, &openai.Assistants)
+	utils.LoadConfig(appConfig.ConfigsDir, openai.AssistantsFileConfigFile, &openai.AssistantFiles)
+
+	app.Get("/swagger/*", swagger.HandlerDefault) // default
+
+	welcomeRoute(
+		app,
+		cl,
+		ml,
+		appConfig,
+		auth,
+	)
 
 	modelGalleryEndpointService := localai.CreateModelGalleryEndpointService(appConfig.Galleries, appConfig.ModelPath, galleryService)
 	app.Post("/models/apply", auth, modelGalleryEndpointService.ApplyModelGalleryEndpoint())
@@ -189,6 +225,26 @@ func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *confi
 	app.Post("/v1/edits", auth, openai.EditEndpoint(cl, ml, appConfig))
 	app.Post("/edits", auth, openai.EditEndpoint(cl, ml, appConfig))
 
+	// assistant
+	app.Get("/v1/assistants", auth, openai.ListAssistantsEndpoint(cl, ml, appConfig))
+	app.Get("/assistants", auth, openai.ListAssistantsEndpoint(cl, ml, appConfig))
+	app.Post("/v1/assistants", auth, openai.CreateAssistantEndpoint(cl, ml, appConfig))
+	app.Post("/assistants", auth, openai.CreateAssistantEndpoint(cl, ml, appConfig))
+	app.Delete("/v1/assistants/:assistant_id", auth, openai.DeleteAssistantEndpoint(cl, ml, appConfig))
+	app.Delete("/assistants/:assistant_id", auth, openai.DeleteAssistantEndpoint(cl, ml, appConfig))
+	app.Get("/v1/assistants/:assistant_id", auth, openai.GetAssistantEndpoint(cl, ml, appConfig))
+	app.Get("/assistants/:assistant_id", auth, openai.GetAssistantEndpoint(cl, ml, appConfig))
+	app.Post("/v1/assistants/:assistant_id", auth, openai.ModifyAssistantEndpoint(cl, ml, appConfig))
+	app.Post("/assistants/:assistant_id", auth, openai.ModifyAssistantEndpoint(cl, ml, appConfig))
+	app.Get("/v1/assistants/:assistant_id/files", auth, openai.ListAssistantFilesEndpoint(cl, ml, appConfig))
+	app.Get("/assistants/:assistant_id/files", auth, openai.ListAssistantFilesEndpoint(cl, ml, appConfig))
+	app.Post("/v1/assistants/:assistant_id/files", auth, openai.CreateAssistantFileEndpoint(cl, ml, appConfig))
+	app.Post("/assistants/:assistant_id/files", auth, openai.CreateAssistantFileEndpoint(cl, ml, appConfig))
+	app.Delete("/v1/assistants/:assistant_id/files/:file_id", auth, openai.DeleteAssistantFileEndpoint(cl, ml, appConfig))
+	app.Delete("/assistants/:assistant_id/files/:file_id", auth, openai.DeleteAssistantFileEndpoint(cl, ml, appConfig))
+	app.Get("/v1/assistants/:assistant_id/files/:file_id", auth, openai.GetAssistantFileEndpoint(cl, ml, appConfig))
+	app.Get("/assistants/:assistant_id/files/:file_id", auth, openai.GetAssistantFileEndpoint(cl, ml, appConfig))
+
 	// files
 	app.Post("/v1/files", auth, openai.UploadFilesEndpoint(cl, appConfig))
 	app.Post("/files", auth, openai.UploadFilesEndpoint(cl, appConfig))
@@ -236,14 +292,18 @@ func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *confi
 
 	// Experimental Backend Statistics Module
 	backendMonitor := services.NewBackendMonitor(cl, ml, appConfig) // Split out for now
-	app.Get("/backend/monitor", localai.BackendMonitorEndpoint(backendMonitor))
-	app.Post("/backend/shutdown", localai.BackendShutdownEndpoint(backendMonitor))
+	app.Get("/backend/monitor", auth, localai.BackendMonitorEndpoint(backendMonitor))
+	app.Post("/backend/shutdown", auth, localai.BackendShutdownEndpoint(backendMonitor))
 
 	// models
 	app.Get("/v1/models", auth, openai.ListModelsEndpoint(cl, ml))
 	app.Get("/models", auth, openai.ListModelsEndpoint(cl, ml))
 
-	app.Get("/metrics", localai.LocalAIMetricsEndpoint())
+	app.Get("/metrics", auth, localai.LocalAIMetricsEndpoint())
+
+	// Define a custom 404 handler
+	// Note: keep this at the bottom!
+	app.Use(notFoundHandler)
 
 	return app, nil
 }

core/http/endpoints/elevenlabs/tts.go

@@ -11,6 +11,12 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
+// @Summary Generates audio from the input text.
+// @Param  voice-id	path string	true	"Account ID"
+// @Param request body schema.TTSRequest true "query params"
+// @Success 200 {string} binary	 "Response"
+// @Router /v1/text-to-speech/{voice-id} [post]
 func TTSEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 

core/http/endpoints/localai/tts.go

@@ -11,6 +11,11 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
+// @Summary Generates audio from the input text.
+// @Param request body schema.TTSRequest true "query params"
+// @Success 200 {string} binary	 "Response"
+// @Router /v1/audio/speech [post]
 func TTSEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 

core/http/endpoints/openai/assistant.go

@@ -0,0 +1,521 @@
+package openai
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/go-skynet/LocalAI/core/config"
+	model "github.com/go-skynet/LocalAI/pkg/model"
+	"github.com/go-skynet/LocalAI/pkg/utils"
+	"github.com/gofiber/fiber/v2"
+	"github.com/rs/zerolog/log"
+)
+
+// ToolType defines a type for tool options
+type ToolType string
+
+const (
+	CodeInterpreter ToolType = "code_interpreter"
+	Retrieval       ToolType = "retrieval"
+	Function        ToolType = "function"
+
+	MaxCharacterInstructions  = 32768
+	MaxCharacterDescription   = 512
+	MaxCharacterName          = 256
+	MaxToolsSize              = 128
+	MaxFileIdSize             = 20
+	MaxCharacterMetadataKey   = 64
+	MaxCharacterMetadataValue = 512
+)
+
+type Tool struct {
+	Type ToolType `json:"type"`
+}
+
+// Assistant represents the structure of an assistant object from the OpenAI API.
+type Assistant struct {
+	ID           string            `json:"id"`                     // The unique identifier of the assistant.
+	Object       string            `json:"object"`                 // Object type, which is "assistant".
+	Created      int64             `json:"created"`                // The time at which the assistant was created.
+	Model        string            `json:"model"`                  // The model ID used by the assistant.
+	Name         string            `json:"name,omitempty"`         // The name of the assistant.
+	Description  string            `json:"description,omitempty"`  // The description of the assistant.
+	Instructions string            `json:"instructions,omitempty"` // The system instructions that the assistant uses.
+	Tools        []Tool            `json:"tools,omitempty"`        // A list of tools enabled on the assistant.
+	FileIDs      []string          `json:"file_ids,omitempty"`     // A list of file IDs attached to this assistant.
+	Metadata     map[string]string `json:"metadata,omitempty"`     // Set of key-value pairs attached to the assistant.
+}
+
+var (
+	Assistants           = []Assistant{} // better to return empty array instead of "null"
+	AssistantsConfigFile = "assistants.json"
+)
+
+type AssistantRequest struct {
+	Model        string            `json:"model"`
+	Name         string            `json:"name,omitempty"`
+	Description  string            `json:"description,omitempty"`
+	Instructions string            `json:"instructions,omitempty"`
+	Tools        []Tool            `json:"tools,omitempty"`
+	FileIDs      []string          `json:"file_ids,omitempty"`
+	Metadata     map[string]string `json:"metadata,omitempty"`
+}
+
+// CreateAssistantEndpoint is the OpenAI Assistant API endpoint https://platform.openai.com/docs/api-reference/assistants/createAssistant
+// @Summary Create an assistant with a model and instructions.
+// @Param request body AssistantRequest true "query params"
+// @Success 200 {object} Assistant "Response"
+// @Router /v1/assistants [post]
+func CreateAssistantEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		request := new(AssistantRequest)
+		if err := c.BodyParser(request); err != nil {
+			log.Warn().AnErr("Unable to parse AssistantRequest", err)
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Cannot parse JSON"})
+		}
+
+		if !modelExists(ml, request.Model) {
+			log.Warn().Msgf("Model: %s was not found in list of models.", request.Model)
+			return c.Status(fiber.StatusBadRequest).SendString("Model " + request.Model + " not found")
+		}
+
+		if request.Tools == nil {
+			request.Tools = []Tool{}
+		}
+
+		if request.FileIDs == nil {
+			request.FileIDs = []string{}
+		}
+
+		if request.Metadata == nil {
+			request.Metadata = make(map[string]string)
+		}
+
+		id := "asst_" + strconv.FormatInt(generateRandomID(), 10)
+
+		assistant := Assistant{
+			ID:           id,
+			Object:       "assistant",
+			Created:      time.Now().Unix(),
+			Model:        request.Model,
+			Name:         request.Name,
+			Description:  request.Description,
+			Instructions: request.Instructions,
+			Tools:        request.Tools,
+			FileIDs:      request.FileIDs,
+			Metadata:     request.Metadata,
+		}
+
+		Assistants = append(Assistants, assistant)
+		utils.SaveConfig(appConfig.ConfigsDir, AssistantsConfigFile, Assistants)
+		return c.Status(fiber.StatusOK).JSON(assistant)
+	}
+}
+
+var currentId int64 = 0
+
+func generateRandomID() int64 {
+	atomic.AddInt64(&currentId, 1)
+	return currentId
+}
+
+func ListAssistantsEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		// Because we're altering the existing assistants list we should just duplicate it for now.
+		returnAssistants := Assistants
+		// Parse query parameters
+		limitQuery := c.Query("limit", "20")
+		orderQuery := c.Query("order", "desc")
+		afterQuery := c.Query("after")
+		beforeQuery := c.Query("before")
+
+		// Convert string limit to integer
+		limit, err := strconv.Atoi(limitQuery)
+		if err != nil {
+			return c.Status(http.StatusBadRequest).SendString(fmt.Sprintf("Invalid limit query value: %s", limitQuery))
+		}
+
+		// Sort assistants
+		sort.SliceStable(returnAssistants, func(i, j int) bool {
+			if orderQuery == "asc" {
+				return returnAssistants[i].Created < returnAssistants[j].Created
+			}
+			return returnAssistants[i].Created > returnAssistants[j].Created
+		})
+
+		// After and before cursors
+		if afterQuery != "" {
+			returnAssistants = filterAssistantsAfterID(returnAssistants, afterQuery)
+		}
+		if beforeQuery != "" {
+			returnAssistants = filterAssistantsBeforeID(returnAssistants, beforeQuery)
+		}
+
+		// Apply limit
+		if limit < len(returnAssistants) {
+			returnAssistants = returnAssistants[:limit]
+		}
+
+		return c.JSON(returnAssistants)
+	}
+}
+
+// FilterAssistantsBeforeID filters out those assistants whose ID comes before the given ID
+// We assume that the assistants are already sorted
+func filterAssistantsBeforeID(assistants []Assistant, id string) []Assistant {
+	idInt, err := strconv.Atoi(id)
+	if err != nil {
+		return assistants // Return original slice if invalid id format is provided
+	}
+
+	var filteredAssistants []Assistant
+
+	for _, assistant := range assistants {
+		aid, err := strconv.Atoi(strings.TrimPrefix(assistant.ID, "asst_"))
+		if err != nil {
+			continue // Skip if invalid id in assistant
+		}
+
+		if aid < idInt {
+			filteredAssistants = append(filteredAssistants, assistant)
+		}
+	}
+
+	return filteredAssistants
+}
+
+// FilterAssistantsAfterID filters out those assistants whose ID comes after the given ID
+// We assume that the assistants are already sorted
+func filterAssistantsAfterID(assistants []Assistant, id string) []Assistant {
+	idInt, err := strconv.Atoi(id)
+	if err != nil {
+		return assistants // Return original slice if invalid id format is provided
+	}
+
+	var filteredAssistants []Assistant
+
+	for _, assistant := range assistants {
+		aid, err := strconv.Atoi(strings.TrimPrefix(assistant.ID, "asst_"))
+		if err != nil {
+			continue // Skip if invalid id in assistant
+		}
+
+		if aid > idInt {
+			filteredAssistants = append(filteredAssistants, assistant)
+		}
+	}
+
+	return filteredAssistants
+}
+
+func modelExists(ml *model.ModelLoader, modelName string) (found bool) {
+	found = false
+	models, err := ml.ListModels()
+	if err != nil {
+		return
+	}
+
+	for _, model := range models {
+		if model == modelName {
+			found = true
+			return
+		}
+	}
+	return
+}
+
+func DeleteAssistantEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	type DeleteAssistantResponse struct {
+		ID      string `json:"id"`
+		Object  string `json:"object"`
+		Deleted bool   `json:"deleted"`
+	}
+
+	return func(c *fiber.Ctx) error {
+		assistantID := c.Params("assistant_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id is required")
+		}
+
+		for i, assistant := range Assistants {
+			if assistant.ID == assistantID {
+				Assistants = append(Assistants[:i], Assistants[i+1:]...)
+				utils.SaveConfig(appConfig.ConfigsDir, AssistantsConfigFile, Assistants)
+				return c.Status(fiber.StatusOK).JSON(DeleteAssistantResponse{
+					ID:      assistantID,
+					Object:  "assistant.deleted",
+					Deleted: true,
+				})
+			}
+		}
+
+		log.Warn().Msgf("Unable to find assistant %s for deletion", assistantID)
+		return c.Status(fiber.StatusNotFound).JSON(DeleteAssistantResponse{
+			ID:      assistantID,
+			Object:  "assistant.deleted",
+			Deleted: false,
+		})
+	}
+}
+
+func GetAssistantEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		assistantID := c.Params("assistant_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id is required")
+		}
+
+		for _, assistant := range Assistants {
+			if assistant.ID == assistantID {
+				return c.Status(fiber.StatusOK).JSON(assistant)
+			}
+		}
+
+		return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find assistant with id: %s", assistantID))
+	}
+}
+
+type AssistantFile struct {
+	ID          string `json:"id"`
+	Object      string `json:"object"`
+	CreatedAt   int64  `json:"created_at"`
+	AssistantID string `json:"assistant_id"`
+}
+
+var (
+	AssistantFiles           []AssistantFile
+	AssistantsFileConfigFile = "assistantsFile.json"
+)
+
+type AssistantFileRequest struct {
+	FileID string `json:"file_id"`
+}
+
+type DeleteAssistantFileResponse struct {
+	ID      string `json:"id"`
+	Object  string `json:"object"`
+	Deleted bool   `json:"deleted"`
+}
+
+func CreateAssistantFileEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		request := new(AssistantFileRequest)
+		if err := c.BodyParser(request); err != nil {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Cannot parse JSON"})
+		}
+
+		assistantID := c.Params("assistant_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id is required")
+		}
+
+		for _, assistant := range Assistants {
+			if assistant.ID == assistantID {
+				if len(assistant.FileIDs) > MaxFileIdSize {
+					return c.Status(fiber.StatusBadRequest).SendString(fmt.Sprintf("Max files %d for assistant %s reached.", MaxFileIdSize, assistant.Name))
+				}
+
+				for _, file := range UploadedFiles {
+					if file.ID == request.FileID {
+						assistant.FileIDs = append(assistant.FileIDs, request.FileID)
+						assistantFile := AssistantFile{
+							ID:          file.ID,
+							Object:      "assistant.file",
+							CreatedAt:   time.Now().Unix(),
+							AssistantID: assistant.ID,
+						}
+						AssistantFiles = append(AssistantFiles, assistantFile)
+						utils.SaveConfig(appConfig.ConfigsDir, AssistantsFileConfigFile, AssistantFiles)
+						return c.Status(fiber.StatusOK).JSON(assistantFile)
+					}
+				}
+
+				return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find file_id: %s", request.FileID))
+			}
+		}
+
+		return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find "))
+	}
+}
+
+func ListAssistantFilesEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	type ListAssistantFiles struct {
+		Data   []File
+		Object string
+	}
+
+	return func(c *fiber.Ctx) error {
+		assistantID := c.Params("assistant_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id is required")
+		}
+
+		limitQuery := c.Query("limit", "20")
+		order := c.Query("order", "desc")
+		limit, err := strconv.Atoi(limitQuery)
+		if err != nil || limit < 1 || limit > 100 {
+			limit = 20 // Default to 20 if there's an error or the limit is out of bounds
+		}
+
+		// Sort files by CreatedAt depending on the order query parameter
+		if order == "asc" {
+			sort.Slice(AssistantFiles, func(i, j int) bool {
+				return AssistantFiles[i].CreatedAt < AssistantFiles[j].CreatedAt
+			})
+		} else { // default to "desc"
+			sort.Slice(AssistantFiles, func(i, j int) bool {
+				return AssistantFiles[i].CreatedAt > AssistantFiles[j].CreatedAt
+			})
+		}
+
+		// Limit the number of files returned
+		var limitedFiles []AssistantFile
+		hasMore := false
+		if len(AssistantFiles) > limit {
+			hasMore = true
+			limitedFiles = AssistantFiles[:limit]
+		} else {
+			limitedFiles = AssistantFiles
+		}
+
+		response := map[string]interface{}{
+			"object": "list",
+			"data":   limitedFiles,
+			"first_id": func() string {
+				if len(limitedFiles) > 0 {
+					return limitedFiles[0].ID
+				}
+				return ""
+			}(),
+			"last_id": func() string {
+				if len(limitedFiles) > 0 {
+					return limitedFiles[len(limitedFiles)-1].ID
+				}
+				return ""
+			}(),
+			"has_more": hasMore,
+		}
+
+		return c.Status(fiber.StatusOK).JSON(response)
+	}
+}
+
+func ModifyAssistantEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		request := new(AssistantRequest)
+		if err := c.BodyParser(request); err != nil {
+			log.Warn().AnErr("Unable to parse AssistantRequest", err)
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Cannot parse JSON"})
+		}
+
+		assistantID := c.Params("assistant_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id is required")
+		}
+
+		for i, assistant := range Assistants {
+			if assistant.ID == assistantID {
+				newAssistant := Assistant{
+					ID:           assistantID,
+					Object:       assistant.Object,
+					Created:      assistant.Created,
+					Model:        request.Model,
+					Name:         request.Name,
+					Description:  request.Description,
+					Instructions: request.Instructions,
+					Tools:        request.Tools,
+					FileIDs:      request.FileIDs, // todo: should probably verify fileids exist
+					Metadata:     request.Metadata,
+				}
+
+				// Remove old one and replace with new one
+				Assistants = append(Assistants[:i], Assistants[i+1:]...)
+				Assistants = append(Assistants, newAssistant)
+				utils.SaveConfig(appConfig.ConfigsDir, AssistantsConfigFile, Assistants)
+				return c.Status(fiber.StatusOK).JSON(newAssistant)
+			}
+		}
+		return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find assistant with id: %s", assistantID))
+	}
+}
+
+func DeleteAssistantFileEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		assistantID := c.Params("assistant_id")
+		fileId := c.Params("file_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id and file_id are required")
+		}
+		// First remove file from assistant
+		for i, assistant := range Assistants {
+			if assistant.ID == assistantID {
+				for j, fileId := range assistant.FileIDs {
+					if fileId == fileId {
+						Assistants[i].FileIDs = append(Assistants[i].FileIDs[:j], Assistants[i].FileIDs[j+1:]...)
+
+						// Check if the file exists in the assistantFiles slice
+						for i, assistantFile := range AssistantFiles {
+							if assistantFile.ID == fileId {
+								// Remove the file from the assistantFiles slice
+								AssistantFiles = append(AssistantFiles[:i], AssistantFiles[i+1:]...)
+								utils.SaveConfig(appConfig.ConfigsDir, AssistantsFileConfigFile, AssistantFiles)
+								return c.Status(fiber.StatusOK).JSON(DeleteAssistantFileResponse{
+									ID:      fileId,
+									Object:  "assistant.file.deleted",
+									Deleted: true,
+								})
+							}
+						}
+					}
+				}
+
+				log.Warn().Msgf("Unable to locate file_id: %s in assistants: %s. Continuing to delete assistant file.", fileId, assistantID)
+				for i, assistantFile := range AssistantFiles {
+					if assistantFile.AssistantID == assistantID {
+
+						AssistantFiles = append(AssistantFiles[:i], AssistantFiles[i+1:]...)
+						utils.SaveConfig(appConfig.ConfigsDir, AssistantsFileConfigFile, AssistantFiles)
+
+						return c.Status(fiber.StatusNotFound).JSON(DeleteAssistantFileResponse{
+							ID:      fileId,
+							Object:  "assistant.file.deleted",
+							Deleted: true,
+						})
+					}
+				}
+			}
+		}
+		log.Warn().Msgf("Unable to find assistant: %s", assistantID)
+
+		return c.Status(fiber.StatusNotFound).JSON(DeleteAssistantFileResponse{
+			ID:      fileId,
+			Object:  "assistant.file.deleted",
+			Deleted: false,
+		})
+	}
+}
+
+func GetAssistantFileEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		assistantID := c.Params("assistant_id")
+		fileId := c.Params("file_id")
+		if assistantID == "" {
+			return c.Status(fiber.StatusBadRequest).SendString("parameter assistant_id and file_id are required")
+		}
+
+		for _, assistantFile := range AssistantFiles {
+			if assistantFile.AssistantID == assistantID {
+				if assistantFile.ID == fileId {
+					return c.Status(fiber.StatusOK).JSON(assistantFile)
+				}
+				return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find assistant file with file_id: %s", fileId))
+			}
+		}
+		return c.Status(fiber.StatusNotFound).SendString(fmt.Sprintf("Unable to find assistant file with assistant_id: %s", assistantID))
+	}
+}

core/http/endpoints/openai/assistant_test.go

@@ -0,0 +1,456 @@
+package openai
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/pkg/model"
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+var configsDir string = "/tmp/localai/configs"
+
+type MockLoader struct {
+	models []string
+}
+
+func tearDown() func() {
+	return func() {
+		UploadedFiles = []File{}
+		Assistants = []Assistant{}
+		AssistantFiles = []AssistantFile{}
+		_ = os.Remove(filepath.Join(configsDir, AssistantsConfigFile))
+		_ = os.Remove(filepath.Join(configsDir, AssistantsFileConfigFile))
+	}
+}
+
+func TestAssistantEndpoints(t *testing.T) {
+	// Preparing the mocked objects
+	cl := &config.BackendConfigLoader{}
+	//configsDir := "/tmp/localai/configs"
+	modelPath := "/tmp/localai/model"
+	var ml = model.NewModelLoader(modelPath)
+
+	appConfig := &config.ApplicationConfig{
+		ConfigsDir:    configsDir,
+		UploadLimitMB: 10,
+		UploadDir:     "test_dir",
+		ModelPath:     modelPath,
+	}
+
+	_ = os.RemoveAll(appConfig.ConfigsDir)
+	_ = os.MkdirAll(appConfig.ConfigsDir, 0755)
+	_ = os.MkdirAll(modelPath, 0755)
+	os.Create(filepath.Join(modelPath, "ggml-gpt4all-j"))
+
+	app := fiber.New(fiber.Config{
+		BodyLimit: 20 * 1024 * 1024, // sets the limit to 20MB.
+	})
+
+	// Create a Test Server
+	app.Get("/assistants", ListAssistantsEndpoint(cl, ml, appConfig))
+	app.Post("/assistants", CreateAssistantEndpoint(cl, ml, appConfig))
+	app.Delete("/assistants/:assistant_id", DeleteAssistantEndpoint(cl, ml, appConfig))
+	app.Get("/assistants/:assistant_id", GetAssistantEndpoint(cl, ml, appConfig))
+	app.Post("/assistants/:assistant_id", ModifyAssistantEndpoint(cl, ml, appConfig))
+
+	app.Post("/files", UploadFilesEndpoint(cl, appConfig))
+	app.Get("/assistants/:assistant_id/files", ListAssistantFilesEndpoint(cl, ml, appConfig))
+	app.Post("/assistants/:assistant_id/files", CreateAssistantFileEndpoint(cl, ml, appConfig))
+	app.Delete("/assistants/:assistant_id/files/:file_id", DeleteAssistantFileEndpoint(cl, ml, appConfig))
+	app.Get("/assistants/:assistant_id/files/:file_id", GetAssistantFileEndpoint(cl, ml, appConfig))
+
+	t.Run("CreateAssistantEndpoint", func(t *testing.T) {
+		t.Cleanup(tearDown())
+		ar := &AssistantRequest{
+			Model:        "ggml-gpt4all-j",
+			Name:         "3.5-turbo",
+			Description:  "Test Assistant",
+			Instructions: "You are computer science teacher answering student questions",
+			Tools:        []Tool{{Type: Function}},
+			FileIDs:      nil,
+			Metadata:     nil,
+		}
+
+		resultAssistant, resp, err := createAssistant(app, *ar)
+		assert.NoError(t, err)
+		assert.Equal(t, fiber.StatusOK, resp.StatusCode)
+
+		assert.Equal(t, 1, len(Assistants))
+		//t.Cleanup(cleanupAllAssistants(t, app, []string{resultAssistant.ID}))
+
+		assert.Equal(t, ar.Name, resultAssistant.Name)
+		assert.Equal(t, ar.Model, resultAssistant.Model)
+		assert.Equal(t, ar.Tools, resultAssistant.Tools)
+		assert.Equal(t, ar.Description, resultAssistant.Description)
+		assert.Equal(t, ar.Instructions, resultAssistant.Instructions)
+		assert.Equal(t, ar.FileIDs, resultAssistant.FileIDs)
+		assert.Equal(t, ar.Metadata, resultAssistant.Metadata)
+	})
+
+	t.Run("ListAssistantsEndpoint", func(t *testing.T) {
+		var ids []string
+		var resultAssistant []Assistant
+		for i := 0; i < 4; i++ {
+			ar := &AssistantRequest{
+				Model:        "ggml-gpt4all-j",
+				Name:         fmt.Sprintf("3.5-turbo-%d", i),
+				Description:  fmt.Sprintf("Test Assistant - %d", i),
+				Instructions: fmt.Sprintf("You are computer science teacher answering student questions - %d", i),
+				Tools:        []Tool{{Type: Function}},
+				FileIDs:      []string{"fid-1234"},
+				Metadata:     map[string]string{"meta": "data"},
+			}
+
+			//var err error
+			ra, _, err := createAssistant(app, *ar)
+			// Because we create the assistants so fast all end up with the same created time.
+			time.Sleep(time.Second)
+			resultAssistant = append(resultAssistant, ra)
+			assert.NoError(t, err)
+			ids = append(ids, resultAssistant[i].ID)
+		}
+
+		t.Cleanup(cleanupAllAssistants(t, app, ids))
+
+		tests := []struct {
+			name                 string
+			reqURL               string
+			expectedStatus       int
+			expectedResult       []Assistant
+			expectedStringResult string
+		}{
+			{
+				name:           "Valid Usage - limit only",
+				reqURL:         "/assistants?limit=2",
+				expectedStatus: http.StatusOK,
+				expectedResult: Assistants[:2], // Expecting the first two assistants
+			},
+			{
+				name:           "Valid Usage - order asc",
+				reqURL:         "/assistants?order=asc",
+				expectedStatus: http.StatusOK,
+				expectedResult: Assistants, // Expecting all assistants in ascending order
+			},
+			{
+				name:           "Valid Usage - order desc",
+				reqURL:         "/assistants?order=desc",
+				expectedStatus: http.StatusOK,
+				expectedResult: []Assistant{Assistants[3], Assistants[2], Assistants[1], Assistants[0]}, // Expecting all assistants in descending order
+			},
+			{
+				name:           "Valid Usage - after specific ID",
+				reqURL:         "/assistants?after=2",
+				expectedStatus: http.StatusOK,
+				// Note this is correct because it's put in descending order already
+				expectedResult: Assistants[:3], // Expecting assistants after (excluding) ID 2
+			},
+			{
+				name:           "Valid Usage - before specific ID",
+				reqURL:         "/assistants?before=4",
+				expectedStatus: http.StatusOK,
+				expectedResult: Assistants[2:], // Expecting assistants before (excluding) ID 3.
+			},
+			{
+				name:                 "Invalid Usage - non-integer limit",
+				reqURL:               "/assistants?limit=two",
+				expectedStatus:       http.StatusBadRequest,
+				expectedStringResult: "Invalid limit query value: two",
+			},
+			{
+				name:           "Invalid Usage - non-existing id in after",
+				reqURL:         "/assistants?after=100",
+				expectedStatus: http.StatusOK,
+				expectedResult: []Assistant(nil), // Expecting empty list as there are no IDs above 100
+			},
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				request := httptest.NewRequest(http.MethodGet, tt.reqURL, nil)
+				response, err := app.Test(request)
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedStatus, response.StatusCode)
+				if tt.expectedStatus != fiber.StatusOK {
+					all, _ := ioutil.ReadAll(response.Body)
+					assert.Equal(t, tt.expectedStringResult, string(all))
+				} else {
+					var result []Assistant
+					err = json.NewDecoder(response.Body).Decode(&result)
+					assert.NoError(t, err)
+
+					assert.Equal(t, tt.expectedResult, result)
+				}
+			})
+		}
+	})
+
+	t.Run("DeleteAssistantEndpoint", func(t *testing.T) {
+		ar := &AssistantRequest{
+			Model:        "ggml-gpt4all-j",
+			Name:         "3.5-turbo",
+			Description:  "Test Assistant",
+			Instructions: "You are computer science teacher answering student questions",
+			Tools:        []Tool{{Type: Function}},
+			FileIDs:      nil,
+			Metadata:     nil,
+		}
+
+		resultAssistant, _, err := createAssistant(app, *ar)
+		assert.NoError(t, err)
+
+		target := fmt.Sprintf("/assistants/%s", resultAssistant.ID)
+		deleteReq := httptest.NewRequest(http.MethodDelete, target, nil)
+		_, err = app.Test(deleteReq)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(Assistants))
+	})
+
+	t.Run("GetAssistantEndpoint", func(t *testing.T) {
+		ar := &AssistantRequest{
+			Model:        "ggml-gpt4all-j",
+			Name:         "3.5-turbo",
+			Description:  "Test Assistant",
+			Instructions: "You are computer science teacher answering student questions",
+			Tools:        []Tool{{Type: Function}},
+			FileIDs:      nil,
+			Metadata:     nil,
+		}
+
+		resultAssistant, _, err := createAssistant(app, *ar)
+		assert.NoError(t, err)
+		t.Cleanup(cleanupAllAssistants(t, app, []string{resultAssistant.ID}))
+
+		target := fmt.Sprintf("/assistants/%s", resultAssistant.ID)
+		request := httptest.NewRequest(http.MethodGet, target, nil)
+		response, err := app.Test(request)
+		assert.NoError(t, err)
+
+		var getAssistant Assistant
+		err = json.NewDecoder(response.Body).Decode(&getAssistant)
+		assert.NoError(t, err)
+
+		assert.Equal(t, resultAssistant.ID, getAssistant.ID)
+	})
+
+	t.Run("ModifyAssistantEndpoint", func(t *testing.T) {
+		ar := &AssistantRequest{
+			Model:        "ggml-gpt4all-j",
+			Name:         "3.5-turbo",
+			Description:  "Test Assistant",
+			Instructions: "You are computer science teacher answering student questions",
+			Tools:        []Tool{{Type: Function}},
+			FileIDs:      nil,
+			Metadata:     nil,
+		}
+
+		resultAssistant, _, err := createAssistant(app, *ar)
+		assert.NoError(t, err)
+
+		modifiedAr := &AssistantRequest{
+			Model:        "ggml-gpt4all-j",
+			Name:         "4.0-turbo",
+			Description:  "Modified Test Assistant",
+			Instructions: "You are math teacher answering student questions",
+			Tools:        []Tool{{Type: CodeInterpreter}},
+			FileIDs:      nil,
+			Metadata:     nil,
+		}
+
+		modifiedArJson, err := json.Marshal(modifiedAr)
+		assert.NoError(t, err)
+
+		target := fmt.Sprintf("/assistants/%s", resultAssistant.ID)
+		request := httptest.NewRequest(http.MethodPost, target, strings.NewReader(string(modifiedArJson)))
+		request.Header.Set(fiber.HeaderContentType, "application/json")
+
+		modifyResponse, err := app.Test(request)
+		assert.NoError(t, err)
+		var getAssistant Assistant
+		err = json.NewDecoder(modifyResponse.Body).Decode(&getAssistant)
+
+		t.Cleanup(cleanupAllAssistants(t, app, []string{getAssistant.ID}))
+
+		assert.Equal(t, resultAssistant.ID, getAssistant.ID) // IDs should match even if contents change
+		assert.Equal(t, modifiedAr.Tools, getAssistant.Tools)
+		assert.Equal(t, modifiedAr.Name, getAssistant.Name)
+		assert.Equal(t, modifiedAr.Instructions, getAssistant.Instructions)
+		assert.Equal(t, modifiedAr.Description, getAssistant.Description)
+	})
+
+	t.Run("CreateAssistantFileEndpoint", func(t *testing.T) {
+		t.Cleanup(tearDown())
+		file, assistant, err := createFileAndAssistant(t, app, appConfig)
+		assert.NoError(t, err)
+
+		afr := AssistantFileRequest{FileID: file.ID}
+		af, _, err := createAssistantFile(app, afr, assistant.ID)
+
+		assert.NoError(t, err)
+		assert.Equal(t, assistant.ID, af.AssistantID)
+	})
+	t.Run("ListAssistantFilesEndpoint", func(t *testing.T) {
+		t.Cleanup(tearDown())
+		file, assistant, err := createFileAndAssistant(t, app, appConfig)
+		assert.NoError(t, err)
+
+		afr := AssistantFileRequest{FileID: file.ID}
+		af, _, err := createAssistantFile(app, afr, assistant.ID)
+		assert.NoError(t, err)
+
+		assert.Equal(t, assistant.ID, af.AssistantID)
+	})
+	t.Run("GetAssistantFileEndpoint", func(t *testing.T) {
+		t.Cleanup(tearDown())
+		file, assistant, err := createFileAndAssistant(t, app, appConfig)
+		assert.NoError(t, err)
+
+		afr := AssistantFileRequest{FileID: file.ID}
+		af, _, err := createAssistantFile(app, afr, assistant.ID)
+		assert.NoError(t, err)
+		t.Cleanup(cleanupAssistantFile(t, app, af.ID, af.AssistantID))
+
+		target := fmt.Sprintf("/assistants/%s/files/%s", assistant.ID, file.ID)
+		request := httptest.NewRequest(http.MethodGet, target, nil)
+		response, err := app.Test(request)
+		assert.NoError(t, err)
+
+		var assistantFile AssistantFile
+		err = json.NewDecoder(response.Body).Decode(&assistantFile)
+		assert.NoError(t, err)
+
+		assert.Equal(t, af.ID, assistantFile.ID)
+		assert.Equal(t, af.AssistantID, assistantFile.AssistantID)
+	})
+	t.Run("DeleteAssistantFileEndpoint", func(t *testing.T) {
+		t.Cleanup(tearDown())
+		file, assistant, err := createFileAndAssistant(t, app, appConfig)
+		assert.NoError(t, err)
+
+		afr := AssistantFileRequest{FileID: file.ID}
+		af, _, err := createAssistantFile(app, afr, assistant.ID)
+		assert.NoError(t, err)
+
+		cleanupAssistantFile(t, app, af.ID, af.AssistantID)()
+
+		assert.Empty(t, AssistantFiles)
+	})
+
+}
+
+func createFileAndAssistant(t *testing.T, app *fiber.App, o *config.ApplicationConfig) (File, Assistant, error) {
+	ar := &AssistantRequest{
+		Model:        "ggml-gpt4all-j",
+		Name:         "3.5-turbo",
+		Description:  "Test Assistant",
+		Instructions: "You are computer science teacher answering student questions",
+		Tools:        []Tool{{Type: Function}},
+		FileIDs:      nil,
+		Metadata:     nil,
+	}
+
+	assistant, _, err := createAssistant(app, *ar)
+	if err != nil {
+		return File{}, Assistant{}, err
+	}
+	t.Cleanup(cleanupAllAssistants(t, app, []string{assistant.ID}))
+
+	file := CallFilesUploadEndpointWithCleanup(t, app, "test.txt", "file", "fine-tune", 5, o)
+	t.Cleanup(func() {
+		_, err := CallFilesDeleteEndpoint(t, app, file.ID)
+		assert.NoError(t, err)
+	})
+	return file, assistant, nil
+}
+
+func createAssistantFile(app *fiber.App, afr AssistantFileRequest, assistantId string) (AssistantFile, *http.Response, error) {
+	afrJson, err := json.Marshal(afr)
+	if err != nil {
+		return AssistantFile{}, nil, err
+	}
+
+	target := fmt.Sprintf("/assistants/%s/files", assistantId)
+	request := httptest.NewRequest(http.MethodPost, target, strings.NewReader(string(afrJson)))
+	request.Header.Set(fiber.HeaderContentType, "application/json")
+	request.Header.Set("OpenAi-Beta", "assistants=v1")
+
+	resp, err := app.Test(request)
+	if err != nil {
+		return AssistantFile{}, resp, err
+	}
+
+	var assistantFile AssistantFile
+	all, err := ioutil.ReadAll(resp.Body)
+	err = json.NewDecoder(strings.NewReader(string(all))).Decode(&assistantFile)
+	if err != nil {
+		return AssistantFile{}, resp, err
+	}
+
+	return assistantFile, resp, nil
+}
+
+func createAssistant(app *fiber.App, ar AssistantRequest) (Assistant, *http.Response, error) {
+	assistant, err := json.Marshal(ar)
+	if err != nil {
+		return Assistant{}, nil, err
+	}
+
+	request := httptest.NewRequest(http.MethodPost, "/assistants", strings.NewReader(string(assistant)))
+	request.Header.Set(fiber.HeaderContentType, "application/json")
+	request.Header.Set("OpenAi-Beta", "assistants=v1")
+
+	resp, err := app.Test(request)
+	if err != nil {
+		return Assistant{}, resp, err
+	}
+
+	bodyString, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return Assistant{}, resp, err
+	}
+
+	var resultAssistant Assistant
+	err = json.NewDecoder(strings.NewReader(string(bodyString))).Decode(&resultAssistant)
+
+	return resultAssistant, resp, nil
+}
+
+func cleanupAllAssistants(t *testing.T, app *fiber.App, ids []string) func() {
+	return func() {
+		for _, assistant := range ids {
+			target := fmt.Sprintf("/assistants/%s", assistant)
+			deleteReq := httptest.NewRequest(http.MethodDelete, target, nil)
+			_, err := app.Test(deleteReq)
+			if err != nil {
+				t.Fatalf("Failed to delete assistant %s: %v", assistant, err)
+			}
+		}
+	}
+}
+
+func cleanupAssistantFile(t *testing.T, app *fiber.App, fileId, assistantId string) func() {
+	return func() {
+		target := fmt.Sprintf("/assistants/%s/files/%s", assistantId, fileId)
+		request := httptest.NewRequest(http.MethodDelete, target, nil)
+		request.Header.Set(fiber.HeaderContentType, "application/json")
+		request.Header.Set("OpenAi-Beta", "assistants=v1")
+
+		resp, err := app.Test(request)
+		assert.NoError(t, err)
+
+		var dafr DeleteAssistantFileResponse
+		err = json.NewDecoder(resp.Body).Decode(&dafr)
+		assert.NoError(t, err)
+		assert.True(t, dafr.Deleted)
+	}
+}

core/http/endpoints/openai/chat.go

@@ -20,6 +20,11 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
+// ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create
+// @Summary Generate a chat completions for a given prompt and model.
+// @Param request body schema.OpenAIRequest true "query params"
+// @Success 200 {object} schema.OpenAIResponse "Response"
+// @Router /v1/chat/completions [post]
 func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startupOptions *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	emptyMessage := ""
 	id := uuid.New().String()
@@ -79,7 +84,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 
 			result, err := handleQuestion(config, req, ml, startupOptions, results[0].arguments, prompt)
 			if err != nil {
-				log.Error().Msgf("error handling question: %s", err.Error())
+				log.Error().Err(err).Msg("error handling question")
 				return
 			}
 
@@ -180,6 +185,8 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 			input.Grammar = grammar.JSONBNF
 		}
 
+		config.Grammar = input.Grammar
+
 		// process functions if we have any defined or if we have a function call string
 		if len(input.Functions) > 0 && config.ShouldUseFunctions() {
 			log.Debug().Msgf("Response needs to process functions")
@@ -231,7 +238,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 
 			// if function call, we might want to customize the role so we can display better that the "assistant called a json action"
 			// if an "assistant_function_call" role is defined, we use it, otherwise we use the role that is passed by in the request
-			if i.FunctionCall != nil && i.Role == "assistant" {
+			if (i.FunctionCall != nil || i.ToolCalls != nil) && i.Role == "assistant" {
 				roleFn := "assistant_function_call"
 				r := config.Roles[roleFn]
 				if r != "" {
@@ -241,6 +248,11 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 			r := config.Roles[role]
 			contentExists := i.Content != nil && i.StringContent != ""
 
+			fcall := i.FunctionCall
+			if len(i.ToolCalls) > 0 {
+				fcall = i.ToolCalls
+			}
+
 			// First attempt to populate content via a chat message specific template
 			if config.TemplateConfig.ChatMessage != "" {
 				chatMessageData := model.ChatMessageTemplateData{
@@ -248,7 +260,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 					Role:         r,
 					RoleName:     role,
 					Content:      i.StringContent,
-					FunctionCall: i.FunctionCall,
+					FunctionCall: fcall,
 					FunctionName: i.Name,
 					LastMessage:  messageIndex == (len(input.Messages) - 1),
 					Function:     config.Grammar != "" && (messageIndex == (len(input.Messages) - 1)),
@@ -256,7 +268,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 				}
 				templatedChatMessage, err := ml.EvaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData)
 				if err != nil {
-					log.Error().Msgf("error processing message %+v using template \"%s\": %v. Skipping!", chatMessageData, config.TemplateConfig.ChatMessage, err)
+					log.Error().Err(err).Interface("message", chatMessageData).Str("template", config.TemplateConfig.ChatMessage).Msg("error processing message with template, skipping")
 				} else {
 					if templatedChatMessage == "" {
 						log.Warn().Msgf("template \"%s\" produced blank output for %+v. Skipping!", config.TemplateConfig.ChatMessage, chatMessageData)
@@ -266,35 +278,49 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 					content = templatedChatMessage
 				}
 			}
+
+			marshalAnyRole := func(f any) {
+				j, err := json.Marshal(f)
+				if err == nil {
+					if contentExists {
+						content += "\n" + fmt.Sprint(r, " ", string(j))
+					} else {
+						content = fmt.Sprint(r, " ", string(j))
+					}
+				}
+			}
+			marshalAny := func(f any) {
+				j, err := json.Marshal(f)
+				if err == nil {
+					if contentExists {
+						content += "\n" + string(j)
+					} else {
+						content = string(j)
+					}
+				}
+			}
 			// If this model doesn't have such a template, or if that template fails to return a value, template at the message level.
 			if content == "" {
 				if r != "" {
 					if contentExists {
 						content = fmt.Sprint(r, i.StringContent)
 					}
+
 					if i.FunctionCall != nil {
-						j, err := json.Marshal(i.FunctionCall)
-						if err == nil {
-							if contentExists {
-								content += "\n" + fmt.Sprint(r, " ", string(j))
-							} else {
-								content = fmt.Sprint(r, " ", string(j))
-							}
-						}
+						marshalAnyRole(i.FunctionCall)
+					}
+					if i.ToolCalls != nil {
+						marshalAnyRole(i.ToolCalls)
 					}
 				} else {
 					if contentExists {
 						content = fmt.Sprint(i.StringContent)
 					}
 					if i.FunctionCall != nil {
-						j, err := json.Marshal(i.FunctionCall)
-						if err == nil {
-							if contentExists {
-								content += "\n" + string(j)
-							} else {
-								content = string(j)
-							}
-						}
+						marshalAny(i.FunctionCall)
+					}
+					if i.ToolCalls != nil {
+						marshalAny(i.ToolCalls)
 					}
 				}
 				// Special Handling: System. We care if it was printed at all, not the r branch, so check seperately
@@ -429,7 +455,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 				case noActionsToRun:
 					result, err := handleQuestion(config, input, ml, startupOptions, results[0].arguments, predInput)
 					if err != nil {
-						log.Error().Msgf("error handling question: %s", err.Error())
+						log.Error().Err(err).Msg("error handling question")
 						return
 					}
 					*c = append(*c, schema.Choice{
@@ -539,13 +565,13 @@ func handleQuestion(config *config.BackendConfig, input *schema.OpenAIRequest, m
 
 	predFunc, err := backend.ModelInference(input.Context, prompt, images, ml, *config, o, nil)
 	if err != nil {
-		log.Error().Msgf("inference error: %s", err.Error())
+		log.Error().Err(err).Msg("model inference failed")
 		return "", err
 	}
 
 	prediction, err := predFunc()
 	if err != nil {
-		log.Error().Msgf("inference error: %s", err.Error())
+		log.Error().Err(err).Msg("prediction failed")
 		return "", err
 	}
 	return backend.Finetune(*config, prompt, prediction.Response), nil

core/http/endpoints/openai/completion.go

@@ -20,7 +20,11 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-// https://platform.openai.com/docs/api-reference/completions
+// CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
+// @Summary Generate completions for a given prompt and model.
+// @Param request body schema.OpenAIRequest true "query params"
+// @Success 200 {object} schema.OpenAIResponse "Response"
+// @Router /v1/completions [post]
 func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	id := uuid.New().String()
 	created := int(time.Now().Unix())
@@ -69,6 +73,8 @@ func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, a
 			input.Grammar = grammar.JSONBNF
 		}
 
+		config.Grammar = input.Grammar
+
 		log.Debug().Msgf("Parameter Config: %+v", config)
 
 		if input.Stream {

core/http/endpoints/openai/embeddings.go

@@ -16,7 +16,11 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-// https://platform.openai.com/docs/api-reference/embeddings
+// EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings
+// @Summary Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms.
+// @Param request body schema.OpenAIRequest true "query params"
+// @Success 200 {object} schema.OpenAIResponse "Response"
+// @Router /v1/embeddings [post]
 func EmbeddingsEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 		model, input, err := readRequest(c, ml, appConfig, true)

core/http/endpoints/openai/files.go

@@ -1,23 +1,22 @@
 package openai
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync/atomic"
 	"time"
 
 	"github.com/go-skynet/LocalAI/core/config"
 
 	"github.com/go-skynet/LocalAI/pkg/utils"
 	"github.com/gofiber/fiber/v2"
-	"github.com/rs/zerolog/log"
 )
 
-var uploadedFiles []File
+var UploadedFiles []File
 
-const uploadedFilesFile = "uploadedFiles.json"
+const UploadedFilesFile = "uploadedFiles.json"
 
 // File represents the structure of a file object from the OpenAI API.
 type File struct {
@@ -29,38 +28,6 @@ type File struct {
 	Purpose   string    `json:"purpose"`    // The purpose of the file (e.g., "fine-tune", "classifications", etc.)
 }
 
-func saveUploadConfig(uploadDir string) {
-	file, err := json.MarshalIndent(uploadedFiles, "", " ")
-	if err != nil {
-		log.Error().Msgf("Failed to JSON marshal the uploadedFiles: %s", err)
-	}
-
-	err = os.WriteFile(filepath.Join(uploadDir, uploadedFilesFile), file, 0644)
-	if err != nil {
-		log.Error().Msgf("Failed to save uploadedFiles to file: %s", err)
-	}
-}
-
-func LoadUploadConfig(uploadPath string) {
-	uploadFilePath := filepath.Join(uploadPath, uploadedFilesFile)
-
-	_, err := os.Stat(uploadFilePath)
-	if os.IsNotExist(err) {
-		log.Debug().Msgf("No uploadedFiles file found at %s", uploadFilePath)
-		return
-	}
-
-	file, err := os.ReadFile(uploadFilePath)
-	if err != nil {
-		log.Error().Msgf("Failed to read file: %s", err)
-	} else {
-		err = json.Unmarshal(file, &uploadedFiles)
-		if err != nil {
-			log.Error().Msgf("Failed to JSON unmarshal the file into uploadedFiles: %s", err)
-		}
-	}
-}
-
 // UploadFilesEndpoint https://platform.openai.com/docs/api-reference/files/create
 func UploadFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
@@ -95,7 +62,7 @@ func UploadFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.Appli
 		}
 
 		f := File{
-			ID:        fmt.Sprintf("file-%d", time.Now().Unix()),
+			ID:        fmt.Sprintf("file-%d", getNextFileId()),
 			Object:    "file",
 			Bytes:     int(file.Size),
 			CreatedAt: time.Now(),
@@ -103,12 +70,19 @@ func UploadFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.Appli
 			Purpose:   purpose,
 		}
 
-		uploadedFiles = append(uploadedFiles, f)
-		saveUploadConfig(appConfig.UploadDir)
+		UploadedFiles = append(UploadedFiles, f)
+		utils.SaveConfig(appConfig.UploadDir, UploadedFilesFile, UploadedFiles)
 		return c.Status(fiber.StatusOK).JSON(f)
 	}
 }
 
+var currentFileId int64 = 0
+
+func getNextFileId() int64 {
+	atomic.AddInt64(&currentId, 1)
+	return currentId
+}
+
 // ListFilesEndpoint https://platform.openai.com/docs/api-reference/files/list
 func ListFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	type ListFiles struct {
@@ -121,9 +95,9 @@ func ListFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.Applica
 
 		purpose := c.Query("purpose")
 		if purpose == "" {
-			listFiles.Data = uploadedFiles
+			listFiles.Data = UploadedFiles
 		} else {
-			for _, f := range uploadedFiles {
+			for _, f := range UploadedFiles {
 				if purpose == f.Purpose {
 					listFiles.Data = append(listFiles.Data, f)
 				}
@@ -140,7 +114,7 @@ func getFileFromRequest(c *fiber.Ctx) (*File, error) {
 		return nil, fmt.Errorf("file_id parameter is required")
 	}
 
-	for _, f := range uploadedFiles {
+	for _, f := range UploadedFiles {
 		if id == f.ID {
 			return &f, nil
 		}
@@ -184,14 +158,14 @@ func DeleteFilesEndpoint(cm *config.BackendConfigLoader, appConfig *config.Appli
 		}
 
 		// Remove upload from list
-		for i, f := range uploadedFiles {
+		for i, f := range UploadedFiles {
 			if f.ID == file.ID {
-				uploadedFiles = append(uploadedFiles[:i], uploadedFiles[i+1:]...)
+				UploadedFiles = append(UploadedFiles[:i], UploadedFiles[i+1:]...)
 				break
 			}
 		}
 
-		saveUploadConfig(appConfig.UploadDir)
+		utils.SaveConfig(appConfig.UploadDir, UploadedFilesFile, UploadedFiles)
 		return c.JSON(DeleteStatus{
 			Id:      file.ID,
 			Object:  "file",

core/http/endpoints/openai/files_test.go

@@ -11,6 +11,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/go-skynet/LocalAI/core/config"
 
 	utils2 "github.com/go-skynet/LocalAI/pkg/utils"
@@ -73,6 +75,7 @@ func TestUploadFileExceedSizeLimit(t *testing.T) {
 	app.Get("/files/:file_id/content", GetFilesContentsEndpoint(loader, option))
 
 	t.Run("UploadFilesEndpoint file size exceeds limit", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		resp, err := CallFilesUploadEndpoint(t, app, "foo.txt", "file", "fine-tune", 11, option)
 		assert.NoError(t, err)
 
@@ -80,46 +83,54 @@ func TestUploadFileExceedSizeLimit(t *testing.T) {
 		assert.Contains(t, bodyToString(resp, t), "exceeds upload limit")
 	})
 	t.Run("UploadFilesEndpoint purpose not defined", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		resp, _ := CallFilesUploadEndpoint(t, app, "foo.txt", "file", "", 5, option)
 
 		assert.Equal(t, fiber.StatusBadRequest, resp.StatusCode)
 		assert.Contains(t, bodyToString(resp, t), "Purpose is not defined")
 	})
 	t.Run("UploadFilesEndpoint file already exists", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		f1 := CallFilesUploadEndpointWithCleanup(t, app, "foo.txt", "file", "fine-tune", 5, option)
 
 		resp, err := CallFilesUploadEndpoint(t, app, "foo.txt", "file", "fine-tune", 5, option)
 		fmt.Println(f1)
-		fmt.Printf("ERror: %v", err)
+		fmt.Printf("ERror: %v\n", err)
+		fmt.Printf("resp: %+v\n", resp)
 
 		assert.Equal(t, fiber.StatusBadRequest, resp.StatusCode)
 		assert.Contains(t, bodyToString(resp, t), "File already exists")
 	})
 	t.Run("UploadFilesEndpoint file uploaded successfully", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		file := CallFilesUploadEndpointWithCleanup(t, app, "test.txt", "file", "fine-tune", 5, option)
 
 		// Check if file exists in the disk
-		filePath := filepath.Join(option.UploadDir, utils2.SanitizeFileName("test.txt"))
+		testName := strings.Split(t.Name(), "/")[1]
+		fileName := testName + "-test.txt"
+		filePath := filepath.Join(option.UploadDir, utils2.SanitizeFileName(fileName))
 		_, err := os.Stat(filePath)
 
 		assert.False(t, os.IsNotExist(err))
 		assert.Equal(t, file.Bytes, 5242880)
 		assert.NotEmpty(t, file.CreatedAt)
-		assert.Equal(t, file.Filename, "test.txt")
+		assert.Equal(t, file.Filename, fileName)
 		assert.Equal(t, file.Purpose, "fine-tune")
 	})
 	t.Run("ListFilesEndpoint without purpose parameter", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		resp, err := CallListFilesEndpoint(t, app, "")
 		assert.NoError(t, err)
 
 		assert.Equal(t, 200, resp.StatusCode)
 
 		listFiles := responseToListFile(t, resp)
-		if len(listFiles.Data) != len(uploadedFiles) {
-			t.Errorf("Expected %v files, got %v files", len(uploadedFiles), len(listFiles.Data))
+		if len(listFiles.Data) != len(UploadedFiles) {
+			t.Errorf("Expected %v files, got %v files", len(UploadedFiles), len(listFiles.Data))
 		}
 	})
 	t.Run("ListFilesEndpoint with valid purpose parameter", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		_ = CallFilesUploadEndpointWithCleanup(t, app, "test.txt", "file", "fine-tune", 5, option)
 
 		resp, err := CallListFilesEndpoint(t, app, "fine-tune")
@@ -131,6 +142,7 @@ func TestUploadFileExceedSizeLimit(t *testing.T) {
 		}
 	})
 	t.Run("ListFilesEndpoint with invalid query parameter", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		resp, err := CallListFilesEndpoint(t, app, "not-so-fine-tune")
 		assert.NoError(t, err)
 		assert.Equal(t, 200, resp.StatusCode)
@@ -142,6 +154,7 @@ func TestUploadFileExceedSizeLimit(t *testing.T) {
 		}
 	})
 	t.Run("GetFilesContentsEndpoint get file content", func(t *testing.T) {
+		t.Cleanup(tearDown())
 		req := httptest.NewRequest("GET", "/files", nil)
 		resp, _ := app.Test(req)
 		assert.Equal(t, 200, resp.StatusCode)
@@ -175,8 +188,10 @@ func CallFilesContentEndpoint(t *testing.T, app *fiber.App, fileId string) (*htt
 }
 
 func CallFilesUploadEndpoint(t *testing.T, app *fiber.App, fileName, tag, purpose string, fileSize int, appConfig *config.ApplicationConfig) (*http.Response, error) {
+	testName := strings.Split(t.Name(), "/")[1]
+
 	// Create a file that exceeds the limit
-	file := createTestFile(t, fileName, fileSize, appConfig)
+	file := createTestFile(t, testName+"-"+fileName, fileSize, appConfig)
 
 	// Creating a new HTTP Request
 	body, writer := newMultipartFile(file.Name(), tag, purpose)
@@ -188,7 +203,8 @@ func CallFilesUploadEndpoint(t *testing.T, app *fiber.App, fileName, tag, purpos
 
 func CallFilesUploadEndpointWithCleanup(t *testing.T, app *fiber.App, fileName, tag, purpose string, fileSize int, appConfig *config.ApplicationConfig) File {
 	// Create a file that exceeds the limit
-	file := createTestFile(t, fileName, fileSize, appConfig)
+	testName := strings.Split(t.Name(), "/")[1]
+	file := createTestFile(t, testName+"-"+fileName, fileSize, appConfig)
 
 	// Creating a new HTTP Request
 	body, writer := newMultipartFile(file.Name(), tag, purpose)
@@ -199,11 +215,12 @@ func CallFilesUploadEndpointWithCleanup(t *testing.T, app *fiber.App, fileName,
 	assert.NoError(t, err)
 	f := responseToFile(t, resp)
 
-	id := f.ID
-	t.Cleanup(func() {
-		_, err := CallFilesDeleteEndpoint(t, app, id)
-		assert.NoError(t, err)
-	})
+	//id := f.ID
+	//t.Cleanup(func() {
+	//	_, err := CallFilesDeleteEndpoint(t, app, id)
+	//	assert.NoError(t, err)
+	//	assert.Empty(t, UploadedFiles)
+	//})
 
 	return f
 
@@ -240,7 +257,8 @@ func createTestFile(t *testing.T, name string, sizeMB int, option *config.Applic
 		t.Fatalf("Error MKDIR: %v", err)
 	}
 
-	file, _ := os.Create(name)
+	file, err := os.Create(name)
+	assert.NoError(t, err)
 	file.WriteString(strings.Repeat("a", sizeMB*1024*1024)) // sizeMB MB File
 
 	t.Cleanup(func() {
@@ -280,7 +298,7 @@ func responseToListFile(t *testing.T, resp *http.Response) ListFiles {
 
 	err := json.NewDecoder(strings.NewReader(responseToString)).Decode(&listFiles)
 	if err != nil {
-		fmt.Printf("Failed to decode response: %s", err)
+		log.Error().Err(err).Msg("failed to decode response")
 	}
 
 	return listFiles

core/http/endpoints/openai/image.go

@@ -44,7 +44,7 @@ func downloadFile(url string) (string, error) {
 	return out.Name(), err
 }
 
-// https://platform.openai.com/docs/api-reference/images/create
+//
 
 /*
 *
@@ -59,6 +59,11 @@ func downloadFile(url string) (string, error) {
 
 *
 */
+// ImageEndpoint is the OpenAI Image generation API endpoint https://platform.openai.com/docs/api-reference/images/create
+// @Summary Creates an image given a prompt.
+// @Param request body schema.OpenAIRequest true "query params"
+// @Success 200 {object} schema.OpenAIResponse "Response"
+// @Router /v1/images/generations [post]
 func ImageEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 		m, input, err := readRequest(c, ml, appConfig, false)

core/http/endpoints/openai/request.go

@@ -146,7 +146,14 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
 
 	if input.ToolsChoice != nil {
 		var toolChoice grammar.Tool
-		json.Unmarshal([]byte(input.ToolsChoice.(string)), &toolChoice)
+
+		switch content := input.ToolsChoice.(type) {
+		case string:
+			_ = json.Unmarshal([]byte(content), &toolChoice)
+		case map[string]interface{}:
+			dat, _ := json.Marshal(content)
+			_ = json.Unmarshal(dat, &toolChoice)
+		}
 		input.FunctionCall = map[string]interface{}{
 			"name": toolChoice.Function.Name,
 		}
@@ -185,11 +192,11 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
 		config.RepeatPenalty = input.RepeatPenalty
 	}
 
-	if input.FrequencyPenalty!= 0 {
+	if input.FrequencyPenalty != 0 {
 		config.FrequencyPenalty = input.FrequencyPenalty
 	}
 
-	if input.PresencePenalty!= 0 {
+	if input.PresencePenalty != 0 {
 		config.PresencePenalty = input.PresencePenalty
 	}
 
@@ -209,7 +216,7 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
 		config.Seed = input.Seed
 	}
 
-	if input.TypicalP != 0 {
+	if input.TypicalP != nil {
 		config.TypicalP = input.TypicalP
 	}
 

core/http/endpoints/openai/transcription.go

@@ -16,7 +16,13 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-// https://platform.openai.com/docs/api-reference/audio/create
+// TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create
+// @Summary Transcribes audio into the input language.
+// @accept multipart/form-data
+// @Param model formData string true "model"
+// @Param file formData file true "file"
+// @Success 200 {object} map[string]string	 "Response"
+// @Router /v1/audio/transcriptions [post]
 func TranscriptEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 		m, input, err := readRequest(c, ml, appConfig, false)

core/http/render.go

@@ -0,0 +1,80 @@
+package http
+
+import (
+	"embed"
+	"fmt"
+	"html/template"
+	"net/http"
+
+	"github.com/Masterminds/sprig/v3"
+	"github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/core/schema"
+	"github.com/go-skynet/LocalAI/internal"
+	"github.com/go-skynet/LocalAI/pkg/model"
+	"github.com/gofiber/fiber/v2"
+	fiberhtml "github.com/gofiber/template/html/v2"
+	"github.com/russross/blackfriday"
+)
+
+//go:embed views/*
+var viewsfs embed.FS
+
+func notFoundHandler(c *fiber.Ctx) error {
+	// Check if the request accepts JSON
+	if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 {
+		// The client expects a JSON response
+		c.Status(fiber.StatusNotFound).JSON(schema.ErrorResponse{
+			Error: &schema.APIError{Message: "Resource not found", Code: fiber.StatusNotFound},
+		})
+	} else {
+		// The client expects an HTML response
+		c.Status(fiber.StatusNotFound).Render("views/404", fiber.Map{})
+	}
+	return nil
+}
+
+func welcomeRoute(
+	app *fiber.App,
+	cl *config.BackendConfigLoader,
+	ml *model.ModelLoader,
+	appConfig *config.ApplicationConfig,
+	auth func(*fiber.Ctx) error,
+) {
+	if appConfig.DisableWelcomePage {
+		return
+	}
+
+	models, _ := ml.ListModels()
+	backendConfigs := cl.GetAllBackendConfigs()
+
+	app.Get("/", auth, func(c *fiber.Ctx) error {
+		summary := fiber.Map{
+			"Title":             "LocalAI API - " + internal.PrintableVersion(),
+			"Version":           internal.PrintableVersion(),
+			"Models":            models,
+			"ModelsConfig":      backendConfigs,
+			"ApplicationConfig": appConfig,
+		}
+
+		if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 {
+			// The client expects a JSON response
+			return c.Status(fiber.StatusOK).JSON(summary)
+		} else {
+			// Render index
+			return c.Render("views/index", summary)
+		}
+	})
+
+}
+
+func renderEngine() *fiberhtml.Engine {
+	engine := fiberhtml.NewFileSystem(http.FS(viewsfs), ".html")
+	engine.AddFuncMap(sprig.FuncMap())
+	engine.AddFunc("MDToHTML", markDowner)
+	return engine
+}
+
+func markDowner(args ...interface{}) template.HTML {
+	s := blackfriday.MarkdownCommon([]byte(fmt.Sprintf("%s", args...)))
+	return template.HTML(s)
+}

core/http/views/404.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en">
+
+{{template "views/partials/head" .}}
+
+<body class="bg-black text-white">
+<div class="flex flex-col min-h-screen">
+   
+    {{template "views/partials/navbar" .}}
+    
+    <div class="container mx-auto px-4 flex-grow">
+        <div class="header text-center py-12">
+            <h1 class="text-5xl font-bold">Welcome to your LocalAI instance!</h1>
+            <div class="mt-6">
+         <!--       <a href="/" aria-label="HomePage" alt="HomePage">           
+                    <img class="mx-auto w-1/4 h-auto" src="https://github.com/go-skynet/LocalAI/assets/2420543/0966aa2a-166e-4f99-a3e5-6c915fc997dd" alt="LocalAI Logo">            
+                </a>
+            -->
+            </div>
+            <p class="mt-4 text-lg">The FOSS alternative to OpenAI, Claude, ...</p>
+            <a href="https://localai.io" target="_blank" class="mt-4 inline-block bg-blue-500 text-white py-2 px-4 rounded transition duration-300 ease-in-out hover:bg-blue-700"><i class="fas fa-book-reader pr-2"></i>Documentation</a>
+        </div>
+
+        <div class="models mt-12">
+            <h2 class="text-center text-3xl font-semibold">Nothing found!</h2>
+        </div>
+    </div>
+
+    {{template "views/partials/footer" .}}
+</div>
+
+</body>
+</html>

core/http/views/index.html

@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+<html lang="en">
+{{template "views/partials/head" .}}
+
+<body class="bg-gray-900 text-gray-200">
+<div class="flex flex-col min-h-screen">
+   
+    {{template "views/partials/navbar" .}}
+    
+    <div class="container mx-auto px-4 flex-grow">
+        <div class="header text-center py-12">
+            <h1 class="text-5xl font-bold text-gray-100">Welcome to <i>your</i> LocalAI instance!</h1>
+            <div class="mt-6">
+                <!-- Logo can be uncommented and updated with a valid URL -->
+            </div>
+            <p class="mt-4 text-lg">The FOSS alternative to OpenAI, Claude, ...</p>
+            <a href="https://localai.io" target="_blank" class="mt-4 inline-block bg-blue-500 text-white py-2 px-4 rounded-lg shadow transition duration-300 ease-in-out hover:bg-blue-700 hover:shadow-lg">
+                <i class="fas fa-book-reader pr-2"></i>Documentation
+            </a>
+        </div>
+
+        <div class="models mt-12">
+            <h2 class="text-center text-3xl font-semibold text-gray-100">Installed models</h2>
+            <p class="text-center mt-4 text-xl">We have {{len .ModelsConfig}} pre-loaded models available.</p>
+            <ul class="mt-8 space-y-4">
+                {{ range .ModelsConfig }}
+                <li class="bg-gray-800 border border-gray-700 p-4 rounded-lg">
+                    <div class="flex justify-between items-center">
+                        <p class="font-bold text-white flex items-center"><i class="fas fa-brain pr-2"></i>{{.Name}}</p>
+                        {{ if .Backend }}
+                        <!-- Badge for Backend -->
+                        <span class="inline-block bg-blue-500 text-white py-1 px-3 rounded-full text-xs">
+                            {{.Backend}}
+                        </span>
+                        {{ else }}
+                        <span class="inline-block bg-yellow-500 text-white py-1 px-3 rounded-full text-xs">
+                            auto
+                        </span>
+                        {{ end }}
+                    </div>
+                    <!-- Additional details can go here -->
+                </li>
+                {{ end }}
+            </ul>
+        </div>
+    </div>
+
+    {{template "views/partials/footer" .}}
+</div>
+
+</body>
+</html>

core/http/views/partials/footer.html

@@ -0,0 +1,4 @@
+<footer class="text-center py-8">
+    LocalAI Version {{.Version}}<br>
+    <a href='https://localai.io' class="text-blue-400 hover:text-blue-600" target="_blank">LocalAI</a> © 2023-2024 <a href='https://mudler.pm' class="text-blue-400 hover:text-blue-600" target="_blank">Ettore Di Giacinto</a>
+</footer>

core/http/views/partials/head.html

@@ -0,0 +1,13 @@
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{{.Title}}</title>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&family=Roboto:wght@400;500&display=swap" rel="stylesheet">
+    <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.1.1/css/all.min.css">
+    <style>
+        body {
+            font-family: 'Inter', sans-serif;
+        }
+    </style>
+</head>

core/http/views/partials/navbar.html

@@ -0,0 +1,16 @@
+<nav class="bg-gray-800 shadow-lg">
+    <div class="container mx-auto px-4 py-4">
+        <div class="flex items-center justify-between">
+            <div class="flex items-center">
+                <!-- Logo Image: Replace 'logo_url_here' with your actual logo URL -->
+                <a href="/" class="text-white text-xl font-bold"><img src="https://github.com/go-skynet/LocalAI/assets/2420543/0966aa2a-166e-4f99-a3e5-6c915fc997dd" alt="LocalAI Logo" class="h-10 mr-3 border-2 border-gray-300 shadow rounded"></a>
+                <a href="/" class="text-white text-xl font-bold">LocalAI</a>
+            </div>
+            <div>
+                <a href="/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-home pr-2"></i>Home</a>
+                <a href="https://localai.io" class="text-gray-400 hover:text-white px-3 py-2 rounded" target="_blank" ><i class="fas fa-book-reader pr-2"></i> Documentation</a>
+                <a href="/swagger/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-code pr-2"></i> API</a>
+            </div>
+        </div>
+    </div>
+</nav>

core/schema/prediction.go

@@ -24,12 +24,12 @@ type PredictionOptions struct {
 	RepeatPenalty float64 `json:"repeat_penalty" yaml:"repeat_penalty"`
 	Keep          int     `json:"n_keep" yaml:"n_keep"`
 
-	FrequencyPenalty float64 `json:"frequency_penalty" yaml:"frequency_penalty"`
-	PresencePenalty  float64 `json:"presence_penalty" yaml:"presence_penalty"`
-	TFZ              float64 `json:"tfz" yaml:"tfz"`
+	FrequencyPenalty float64  `json:"frequency_penalty" yaml:"frequency_penalty"`
+	PresencePenalty  float64  `json:"presence_penalty" yaml:"presence_penalty"`
+	TFZ              *float64 `json:"tfz" yaml:"tfz"`
 
-	TypicalP float64 `json:"typical_p" yaml:"typical_p"`
-	Seed     *int    `json:"seed" yaml:"seed"`
+	TypicalP *float64 `json:"typical_p" yaml:"typical_p"`
+	Seed     *int     `json:"seed" yaml:"seed"`
 
 	NegativePrompt      string  `json:"negative_prompt" yaml:"negative_prompt"`
 	RopeFreqBase        float32 `json:"rope_freq_base" yaml:"rope_freq_base"`

core/services/backend_monitor.go

@@ -63,7 +63,7 @@ func (bm *BackendMonitor) SampleLocalBackendProcess(model string) (*schema.Backe
 	pid, err := bm.modelLoader.GetGRPCPID(backend)
 
 	if err != nil {
-		log.Error().Msgf("model %s : failed to find pid %+v", model, err)
+		log.Error().Err(err).Str("model", model).Msg("failed to find GRPC pid")
 		return nil, err
 	}
 
@@ -71,26 +71,26 @@ func (bm *BackendMonitor) SampleLocalBackendProcess(model string) (*schema.Backe
 	backendProcess, err := gopsutil.NewProcess(int32(pid))
 
 	if err != nil {
-		log.Error().Msgf("model %s [PID %d] : error getting process info %+v", model, pid, err)
+		log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting process info")
 		return nil, err
 	}
 
 	memInfo, err := backendProcess.MemoryInfo()
 
 	if err != nil {
-		log.Error().Msgf("model %s [PID %d] : error getting memory info %+v", model, pid, err)
+		log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory info")
 		return nil, err
 	}
 
 	memPercent, err := backendProcess.MemoryPercent()
 	if err != nil {
-		log.Error().Msgf("model %s [PID %d] : error getting memory percent %+v", model, pid, err)
+		log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory percent")
 		return nil, err
 	}
 
 	cpuPercent, err := backendProcess.CPUPercent()
 	if err != nil {
-		log.Error().Msgf("model %s [PID %d] : error getting cpu percent %+v", model, pid, err)
+		log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting cpu percent")
 		return nil, err
 	}
 

core/startup/config_file_watcher.go

@@ -85,7 +85,7 @@ func WatchConfigDirectory(configDir string, appConfig *config.ApplicationConfig)
 				if !ok {
 					return
 				}
-				log.Error().Msgf("WatchConfigDirectory goroutine error: %+v", err)
+				log.Error().Err(err).Msg("error encountered while watching config directory")
 			}
 		}
 	}()

core/startup/startup.go

@@ -61,17 +61,17 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
 	configLoaderOpts := options.ToConfigLoaderOptions()
 
 	if err := cl.LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
-		log.Error().Msgf("error loading config files: %s", err.Error())
+		log.Error().Err(err).Msg("error loading config files")
 	}
 
 	if options.ConfigFile != "" {
 		if err := cl.LoadBackendConfigFile(options.ConfigFile, configLoaderOpts...); err != nil {
-			log.Error().Msgf("error loading config file: %s", err.Error())
+			log.Error().Err(err).Msg("error loading config file")
 		}
 	}
 
 	if err := cl.Preload(options.ModelPath); err != nil {
-		log.Error().Msgf("error downloading models: %s", err.Error())
+		log.Error().Err(err).Msg("error downloading models")
 	}
 
 	if options.PreloadJSONModels != "" {

docs/content/docs/features/text-generation.md

@@ -304,6 +304,7 @@ The backend will automatically download the required files in order to run the m
 | Type | Description |
 | --- | --- |
 | `AutoModelForCausalLM` | `AutoModelForCausalLM` is a model that can be used to generate sequences. |
+| `OVModelForCausalLM` | for OpenVINO models |
 | N/A | Defaults to `AutoModel` |
 
 
@@ -324,4 +325,35 @@ curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d
    "prompt": "Hello, my name is",
    "temperature": 0.1, "top_p": 0.1
  }'
+```
+
+#### Examples
+
+##### OpenVINO
+
+A model configuration file for openvion and starling model:
+
+```yaml
+name: starling-openvino
+backend: transformers
+parameters:
+  model: fakezeta/Starling-LM-7B-beta-openvino-int8
+context_size: 8192
+threads: 6
+f16: true
+type: OVModelForCausalLM
+stopwords:
+- <|end_of_turn|>
+- <|endoftext|>
+prompt_cache_path: "cache"
+prompt_cache_all: true
+template:
+  chat_message: |
+    {{if eq .RoleName "system"}}{{.Content}}<|end_of_turn|>{{end}}{{if eq .RoleName "assistant"}}<|end_of_turn|>GPT4 Correct Assistant: {{.Content}}<|end_of_turn|>{{end}}{{if eq .RoleName "user"}}GPT4 Correct User: {{.Content}}{{end}}
+
+  chat: |
+    {{.Input}}<|end_of_turn|>GPT4 Correct Assistant:
+
+  completion: |
+    {{.Input}}
 ```

docs/content/docs/getting-started/quickstart.md

@@ -50,8 +50,8 @@ Start the image with Docker:
 ```bash
 docker run -p 8080:8080 --name local-ai -ti localai/localai:latest-aio-cpu
 # For Nvidia GPUs:
-# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-cuda-11
-# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-cuda-12
+# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-nvidia-cuda-11
+# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-nvidia-cuda-12
 ```
 
 
@@ -61,15 +61,19 @@ Or with a docker-compose file:
 version: "3.9"
 services:
   api:
-    image: localai/localai:{{< version >}}-aio-cpu
+    image: localai/localai:latest-aio-cpu
+    # For a specific version:
+    # image: localai/localai:{{< version >}}-aio-cpu
     # For Nvidia GPUs decomment one of the following (cuda11 or cuda12):
-    # image: localai/localai:{{< version >}}-aio-gpu-cuda-11
-    # image: localai/localai:{{< version >}}-aio-gpu-cuda-12
+    # image: localai/localai:{{< version >}}-aio-gpu-nvidia-cuda-11
+    # image: localai/localai:{{< version >}}-aio-gpu-nvidia-cuda-12
+    # image: localai/localai:latest-aio-gpu-nvidia-cuda-11
+    # image: localai/localai:latest-aio-gpu-nvidia-cuda-12
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
       interval: 1m
-      timeout: 120m
-      retries: 120
+      timeout: 20m
+      retries: 5
     ports:
       - 8080:8080
     environment:
@@ -89,8 +93,208 @@ services:
 
 For a list of all the container-images available, see [Container images]({{%relref "docs/reference/container-images" %}}). To learn more about All-in-one images instead, see [All-in-one Images]({{%relref "docs/reference/aio-images" %}}).
 
+{{% alert icon="💡" %}}
+
+**Models caching**: The **AIO** image will download the needed models on the first run if not already present and store those in `/build/models` inside the container. The AIO models will be automatically updated with new versions of AIO images.
+
+You can change the directory inside the container by specifying a `MODELS_PATH` environment variable (or `--models-path`). 
+
+If you want to use a named model or a local directory, you can mount it as a volume to `/build/models`:
+
+```bash
+docker run -p 8080:8080 --name local-ai -ti -v $PWD/models:/build/models localai/localai:latest-aio-cpu
+```
+
+or associate a volume:
+
+```bash
+docker volume create localai-models
+docker run -p 8080:8080 --name local-ai -ti -v localai-models:/build/models localai/localai:latest-aio-cpu
+```
+
+{{% /alert %}}
+
+## Try it out
+
+LocalAI does not ship a webui by default, however you can use 3rd party projects to interact with it (see also [Integrations]({{%relref "docs/integrations" %}}) ). However, you can test out the API endpoints using `curl`, you can find few examples below.
+
+### Text Generation
+
+Creates a model response for the given chat conversation. [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
+
+<details>
+
+```bash
+curl http://localhost:8080/v1/chat/completions \
+      -H "Content-Type: application/json" \
+      -d '{ "model": "gpt-4", "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}] }' 
+```
+
+</details>
+
+### GPT Vision
+
+Understand images.
+
+<details>
+
+```bash
+curl http://localhost:8080/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d '{ 
+        "model": "gpt-4-vision-preview", 
+        "messages": [
+          {
+            "role": "user", "content": [
+              {"type":"text", "text": "What is in the image?"},
+              {
+                "type": "image_url", 
+                "image_url": {
+                  "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" 
+                }
+              }
+            ], 
+          "temperature": 0.9
+          }
+        ]
+      }' 
+```
+
+</details>
+
+### Function calling
+
+Call functions
+
+<details>
+
+```bash
+curl https://localhost:8080/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4",
+    "messages": [
+      {
+        "role": "user",
+        "content": "What is the weather like in Boston?"
+      }
+    ],
+    "tools": [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_weather",
+          "description": "Get the current weather in a given location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "The city and state, e.g. San Francisco, CA"
+              },
+              "unit": {
+                "type": "string",
+                "enum": ["celsius", "fahrenheit"]
+              }
+            },
+            "required": ["location"]
+          }
+        }
+      }
+    ],
+    "tool_choice": "auto"
+  }'
+```
+
+</details>
+
+### Image Generation
+
+Creates an image given a prompt. [OpenAI documentation](https://platform.openai.com/docs/api-reference/images/create).
+
+<details>
+
+```bash
+curl http://localhost:8080/v1/images/generations \
+      -H "Content-Type: application/json" -d '{
+          "prompt": "A cute baby sea otter",
+          "size": "256x256"
+        }'
+```
+
+</details>
+
+### Text to speech
+
+
+Generates audio from the input text. [OpenAI documentation](https://platform.openai.com/docs/api-reference/audio/createSpeech).
+
+<details>
+
+```bash
+curl http://localhost:8080/v1/audio/speech \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "tts-1",
+    "input": "The quick brown fox jumped over the lazy dog.",
+    "voice": "alloy"
+  }' \
+  --output speech.mp3
+```
+
+</details>
+
+
+### Audio Transcription
+
+Transcribes audio into the input language. [OpenAI Documentation](https://platform.openai.com/docs/api-reference/audio/createTranscription).
+
+<details>
+
+Download first a sample to transcribe:
+
+```bash
+wget --quiet --show-progress -O gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg 
+```
+
+Send the example audio file to the transcriptions endpoint :
+```bash
+curl http://localhost:8080/v1/audio/transcriptions \
+    -H "Content-Type: multipart/form-data" \
+    -F file="@$PWD/gb1.ogg" -F model="whisper-1"
+```
+
+</details>
+
+### Embeddings Generation
+
+Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. [OpenAI Embeddings](https://platform.openai.com/docs/api-reference/embeddings).
+
+<details>
+
+```bash
+curl http://localhost:8080/embeddings \
+    -X POST -H "Content-Type: application/json" \
+    -d '{ 
+        "input": "Your text string goes here", 
+        "model": "text-embedding-ada-002"
+      }'
+```
+
+</details>
+
+{{% alert icon="💡" %}}
+
+Don't use the model file as `model` in the request unless you want to handle the prompt template for yourself.
+
+Use the model names like you would do with OpenAI like in the examples below. For instance `gpt-4-vision-preview`, or `gpt-4`.
+
+{{% /alert %}}
+
 ## What's next?
 
+There is much more to explore! run any model from huggingface, video generation, and voice cloning with LocalAI, check out the [features]({{%relref "docs/features" %}}) section for a full overview.
+
 Explore further resources and community contributions:
 
 - [Build LocalAI and the container image]({{%relref "docs/getting-started/build" %}})
@@ -98,4 +302,4 @@ Explore further resources and community contributions:
 - [Run other models]({{%relref "docs/getting-started/run-other-models" %}})
 - [Container images]({{%relref "docs/reference/container-images" %}})
 - [All-in-one Images]({{%relref "docs/reference/aio-images" %}})
-- [Examples](https://github.com/mudler/LocalAI/tree/master/examples#examples)
+- [Examples](https://github.com/mudler/LocalAI/tree/master/examples#examples)

docs/content/docs/getting-started/run-other-models.md

@@ -80,7 +80,7 @@ To customize the models, see [Model customization]({{%relref "docs/getting-start
 | transformers-tinyllama | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda11 transformers-tinyllama``` |
 | [codellama-7b](https://huggingface.co/codellama/CodeLlama-7b-hf) | [LLM]({{%relref "docs/features/text-generation" %}})  | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda11 codellama-7b``` |
 | [codellama-7b-gguf](https://huggingface.co/TheBloke/CodeLlama-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}})  | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda11-core codellama-7b-gguf``` |
-| [hermes-2-pro-mistral](https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 localai/localai:{{< version >}}-cublas-cuda11-core hermes-2-pro-mistral``` |
+| [hermes-2-pro-mistral](https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda11-core hermes-2-pro-mistral``` |
 {{% /tab %}}
 
 
@@ -111,7 +111,7 @@ To customize the models, see [Model customization]({{%relref "docs/getting-start
 | transformers-tinyllama | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda12 transformers-tinyllama``` |
 | [codellama-7b](https://huggingface.co/codellama/CodeLlama-7b-hf) | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda12 codellama-7b``` |
 | [codellama-7b-gguf](https://huggingface.co/TheBloke/CodeLlama-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}})  | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda12-core codellama-7b-gguf``` |
-| [hermes-2-pro-mistral](https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 localai/localai:{{< version >}}-cublas-cuda12-core hermes-2-pro-mistral``` |
+| [hermes-2-pro-mistral](https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF) | [LLM]({{%relref "docs/features/text-generation" %}}) | ```docker run -ti -p 8080:8080 --gpus all localai/localai:{{< version >}}-cublas-cuda12-core hermes-2-pro-mistral``` |
 {{% /tab %}}
 
 {{< /tabs >}}
@@ -123,4 +123,4 @@ To customize the models, see [Model customization]({{%relref "docs/getting-start
 docker run -ti -p 8080:8080 localai/localai:{{< version >}}-ffmpeg-core llava phi-2
 ```
 
-{{% /alert %}}
+{{% /alert %}}

docs/content/docs/overview.md

@@ -49,7 +49,6 @@ icon = "info"
 </a>
 </p>
 
-
 > 💡 Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [💭Discord](https://discord.gg/uJAeKSAGDy)
 >
 > [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/) [ 🚀 Roadmap ](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)

docs/content/docs/reference/aio-images.md

@@ -7,6 +7,28 @@ weight = 26
 
 All-In-One images are images that come pre-configured with a set of models and backends to fully leverage almost all the LocalAI featureset. These images are available for both CPU and GPU environments. The AIO images are designed to be easy to use and requires no configuration. Models configuration can be found [here](https://github.com/mudler/LocalAI/tree/master/aio) separated by size.
 
+In the AIO images there are models configured with the names of OpenAI models, however, they are really backed by Open Source models. You can find the table below
+
+| Category | Model name | Real model |
+| Text Generation | `gpt-4` | `phi-2`(CPU) or `hermes-2-pro-mistral`(GPU) |
+| Multimodal | `gpt-4-vision-preview` | `bakllava`(CPU) or `llava-1.6-mistral`(GPU) |
+| Text generation | `stablediffusion` | `stablediffusion`(CPU) `dreamshaper-8` (GPU) |
+| Audio transcription | `whisper-1` | `whisper` with the `whisper-base` model |
+| Text to Audio | `tts-1` | the `en-us-amy-low.onnx` model with `rhasspy` |
+| Embeddings | `text-embedding-ada-002` | | 
+
+## Usage
+
+Select the image (CPU or GPU) and start the container with Docker:
+
+```bash
+# CPU example
+docker run -p 8080:8080 --name local-ai -ti localai/localai:latest-aio-cpu
+```
+
+LocalAI will automatically download all the required models, and the API will be available at [localhost:8080](http://localhost:8080/v1/models).
+
+## Available images
 
 | Description | Quay | Docker Hub                                   |
 | --- | --- |-----------------------------------------------|
@@ -24,16 +46,7 @@ The AIO Images are inheriting the same environment variables as the base images
 
 | Variable | Default | Description |
 | ---------------------| ------- | ----------- |
-| `SIZE` | Auto-detected | The size of the model to use. Available: `cpu`, `gpu-8g` |
+| `PROFILE` | Auto-detected | The size of the model to use. Available: `cpu`, `gpu-8g` |
 | `MODELS` | Auto-detected | A list of models YAML Configuration file URI/URL (see also [running models]({{%relref "docs/getting-started/run-other-models" %}})) |
 
 
-## Example
-
-Start the image with Docker:
-
-```bash
-docker run -p 8080:8080 --name local-ai -ti localai/localai:latest-aio-cpu
-```
-
-LocalAI will automatically download all the required models, and will be available at [localhost:8080](http://localhost:8080/v1/models).

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v2.10.1"
+  "version": "v2.12.1"
 }

embedded/models/hermes-2-pro-mistral.yaml

@@ -3,39 +3,39 @@ mmap: true
 parameters:
   model: huggingface://NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/Hermes-2-Pro-Mistral-7B.Q6_K.gguf
 
-roles:
-  assistant_function_call: assistant
-  function: tool
 template:
   chat_message: |
-    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "function"}}{{.Role}}{{else if eq .RoleName "user"}}user{{end}}
-    {{ if eq .RoleName "assistant_function_call" }}<tool_call>{{end}}
-    {{ if eq .RoleName "function" }}<tool_result>{{end}}
-    {{if .Content}}{{.Content}}{{end}}
-    {{if .FunctionCall}}{{toJson .FunctionCall}}{{end}}
-    {{ if eq .RoleName "assistant_function_call" }}</tool_call>{{end}}
-    {{ if eq .RoleName "function" }}</tool_result>{{end}}
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}
+    {{- if .FunctionCall }}<tool_call>{{end}}
+    {{- if eq .RoleName "tool" }}<tool_result>{{end }}
+    {{- if .Content}}
+    {{.Content}}
+    {{- end }}
+    {{- if .FunctionCall}}{{toJson .FunctionCall}}{{end }}
+    {{- if .FunctionCall }}</tool_call>{{end }}
+    {{- if eq .RoleName "tool" }}</tool_result>{{end }}
     <|im_end|>
   # https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF#prompt-format-for-function-calling
   function: |
     <|im_start|>system
-    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: 
+    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
     <tools>
     {{range .Functions}}
     {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
     {{end}}
-    </tools> 
-    Use the following pydantic model json schema for each tool call you will make: 
-    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']} 
+    </tools>
+    Use the following pydantic model json schema for each tool call you will make:
+    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}
     For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
     <tool_call>
     {'arguments': <args-dict>, 'name': <function-name>}
-    </tool_call><|im_end|>
-    {{.Input}}
+    </tool_call>
+    <|im_end|>
+    {{.Input -}}
     <|im_start|>assistant
     <tool_call>
   chat: |
-    {{.Input}}
+    {{.Input -}}
     <|im_start|>assistant
   completion: |
     {{.Input}}
@@ -44,6 +44,8 @@ f16: true
 stopwords:
 - <|im_end|>
 - <dummy32000>
+- "\n</tool_call>"
+- "\n\n\n"
 usage: |
       curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
           "model": "hermes-2-pro-mistral",

embedded/models/phi-2-orange.yaml

@@ -0,0 +1,30 @@
+name: phi-2-orange
+mmap: true
+parameters:
+  model: huggingface://l3utterfly/phi-2-orange-GGUF/phi-2-orange.Q6_K.gguf
+
+template:
+  chat_message: |
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
+    {{if .Content}}{{.Content}}{{end}}
+    <|im_end|>
+  chat: |
+    {{.Input}}
+    <|im_start|>assistant
+  completion: |
+    {{.Input}}
+context_size: 4096
+f16: true
+stopwords:
+- <|im_end|>
+- <dummy32000>
+
+description: |
+  This model is a chatbot that can be used for general conversation.
+  [Model card](https://huggingface.co/TheBloke/phi-2-orange-GGUF)
+
+usage: |
+      curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+          "model": "phi-2-orange",
+          "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}]
+      }'

entrypoint.sh

@@ -47,4 +47,4 @@ else
 	echo "@@@@@"
 fi
 
-./local-ai "$@"
+exec ./local-ai "$@"

go.mod

@@ -4,14 +4,18 @@ go 1.21
 
 require (
 	github.com/M0Rf30/go-tiny-dream v0.0.0-20231128165230-772a9c0d9aaf
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/charmbracelet/glamour v0.6.0
 	github.com/donomii/go-rwkv.cpp v0.0.0-20230715075832-c898cd0f62df
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230628193450-85ed71aaec8e
 	github.com/go-audio/wav v1.1.0
 	github.com/go-skynet/go-bert.cpp v0.0.0-20230716133540-6abe312cded1
 	github.com/go-skynet/go-llama.cpp v0.0.0-20231009155254-aeba71ee8428
-	github.com/gofiber/fiber/v2 v2.50.0
-	github.com/google/uuid v1.3.1
+	github.com/gofiber/fiber/v2 v2.52.0
+	github.com/gofiber/template/html/v2 v2.1.1
+	github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47
+	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hpcloud/tail v1.0.0
 	github.com/imdario/mergo v0.3.16
@@ -21,16 +25,17 @@ require (
 	github.com/nomic-ai/gpt4all/gpt4all-bindings/golang v0.0.0-20231022042237-c25dc5193530
 	github.com/onsi/ginkgo/v2 v2.13.0
 	github.com/onsi/gomega v1.28.1
+	github.com/ory/dockertest/v3 v3.10.0
 	github.com/otiai10/openaigo v1.6.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/prometheus/client_golang v1.17.0
 	github.com/rs/zerolog v1.31.0
 	github.com/sashabaranov/go-openai v1.20.4
 	github.com/schollz/progressbar/v3 v3.13.1
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/tmc/langchaingo v0.0.0-20231019140956-c636b3da7701
-	github.com/urfave/cli/v2 v2.25.7
-	github.com/valyala/fasthttp v1.50.0
+	github.com/urfave/cli/v2 v2.27.1
+	github.com/valyala/fasthttp v1.51.0
 	go.opentelemetry.io/otel v1.19.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.42.0
 	go.opentelemetry.io/otel/metric v1.19.0
@@ -54,18 +59,19 @@ require (
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
-	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/PuerkitoBio/purell v1.2.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/aymanbagabas/go-osc52 v1.0.3 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/charmbracelet/glamour v0.6.0 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
@@ -75,14 +81,23 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/gofiber/swagger v1.0.0 // indirect
+	github.com/gofiber/template v1.8.3 // indirect
+	github.com/gofiber/utils v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
@@ -97,7 +112,6 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opencontainers/runc v1.1.5 // indirect
-	github.com/ory/dockertest/v3 v3.10.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkoukk/tiktoken-go v0.1.2 // indirect
@@ -105,9 +119,12 @@ require (
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
+	github.com/swaggo/files/v2 v2.0.0 // indirect
+	github.com/swaggo/swag v1.16.3 // indirect
 	github.com/ulikunitz/xz v0.5.9 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -117,17 +134,18 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/go-audio/audio v1.0.0 // indirect
 	github.com/go-audio/riff v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -135,18 +153,18 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mudler/go-piper v0.0.0-20230621222733-56b8a81b4760
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 )

go.sum

@@ -1,8 +1,8 @@
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/M0Rf30/go-tiny-dream v0.0.0-20231128165230-772a9c0d9aaf h1:UgjXLcE9I+VaVz7uBIlzAnyZIXwiDlIiTWqCh159aUI=
-github.com/M0Rf30/go-tiny-dream v0.0.0-20231128165230-772a9c0d9aaf/go.mod h1:UOf2Mb/deUri5agct5OJ4SLWjhI+kZKbsUVUeRb24I0=
+github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
+github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
@@ -13,6 +13,10 @@ github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2y
 github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
+github.com/PuerkitoBio/purell v1.2.1 h1:QsZ4TjvwiMpat6gBCBxEQI0rcS9ehtkKtSpiUnd9N28=
+github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPngi8iYYv/Eo=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
 github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
@@ -43,6 +47,9 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -59,19 +66,14 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/donomii/go-rwkv.cpp v0.0.0-20230715075832-c898cd0f62df h1:qVcBEZlvp5A1gGWNJj02xyDtbsUI2hohlQMSB1fgER4=
-github.com/donomii/go-rwkv.cpp v0.0.0-20230715075832-c898cd0f62df/go.mod h1:gWy7FIWioqYmYxkaoFyBnaKApeZVrUkHhv9EV9pz4dM=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230628193450-85ed71aaec8e h1:KtbU2JR3lJuXFASHG2+sVLucfMPBjWKUUKByX6C81mQ=
-github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230628193450-85ed71aaec8e/go.mod h1:QIjZ9OktHFG7p+/m3sMvrAJKKdWrr1fZIK0rM6HZlyo=
 github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=
 github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
 github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA=
@@ -85,10 +87,18 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-skynet/go-bert.cpp v0.0.0-20230716133540-6abe312cded1 h1:yXvc7QfGtoZ51tUW/YVjoTwAfh8HG88XU7UOrbNlz5Y=
-github.com/go-skynet/go-bert.cpp v0.0.0-20230716133540-6abe312cded1/go.mod h1:fYjkCDRzC+oRLHSjQoajmYK6AmeJnmEanV27CClAcDc=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
+github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
+github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
+github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-skynet/go-llama.cpp v0.0.0-20231009155254-aeba71ee8428 h1:WYjkXL0Nw7dN2uDBMVCWQ8xLavrIhjF/DLczuh5L9TY=
 github.com/go-skynet/go-llama.cpp v0.0.0-20231009155254-aeba71ee8428/go.mod h1:iub0ugfTnflE3rcIuqV2pQSo15nEw3GLW/utm5gyERo=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
@@ -96,6 +106,16 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
 github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
+github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
+github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/swagger v1.0.0 h1:BzUzDS9ZT6fDUa692kxmfOjc1DZiloLiPK/W5z1H1tc=
+github.com/gofiber/swagger v1.0.0/go.mod h1:QrYNF1Yrc7ggGK6ATsJ6yfH/8Zi5bu9lA7wB8TmCecg=
+github.com/gofiber/template v1.8.3 h1:hzHdvMwMo/T2kouz2pPCA0zGiLCeMnoGsQZBTSYgZxc=
+github.com/gofiber/template v1.8.3/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8=
+github.com/gofiber/template/html/v2 v2.1.1 h1:QEy3O3EBkvwDthy5bXVGUseOyO6ldJoiDxlF4+MJiV8=
+github.com/gofiber/template/html/v2 v2.1.1/go.mod h1:2G0GHHOUx70C1LDncoBpe4T6maQbNa4x1CVNFW0wju0=
+github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM=
+github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -111,6 +131,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47 h1:k4Tw0nt6lwro3Uin8eqoET7MDA4JnT8YgbCjc/g5E3k=
+github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -127,6 +149,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -141,6 +165,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -148,6 +174,8 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -158,16 +186,22 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394=
+github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -178,8 +212,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
 github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
 github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
 github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
-github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
-github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
 github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
 github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
@@ -194,18 +226,12 @@ github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/mudler/go-piper v0.0.0-20230621222733-56b8a81b4760 h1:OFVkSxR7CRSRSNm5dvpMRZwmSwWa8EMMnHbc84fW5tU=
-github.com/mudler/go-piper v0.0.0-20230621222733-56b8a81b4760/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig=
 github.com/mudler/go-processmanager v0.0.0-20230818213616-f204007f963c h1:CI5uGwqBpN8N7BrSKC+nmdfw+9nPQIDyjHHlaIiitZI=
 github.com/mudler/go-processmanager v0.0.0-20230818213616-f204007f963c/go.mod h1:gY3wyrhkRySJtmtI/JPt4a2mKv48h/M9pEZIW+SjeC0=
-github.com/mudler/go-stable-diffusion v0.0.0-20230605122230-d89260f598af h1:XFq6OUqsWQam0OrEr05okXsJK/TQur3zoZTHbiZD3Ks=
-github.com/mudler/go-stable-diffusion v0.0.0-20230605122230-d89260f598af/go.mod h1:8ufRkpz/S/9ahkaxzZ5i4WMgO9w4InEhuRoT7vK5Rnw=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
 github.com/muesli/termenv v0.13.0 h1:wK20DRpJdDX8b7Ek2QfhvqhRQFZ237RGRO0RQ/Iqdy0=
 github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc=
-github.com/nomic-ai/gpt4all/gpt4all-bindings/golang v0.0.0-20231022042237-c25dc5193530 h1:YXMxHwHMB9jCBo2Yu5gz3mTB3T1TnZs/HmPLv15LUSA=
-github.com/nomic-ai/gpt4all/gpt4all-bindings/golang v0.0.0-20231022042237-c25dc5193530/go.mod h1:4T3CHXyrt+7FQHXaxULZfPjHbD8/99WuDDJa0YVZARI=
 github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -267,11 +293,11 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
+github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sashabaranov/go-openai v1.16.0 h1:34W6WV84ey6OpW0p2UewZkdMu82AxGC+BzpU6iiauRw=
-github.com/sashabaranov/go-openai v1.16.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/sashabaranov/go-openai v1.20.4 h1:095xQ/fAtRa0+Rj21sezVJABgKfGPNbyx/sAN/hJUmg=
 github.com/sashabaranov/go-openai v1.20.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
@@ -304,6 +330,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
+github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
+github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
+github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
@@ -319,10 +351,14 @@ github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
+github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
 github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
@@ -337,6 +373,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
+github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -365,11 +403,15 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -385,12 +427,17 @@ golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfS
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -424,16 +471,18 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -441,6 +490,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -450,6 +501,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -486,3 +539,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
+gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

main.go

@@ -25,6 +25,8 @@ import (
 	"github.com/rs/zerolog/log"
 	progressbar "github.com/schollz/progressbar/v3"
 	"github.com/urfave/cli/v2"
+
+	_ "github.com/go-skynet/LocalAI/swagger"
 )
 
 const (
@@ -43,7 +45,7 @@ func main() {
 
 	path, err := os.Getwd()
 	if err != nil {
-		log.Error().Msgf("error: %s", err.Error())
+		log.Error().Err(err).Msg("failed to get current directory")
 		os.Exit(1)
 	}
 
@@ -149,6 +151,12 @@ func main() {
 				EnvVars: []string{"UPLOAD_PATH"},
 				Value:   "/tmp/localai/upload",
 			},
+			&cli.StringFlag{
+				Name:    "config-path",
+				Usage:   "Path to store uploads from files api",
+				EnvVars: []string{"CONFIG_PATH"},
+				Value:   "/tmp/localai/config",
+			},
 			&cli.StringFlag{
 				Name:    "backend-assets-path",
 				Usage:   "Path used to extract libraries that are required by some of the backends in runtime.",
@@ -183,6 +191,12 @@ func main() {
 				EnvVars: []string{"WATCHDOG_IDLE"},
 				Value:   false,
 			},
+			&cli.BoolFlag{
+				Name:    "disable-welcome",
+				Usage:   "Disable welcome pages",
+				EnvVars: []string{"DISABLE_WELCOME"},
+				Value:   false,
+			},
 			&cli.BoolFlag{
 				Name:    "enable-watchdog-busy",
 				Usage:   "Enable watchdog for stopping busy backends that exceed a defined threshold.",
@@ -241,6 +255,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 				config.WithImageDir(ctx.String("image-path")),
 				config.WithAudioDir(ctx.String("audio-path")),
 				config.WithUploadDir(ctx.String("upload-path")),
+				config.WithConfigsDir(ctx.String("config-path")),
 				config.WithF16(ctx.Bool("f16")),
 				config.WithStringGalleries(ctx.String("galleries")),
 				config.WithModelLibraryURL(ctx.String("remote-library")),
@@ -257,6 +272,11 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 
 			idleWatchDog := ctx.Bool("enable-watchdog-idle")
 			busyWatchDog := ctx.Bool("enable-watchdog-busy")
+
+			if ctx.Bool("disable-welcome") {
+				opts = append(opts, config.DisableWelcomePage)
+			}
+
 			if idleWatchDog || busyWatchDog {
 				opts = append(opts, config.EnableWatchDog)
 				if idleWatchDog {
@@ -320,7 +340,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 
 			appHTTP, err := http.App(cl, ml, options)
 			if err != nil {
-				log.Error().Msg("Error during HTTP App constructor")
+				log.Error().Err(err).Msg("error during HTTP App construction")
 				return err
 			}
 
@@ -337,7 +357,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 						Action: func(ctx *cli.Context) error {
 							var galleries []gallery.Gallery
 							if err := json.Unmarshal([]byte(ctx.String("galleries")), &galleries); err != nil {
-								log.Error().Msgf("unable to load galleries: %s", err.Error())
+								log.Error().Err(err).Msg("unable to load galleries")
 							}
 
 							models, err := gallery.AvailableGalleryModels(galleries, ctx.String("models-path"))
@@ -362,7 +382,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 
 							var galleries []gallery.Gallery
 							if err := json.Unmarshal([]byte(ctx.String("galleries")), &galleries); err != nil {
-								log.Error().Msgf("unable to load galleries: %s", err.Error())
+								log.Error().Err(err).Msg("unable to load galleries")
 							}
 
 							progressBar := progressbar.NewOptions(
@@ -527,7 +547,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
 
 	err = app.Run(os.Args)
 	if err != nil {
-		log.Error().Msgf("error: %s", err.Error())
+		log.Error().Err(err).Msg("application runtime error")
 		os.Exit(1)
 	}
 }

pkg/gallery/models.go

@@ -67,7 +67,7 @@ func GetGalleryConfigFromURL(url string) (Config, error) {
 		return yaml.Unmarshal(d, &config)
 	})
 	if err != nil {
-		log.Error().Msgf("GetGalleryConfigFromURL error for url %s\n%s", url, err.Error())
+		log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
 		return config, err
 	}
 	return config, nil

pkg/model/initializers.go

@@ -15,8 +15,8 @@ import (
 )
 
 var Aliases map[string]string = map[string]string{
-	"go-llama": LLamaCPP,
-	"llama":    LLamaCPP,
+	"go-llama":       LLamaCPP,
+	"llama":          LLamaCPP,
 	"embedded-store": LocalStoreBackend,
 }
 
@@ -127,7 +127,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
 				break
 			}
 			if err != nil && i == o.grpcAttempts-1 {
-				log.Error().Msgf("Failed starting/connecting to the gRPC service: %s", err.Error())
+				log.Error().Err(err).Msg("failed starting/connecting to the gRPC service")
 			}
 			time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second)
 		}

pkg/model/watchdog.go

@@ -110,10 +110,10 @@ func (wd *WatchDog) checkIdle() {
 		log.Debug().Msgf("[WatchDog] %s: idle connection", address)
 		if time.Since(t) > wd.idletimeout {
 			log.Warn().Msgf("[WatchDog] Address %s is idle for too long, killing it", address)
-			p, ok := wd.addressModelMap[address]
+			model, ok := wd.addressModelMap[address]
 			if ok {
-				if err := wd.pm.ShutdownModel(p); err != nil {
-					log.Error().Msgf("[watchdog] Error shutting down model %s: %v", p, err)
+				if err := wd.pm.ShutdownModel(model); err != nil {
+					log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
 				}
 				log.Debug().Msgf("[WatchDog] model shut down: %s", address)
 				delete(wd.idleTime, address)
@@ -141,7 +141,7 @@ func (wd *WatchDog) checkBusy() {
 			if ok {
 				log.Warn().Msgf("[WatchDog] Model %s is busy for too long, killing it", model)
 				if err := wd.pm.ShutdownModel(model); err != nil {
-					log.Error().Msgf("[watchdog] Error shutting down model %s: %v", model, err)
+					log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
 				}
 				log.Debug().Msgf("[WatchDog] model shut down: %s", address)
 				delete(wd.timetable, address)

pkg/startup/model_preload.go

@@ -35,14 +35,15 @@ func PreloadModelsConfigurations(modelLibraryURL string, modelPath string, model
 			modelYAML, err := embedded.ResolveContent(url)
 			// If we resolve something, just save it to disk and continue
 			if err != nil {
-				log.Error().Msgf("error loading model: %s", err.Error())
+				log.Error().Err(err).Msg("error resolving model content")
 				continue
 			}
 
 			log.Debug().Msgf("[startup] resolved embedded model: %s", url)
 			md5Name := utils.MD5(url)
-			if err := os.WriteFile(filepath.Join(modelPath, md5Name)+".yaml", modelYAML, os.ModePerm); err != nil {
-				log.Error().Msgf("error loading model: %s", err.Error())
+			modelDefinitionFilePath := filepath.Join(modelPath, md5Name) + ".yaml"
+			if err := os.WriteFile(modelDefinitionFilePath, modelYAML, os.ModePerm); err != nil {
+				log.Error().Err(err).Str("filepath", modelDefinitionFilePath).Msg("error writing model definition")
 			}
 		case downloader.LooksLikeURL(url):
 			log.Debug().Msgf("[startup] resolved model to download: %s", url)
@@ -52,11 +53,12 @@ func PreloadModelsConfigurations(modelLibraryURL string, modelPath string, model
 
 			// check if file exists
 			if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
-				err := downloader.DownloadFile(url, filepath.Join(modelPath, md5Name)+".yaml", "", func(fileName, current, total string, percent float64) {
+				modelDefinitionFilePath := filepath.Join(modelPath, md5Name) + ".yaml"
+				err := downloader.DownloadFile(url, modelDefinitionFilePath, "", func(fileName, current, total string, percent float64) {
 					utils.DisplayDownloadFunction(fileName, current, total, percent)
 				})
 				if err != nil {
-					log.Error().Msgf("error loading model: %s", err.Error())
+					log.Error().Err(err).Str("url", url).Str("filepath", modelDefinitionFilePath).Msg("error downloading model")
 				}
 			}
 		default:
@@ -67,12 +69,13 @@ func PreloadModelsConfigurations(modelLibraryURL string, modelPath string, model
 
 				modelYAML, err := os.ReadFile(url)
 				if err != nil {
-					log.Error().Msgf("error loading model: %s", err.Error())
+					log.Error().Err(err).Str("filepath", url).Msg("error reading model definition")
 					continue
 				}
 
-				if err := os.WriteFile(filepath.Join(modelPath, md5Name)+".yaml", modelYAML, os.ModePerm); err != nil {
-					log.Error().Msgf("error loading model: %s", err.Error())
+				modelDefinitionFilePath := filepath.Join(modelPath, md5Name) + ".yaml"
+				if err := os.WriteFile(modelDefinitionFilePath, modelYAML, os.ModePerm); err != nil {
+					log.Error().Err(err).Str("filepath", modelDefinitionFilePath).Msg("error loading model: %s")
 				}
 			} else {
 				log.Warn().Msgf("[startup] failed resolving model '%s'", url)

pkg/utils/config.go

@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/rs/zerolog/log"
+)
+
+func SaveConfig(filePath, fileName string, obj any) {
+	file, err := json.MarshalIndent(obj, "", " ")
+	if err != nil {
+		log.Error().Err(err).Msg("failed to JSON marshal the uploadedFiles")
+	}
+
+	absolutePath := filepath.Join(filePath, fileName)
+	err = os.WriteFile(absolutePath, file, 0644)
+	if err != nil {
+		log.Error().Err(err).Str("filepath", absolutePath).Msg("failed to save configuration file")
+	}
+}
+
+func LoadConfig(filePath, fileName string, obj interface{}) {
+	uploadFilePath := filepath.Join(filePath, fileName)
+
+	_, err := os.Stat(uploadFilePath)
+	if os.IsNotExist(err) {
+		log.Debug().Msgf("No configuration file found at %s", uploadFilePath)
+		return
+	}
+
+	file, err := os.ReadFile(uploadFilePath)
+	if err != nil {
+		log.Error().Err(err).Str("filepath", uploadFilePath).Msg("failed to read file")
+	} else {
+		err = json.Unmarshal(file, &obj)
+		if err != nil {
+			log.Error().Err(err).Str("filepath", uploadFilePath).Msg("failed to parse file as JSON")
+		}
+	}
+}

swagger/docs.go

@@ -0,0 +1,801 @@
+// Code generated by swaggo/swag. DO NOT EDIT.
+
+package swagger
+
+import "github.com/swaggo/swag"
+
+const docTemplate = `{
+    "schemes": {{ marshal .Schemes }},
+    "swagger": "2.0",
+    "info": {
+        "description": "{{escape .Description}}",
+        "title": "{{.Title}}",
+        "contact": {
+            "name": "LocalAI",
+            "url": "https://localai.io"
+        },
+        "license": {
+            "name": "MIT",
+            "url": "https://raw.githubusercontent.com/mudler/LocalAI/master/LICENSE"
+        },
+        "version": "{{.Version}}"
+    },
+    "host": "{{.Host}}",
+    "basePath": "{{.BasePath}}",
+    "paths": {
+        "/v1/assistants": {
+            "post": {
+                "summary": "Create an assistant with a model and instructions.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/openai.AssistantRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/openai.Assistant"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/audio/speech": {
+            "post": {
+                "summary": "Generates audio from the input text.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.TTSRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/audio/transcriptions": {
+            "post": {
+                "consumes": [
+                    "multipart/form-data"
+                ],
+                "summary": "Transcribes audio into the input language.",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "model",
+                        "name": "model",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "file",
+                        "description": "file",
+                        "name": "file",
+                        "in": "formData",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "object",
+                            "additionalProperties": {
+                                "type": "string"
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/chat/completions": {
+            "post": {
+                "summary": "Generate a chat completions for a given prompt and model.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/completions": {
+            "post": {
+                "summary": "Generate completions for a given prompt and model.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/embeddings": {
+            "post": {
+                "summary": "Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/images/generations": {
+            "post": {
+                "summary": "Creates an image given a prompt.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/text-to-speech/{voice-id}": {
+            "post": {
+                "summary": "Generates audio from the input text.",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Account ID",
+                        "name": "voice-id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.TTSRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "definitions": {
+        "grammar.Argument": {
+            "type": "object",
+            "properties": {
+                "properties": {
+                    "type": "object",
+                    "additionalProperties": true
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.Function": {
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "parameters": {
+                    "type": "object",
+                    "additionalProperties": true
+                }
+            }
+        },
+        "grammar.FunctionName": {
+            "type": "object",
+            "properties": {
+                "const": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.Item": {
+            "type": "object",
+            "properties": {
+                "properties": {
+                    "$ref": "#/definitions/grammar.Properties"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.JSONFunctionStructure": {
+            "type": "object",
+            "properties": {
+                "$defs": {
+                    "type": "object",
+                    "additionalProperties": true
+                },
+                "anyOf": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Item"
+                    }
+                },
+                "oneOf": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Item"
+                    }
+                }
+            }
+        },
+        "grammar.Properties": {
+            "type": "object",
+            "properties": {
+                "arguments": {
+                    "$ref": "#/definitions/grammar.Argument"
+                },
+                "function": {
+                    "$ref": "#/definitions/grammar.FunctionName"
+                }
+            }
+        },
+        "grammar.Tool": {
+            "type": "object",
+            "properties": {
+                "function": {
+                    "$ref": "#/definitions/grammar.Function"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "openai.Assistant": {
+            "type": "object",
+            "properties": {
+                "created": {
+                    "description": "The time at which the assistant was created.",
+                    "type": "integer"
+                },
+                "description": {
+                    "description": "The description of the assistant.",
+                    "type": "string"
+                },
+                "file_ids": {
+                    "description": "A list of file IDs attached to this assistant.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "description": "The unique identifier of the assistant.",
+                    "type": "string"
+                },
+                "instructions": {
+                    "description": "The system instructions that the assistant uses.",
+                    "type": "string"
+                },
+                "metadata": {
+                    "description": "Set of key-value pairs attached to the assistant.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "model": {
+                    "description": "The model ID used by the assistant.",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "The name of the assistant.",
+                    "type": "string"
+                },
+                "object": {
+                    "description": "Object type, which is \"assistant\".",
+                    "type": "string"
+                },
+                "tools": {
+                    "description": "A list of tools enabled on the assistant.",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/openai.Tool"
+                    }
+                }
+            }
+        },
+        "openai.AssistantRequest": {
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "file_ids": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "instructions": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "model": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "tools": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/openai.Tool"
+                    }
+                }
+            }
+        },
+        "openai.Tool": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "$ref": "#/definitions/openai.ToolType"
+                }
+            }
+        },
+        "openai.ToolType": {
+            "type": "string",
+            "enum": [
+                "code_interpreter",
+                "retrieval",
+                "function"
+            ],
+            "x-enum-varnames": [
+                "CodeInterpreter",
+                "Retrieval",
+                "Function"
+            ]
+        },
+        "schema.ChatCompletionResponseFormat": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Choice": {
+            "type": "object",
+            "properties": {
+                "delta": {
+                    "$ref": "#/definitions/schema.Message"
+                },
+                "finish_reason": {
+                    "type": "string"
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "message": {
+                    "$ref": "#/definitions/schema.Message"
+                },
+                "text": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.FunctionCall": {
+            "type": "object",
+            "properties": {
+                "arguments": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Item": {
+            "type": "object",
+            "properties": {
+                "b64_json": {
+                    "type": "string"
+                },
+                "embedding": {
+                    "type": "array",
+                    "items": {
+                        "type": "number"
+                    }
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "object": {
+                    "type": "string"
+                },
+                "url": {
+                    "description": "Images",
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Message": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "description": "The message content"
+                },
+                "function_call": {
+                    "description": "A result of a function call"
+                },
+                "name": {
+                    "description": "The message name (used for tools calls)",
+                    "type": "string"
+                },
+                "role": {
+                    "description": "The message role",
+                    "type": "string"
+                },
+                "string_content": {
+                    "type": "string"
+                },
+                "string_images": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "tool_calls": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ToolCall"
+                    }
+                }
+            }
+        },
+        "schema.OpenAIRequest": {
+            "type": "object",
+            "required": [
+                "file"
+            ],
+            "properties": {
+                "backend": {
+                    "type": "string"
+                },
+                "batch": {
+                    "description": "Custom parameters - not present in the OpenAI API",
+                    "type": "integer"
+                },
+                "clip_skip": {
+                    "description": "Diffusers",
+                    "type": "integer"
+                },
+                "echo": {
+                    "type": "boolean"
+                },
+                "file": {
+                    "description": "whisper",
+                    "type": "string"
+                },
+                "frequency_penalty": {
+                    "type": "number"
+                },
+                "function_call": {
+                    "description": "might be a string or an object"
+                },
+                "functions": {
+                    "description": "A list of available functions to call",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Function"
+                    }
+                },
+                "grammar": {
+                    "description": "A grammar to constrain the LLM output",
+                    "type": "string"
+                },
+                "grammar_json_functions": {
+                    "$ref": "#/definitions/grammar.JSONFunctionStructure"
+                },
+                "ignore_eos": {
+                    "type": "boolean"
+                },
+                "input": {},
+                "instruction": {
+                    "description": "Edit endpoint",
+                    "type": "string"
+                },
+                "language": {
+                    "description": "Also part of the OpenAI official spec",
+                    "type": "string"
+                },
+                "max_tokens": {
+                    "type": "integer"
+                },
+                "messages": {
+                    "description": "Messages is read only by chat/completion API calls",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Message"
+                    }
+                },
+                "mode": {
+                    "description": "Image (not supported by OpenAI)",
+                    "type": "integer"
+                },
+                "model": {
+                    "description": "Also part of the OpenAI official spec",
+                    "type": "string"
+                },
+                "model_base_name": {
+                    "description": "AutoGPTQ",
+                    "type": "string"
+                },
+                "n": {
+                    "description": "Also part of the OpenAI official spec. use it for returning multiple results",
+                    "type": "integer"
+                },
+                "n_keep": {
+                    "type": "integer"
+                },
+                "negative_prompt": {
+                    "type": "string"
+                },
+                "negative_prompt_scale": {
+                    "type": "number"
+                },
+                "presence_penalty": {
+                    "type": "number"
+                },
+                "prompt": {
+                    "description": "Prompt is read only by completion/image API calls"
+                },
+                "repeat_penalty": {
+                    "type": "number"
+                },
+                "response_format": {
+                    "description": "whisper/image",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/schema.ChatCompletionResponseFormat"
+                        }
+                    ]
+                },
+                "rope_freq_base": {
+                    "type": "number"
+                },
+                "rope_freq_scale": {
+                    "type": "number"
+                },
+                "seed": {
+                    "type": "integer"
+                },
+                "size": {
+                    "description": "image",
+                    "type": "string"
+                },
+                "step": {
+                    "type": "integer"
+                },
+                "stop": {},
+                "stream": {
+                    "type": "boolean"
+                },
+                "temperature": {
+                    "type": "number"
+                },
+                "tfz": {
+                    "type": "number"
+                },
+                "tokenizer": {
+                    "description": "RWKV (?)",
+                    "type": "string"
+                },
+                "tool_choice": {},
+                "tools": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Tool"
+                    }
+                },
+                "top_k": {
+                    "type": "integer"
+                },
+                "top_p": {
+                    "description": "Common options between all the API calls, part of the OpenAI spec",
+                    "type": "number"
+                },
+                "typical_p": {
+                    "type": "number"
+                },
+                "use_fast_tokenizer": {
+                    "description": "AutoGPTQ",
+                    "type": "boolean"
+                }
+            }
+        },
+        "schema.OpenAIResponse": {
+            "type": "object",
+            "properties": {
+                "choices": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Choice"
+                    }
+                },
+                "created": {
+                    "type": "integer"
+                },
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Item"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "model": {
+                    "type": "string"
+                },
+                "object": {
+                    "type": "string"
+                },
+                "usage": {
+                    "$ref": "#/definitions/schema.OpenAIUsage"
+                }
+            }
+        },
+        "schema.OpenAIUsage": {
+            "type": "object",
+            "properties": {
+                "completion_tokens": {
+                    "type": "integer"
+                },
+                "prompt_tokens": {
+                    "type": "integer"
+                },
+                "total_tokens": {
+                    "type": "integer"
+                }
+            }
+        },
+        "schema.TTSRequest": {
+            "type": "object",
+            "properties": {
+                "backend": {
+                    "type": "string"
+                },
+                "input": {
+                    "type": "string"
+                },
+                "model": {
+                    "type": "string"
+                },
+                "voice": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.ToolCall": {
+            "type": "object",
+            "properties": {
+                "function": {
+                    "$ref": "#/definitions/schema.FunctionCall"
+                },
+                "id": {
+                    "type": "string"
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        }
+    },
+    "securityDefinitions": {
+        "BearerAuth": {
+            "type": "apiKey",
+            "name": "Authorization",
+            "in": "header"
+        }
+    }
+}`
+
+// SwaggerInfo holds exported Swagger Info so clients can modify it
+var SwaggerInfo = &swag.Spec{
+	Version:          "2.0.0",
+	Host:             "",
+	BasePath:         "/",
+	Schemes:          []string{},
+	Title:            "LocalAI API",
+	Description:      "The LocalAI Rest API.",
+	InfoInstanceName: "swagger",
+	SwaggerTemplate:  docTemplate,
+	LeftDelim:        "{{",
+	RightDelim:       "}}",
+}
+
+func init() {
+	swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
+}

swagger/swagger.json

@@ -0,0 +1,775 @@
+{
+    "swagger": "2.0",
+    "info": {
+        "description": "The LocalAI Rest API.",
+        "title": "LocalAI API",
+        "contact": {
+            "name": "LocalAI",
+            "url": "https://localai.io"
+        },
+        "license": {
+            "name": "MIT",
+            "url": "https://raw.githubusercontent.com/mudler/LocalAI/master/LICENSE"
+        },
+        "version": "2.0.0"
+    },
+    "basePath": "/",
+    "paths": {
+        "/v1/assistants": {
+            "post": {
+                "summary": "Create an assistant with a model and instructions.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/openai.AssistantRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/openai.Assistant"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/audio/speech": {
+            "post": {
+                "summary": "Generates audio from the input text.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.TTSRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/audio/transcriptions": {
+            "post": {
+                "consumes": [
+                    "multipart/form-data"
+                ],
+                "summary": "Transcribes audio into the input language.",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "model",
+                        "name": "model",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "file",
+                        "description": "file",
+                        "name": "file",
+                        "in": "formData",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "object",
+                            "additionalProperties": {
+                                "type": "string"
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/chat/completions": {
+            "post": {
+                "summary": "Generate a chat completions for a given prompt and model.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/completions": {
+            "post": {
+                "summary": "Generate completions for a given prompt and model.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/embeddings": {
+            "post": {
+                "summary": "Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/images/generations": {
+            "post": {
+                "summary": "Creates an image given a prompt.",
+                "parameters": [
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.OpenAIResponse"
+                        }
+                    }
+                }
+            }
+        },
+        "/v1/text-to-speech/{voice-id}": {
+            "post": {
+                "summary": "Generates audio from the input text.",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Account ID",
+                        "name": "voice-id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "query params",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/schema.TTSRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "definitions": {
+        "grammar.Argument": {
+            "type": "object",
+            "properties": {
+                "properties": {
+                    "type": "object",
+                    "additionalProperties": true
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.Function": {
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "parameters": {
+                    "type": "object",
+                    "additionalProperties": true
+                }
+            }
+        },
+        "grammar.FunctionName": {
+            "type": "object",
+            "properties": {
+                "const": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.Item": {
+            "type": "object",
+            "properties": {
+                "properties": {
+                    "$ref": "#/definitions/grammar.Properties"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "grammar.JSONFunctionStructure": {
+            "type": "object",
+            "properties": {
+                "$defs": {
+                    "type": "object",
+                    "additionalProperties": true
+                },
+                "anyOf": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Item"
+                    }
+                },
+                "oneOf": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Item"
+                    }
+                }
+            }
+        },
+        "grammar.Properties": {
+            "type": "object",
+            "properties": {
+                "arguments": {
+                    "$ref": "#/definitions/grammar.Argument"
+                },
+                "function": {
+                    "$ref": "#/definitions/grammar.FunctionName"
+                }
+            }
+        },
+        "grammar.Tool": {
+            "type": "object",
+            "properties": {
+                "function": {
+                    "$ref": "#/definitions/grammar.Function"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "openai.Assistant": {
+            "type": "object",
+            "properties": {
+                "created": {
+                    "description": "The time at which the assistant was created.",
+                    "type": "integer"
+                },
+                "description": {
+                    "description": "The description of the assistant.",
+                    "type": "string"
+                },
+                "file_ids": {
+                    "description": "A list of file IDs attached to this assistant.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "description": "The unique identifier of the assistant.",
+                    "type": "string"
+                },
+                "instructions": {
+                    "description": "The system instructions that the assistant uses.",
+                    "type": "string"
+                },
+                "metadata": {
+                    "description": "Set of key-value pairs attached to the assistant.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "model": {
+                    "description": "The model ID used by the assistant.",
+                    "type": "string"
+                },
+                "name": {
+                    "description": "The name of the assistant.",
+                    "type": "string"
+                },
+                "object": {
+                    "description": "Object type, which is \"assistant\".",
+                    "type": "string"
+                },
+                "tools": {
+                    "description": "A list of tools enabled on the assistant.",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/openai.Tool"
+                    }
+                }
+            }
+        },
+        "openai.AssistantRequest": {
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "file_ids": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "instructions": {
+                    "type": "string"
+                },
+                "metadata": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "model": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "tools": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/openai.Tool"
+                    }
+                }
+            }
+        },
+        "openai.Tool": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "$ref": "#/definitions/openai.ToolType"
+                }
+            }
+        },
+        "openai.ToolType": {
+            "type": "string",
+            "enum": [
+                "code_interpreter",
+                "retrieval",
+                "function"
+            ],
+            "x-enum-varnames": [
+                "CodeInterpreter",
+                "Retrieval",
+                "Function"
+            ]
+        },
+        "schema.ChatCompletionResponseFormat": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Choice": {
+            "type": "object",
+            "properties": {
+                "delta": {
+                    "$ref": "#/definitions/schema.Message"
+                },
+                "finish_reason": {
+                    "type": "string"
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "message": {
+                    "$ref": "#/definitions/schema.Message"
+                },
+                "text": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.FunctionCall": {
+            "type": "object",
+            "properties": {
+                "arguments": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Item": {
+            "type": "object",
+            "properties": {
+                "b64_json": {
+                    "type": "string"
+                },
+                "embedding": {
+                    "type": "array",
+                    "items": {
+                        "type": "number"
+                    }
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "object": {
+                    "type": "string"
+                },
+                "url": {
+                    "description": "Images",
+                    "type": "string"
+                }
+            }
+        },
+        "schema.Message": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "description": "The message content"
+                },
+                "function_call": {
+                    "description": "A result of a function call"
+                },
+                "name": {
+                    "description": "The message name (used for tools calls)",
+                    "type": "string"
+                },
+                "role": {
+                    "description": "The message role",
+                    "type": "string"
+                },
+                "string_content": {
+                    "type": "string"
+                },
+                "string_images": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "tool_calls": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ToolCall"
+                    }
+                }
+            }
+        },
+        "schema.OpenAIRequest": {
+            "type": "object",
+            "required": [
+                "file"
+            ],
+            "properties": {
+                "backend": {
+                    "type": "string"
+                },
+                "batch": {
+                    "description": "Custom parameters - not present in the OpenAI API",
+                    "type": "integer"
+                },
+                "clip_skip": {
+                    "description": "Diffusers",
+                    "type": "integer"
+                },
+                "echo": {
+                    "type": "boolean"
+                },
+                "file": {
+                    "description": "whisper",
+                    "type": "string"
+                },
+                "frequency_penalty": {
+                    "type": "number"
+                },
+                "function_call": {
+                    "description": "might be a string or an object"
+                },
+                "functions": {
+                    "description": "A list of available functions to call",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Function"
+                    }
+                },
+                "grammar": {
+                    "description": "A grammar to constrain the LLM output",
+                    "type": "string"
+                },
+                "grammar_json_functions": {
+                    "$ref": "#/definitions/grammar.JSONFunctionStructure"
+                },
+                "ignore_eos": {
+                    "type": "boolean"
+                },
+                "input": {},
+                "instruction": {
+                    "description": "Edit endpoint",
+                    "type": "string"
+                },
+                "language": {
+                    "description": "Also part of the OpenAI official spec",
+                    "type": "string"
+                },
+                "max_tokens": {
+                    "type": "integer"
+                },
+                "messages": {
+                    "description": "Messages is read only by chat/completion API calls",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Message"
+                    }
+                },
+                "mode": {
+                    "description": "Image (not supported by OpenAI)",
+                    "type": "integer"
+                },
+                "model": {
+                    "description": "Also part of the OpenAI official spec",
+                    "type": "string"
+                },
+                "model_base_name": {
+                    "description": "AutoGPTQ",
+                    "type": "string"
+                },
+                "n": {
+                    "description": "Also part of the OpenAI official spec. use it for returning multiple results",
+                    "type": "integer"
+                },
+                "n_keep": {
+                    "type": "integer"
+                },
+                "negative_prompt": {
+                    "type": "string"
+                },
+                "negative_prompt_scale": {
+                    "type": "number"
+                },
+                "presence_penalty": {
+                    "type": "number"
+                },
+                "prompt": {
+                    "description": "Prompt is read only by completion/image API calls"
+                },
+                "repeat_penalty": {
+                    "type": "number"
+                },
+                "response_format": {
+                    "description": "whisper/image",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/schema.ChatCompletionResponseFormat"
+                        }
+                    ]
+                },
+                "rope_freq_base": {
+                    "type": "number"
+                },
+                "rope_freq_scale": {
+                    "type": "number"
+                },
+                "seed": {
+                    "type": "integer"
+                },
+                "size": {
+                    "description": "image",
+                    "type": "string"
+                },
+                "step": {
+                    "type": "integer"
+                },
+                "stop": {},
+                "stream": {
+                    "type": "boolean"
+                },
+                "temperature": {
+                    "type": "number"
+                },
+                "tfz": {
+                    "type": "number"
+                },
+                "tokenizer": {
+                    "description": "RWKV (?)",
+                    "type": "string"
+                },
+                "tool_choice": {},
+                "tools": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/grammar.Tool"
+                    }
+                },
+                "top_k": {
+                    "type": "integer"
+                },
+                "top_p": {
+                    "description": "Common options between all the API calls, part of the OpenAI spec",
+                    "type": "number"
+                },
+                "typical_p": {
+                    "type": "number"
+                },
+                "use_fast_tokenizer": {
+                    "description": "AutoGPTQ",
+                    "type": "boolean"
+                }
+            }
+        },
+        "schema.OpenAIResponse": {
+            "type": "object",
+            "properties": {
+                "choices": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Choice"
+                    }
+                },
+                "created": {
+                    "type": "integer"
+                },
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Item"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "model": {
+                    "type": "string"
+                },
+                "object": {
+                    "type": "string"
+                },
+                "usage": {
+                    "$ref": "#/definitions/schema.OpenAIUsage"
+                }
+            }
+        },
+        "schema.OpenAIUsage": {
+            "type": "object",
+            "properties": {
+                "completion_tokens": {
+                    "type": "integer"
+                },
+                "prompt_tokens": {
+                    "type": "integer"
+                },
+                "total_tokens": {
+                    "type": "integer"
+                }
+            }
+        },
+        "schema.TTSRequest": {
+            "type": "object",
+            "properties": {
+                "backend": {
+                    "type": "string"
+                },
+                "input": {
+                    "type": "string"
+                },
+                "model": {
+                    "type": "string"
+                },
+                "voice": {
+                    "type": "string"
+                }
+            }
+        },
+        "schema.ToolCall": {
+            "type": "object",
+            "properties": {
+                "function": {
+                    "$ref": "#/definitions/schema.FunctionCall"
+                },
+                "id": {
+                    "type": "string"
+                },
+                "index": {
+                    "type": "integer"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        }
+    },
+    "securityDefinitions": {
+        "BearerAuth": {
+            "type": "apiKey",
+            "name": "Authorization",
+            "in": "header"
+        }
+    }
+}

swagger/swagger.yaml

@@ -0,0 +1,518 @@
+basePath: /
+definitions:
+  grammar.Argument:
+    properties:
+      properties:
+        additionalProperties: true
+        type: object
+      type:
+        type: string
+    type: object
+  grammar.Function:
+    properties:
+      description:
+        type: string
+      name:
+        type: string
+      parameters:
+        additionalProperties: true
+        type: object
+    type: object
+  grammar.FunctionName:
+    properties:
+      const:
+        type: string
+    type: object
+  grammar.Item:
+    properties:
+      properties:
+        $ref: '#/definitions/grammar.Properties'
+      type:
+        type: string
+    type: object
+  grammar.JSONFunctionStructure:
+    properties:
+      $defs:
+        additionalProperties: true
+        type: object
+      anyOf:
+        items:
+          $ref: '#/definitions/grammar.Item'
+        type: array
+      oneOf:
+        items:
+          $ref: '#/definitions/grammar.Item'
+        type: array
+    type: object
+  grammar.Properties:
+    properties:
+      arguments:
+        $ref: '#/definitions/grammar.Argument'
+      function:
+        $ref: '#/definitions/grammar.FunctionName'
+    type: object
+  grammar.Tool:
+    properties:
+      function:
+        $ref: '#/definitions/grammar.Function'
+      type:
+        type: string
+    type: object
+  openai.Assistant:
+    properties:
+      created:
+        description: The time at which the assistant was created.
+        type: integer
+      description:
+        description: The description of the assistant.
+        type: string
+      file_ids:
+        description: A list of file IDs attached to this assistant.
+        items:
+          type: string
+        type: array
+      id:
+        description: The unique identifier of the assistant.
+        type: string
+      instructions:
+        description: The system instructions that the assistant uses.
+        type: string
+      metadata:
+        additionalProperties:
+          type: string
+        description: Set of key-value pairs attached to the assistant.
+        type: object
+      model:
+        description: The model ID used by the assistant.
+        type: string
+      name:
+        description: The name of the assistant.
+        type: string
+      object:
+        description: Object type, which is "assistant".
+        type: string
+      tools:
+        description: A list of tools enabled on the assistant.
+        items:
+          $ref: '#/definitions/openai.Tool'
+        type: array
+    type: object
+  openai.AssistantRequest:
+    properties:
+      description:
+        type: string
+      file_ids:
+        items:
+          type: string
+        type: array
+      instructions:
+        type: string
+      metadata:
+        additionalProperties:
+          type: string
+        type: object
+      model:
+        type: string
+      name:
+        type: string
+      tools:
+        items:
+          $ref: '#/definitions/openai.Tool'
+        type: array
+    type: object
+  openai.Tool:
+    properties:
+      type:
+        $ref: '#/definitions/openai.ToolType'
+    type: object
+  openai.ToolType:
+    enum:
+    - code_interpreter
+    - retrieval
+    - function
+    type: string
+    x-enum-varnames:
+    - CodeInterpreter
+    - Retrieval
+    - Function
+  schema.ChatCompletionResponseFormat:
+    properties:
+      type:
+        type: string
+    type: object
+  schema.Choice:
+    properties:
+      delta:
+        $ref: '#/definitions/schema.Message'
+      finish_reason:
+        type: string
+      index:
+        type: integer
+      message:
+        $ref: '#/definitions/schema.Message'
+      text:
+        type: string
+    type: object
+  schema.FunctionCall:
+    properties:
+      arguments:
+        type: string
+      name:
+        type: string
+    type: object
+  schema.Item:
+    properties:
+      b64_json:
+        type: string
+      embedding:
+        items:
+          type: number
+        type: array
+      index:
+        type: integer
+      object:
+        type: string
+      url:
+        description: Images
+        type: string
+    type: object
+  schema.Message:
+    properties:
+      content:
+        description: The message content
+      function_call:
+        description: A result of a function call
+      name:
+        description: The message name (used for tools calls)
+        type: string
+      role:
+        description: The message role
+        type: string
+      string_content:
+        type: string
+      string_images:
+        items:
+          type: string
+        type: array
+      tool_calls:
+        items:
+          $ref: '#/definitions/schema.ToolCall'
+        type: array
+    type: object
+  schema.OpenAIRequest:
+    properties:
+      backend:
+        type: string
+      batch:
+        description: Custom parameters - not present in the OpenAI API
+        type: integer
+      clip_skip:
+        description: Diffusers
+        type: integer
+      echo:
+        type: boolean
+      file:
+        description: whisper
+        type: string
+      frequency_penalty:
+        type: number
+      function_call:
+        description: might be a string or an object
+      functions:
+        description: A list of available functions to call
+        items:
+          $ref: '#/definitions/grammar.Function'
+        type: array
+      grammar:
+        description: A grammar to constrain the LLM output
+        type: string
+      grammar_json_functions:
+        $ref: '#/definitions/grammar.JSONFunctionStructure'
+      ignore_eos:
+        type: boolean
+      input: {}
+      instruction:
+        description: Edit endpoint
+        type: string
+      language:
+        description: Also part of the OpenAI official spec
+        type: string
+      max_tokens:
+        type: integer
+      messages:
+        description: Messages is read only by chat/completion API calls
+        items:
+          $ref: '#/definitions/schema.Message'
+        type: array
+      mode:
+        description: Image (not supported by OpenAI)
+        type: integer
+      model:
+        description: Also part of the OpenAI official spec
+        type: string
+      model_base_name:
+        description: AutoGPTQ
+        type: string
+      "n":
+        description: Also part of the OpenAI official spec. use it for returning multiple
+          results
+        type: integer
+      n_keep:
+        type: integer
+      negative_prompt:
+        type: string
+      negative_prompt_scale:
+        type: number
+      presence_penalty:
+        type: number
+      prompt:
+        description: Prompt is read only by completion/image API calls
+      repeat_penalty:
+        type: number
+      response_format:
+        allOf:
+        - $ref: '#/definitions/schema.ChatCompletionResponseFormat'
+        description: whisper/image
+      rope_freq_base:
+        type: number
+      rope_freq_scale:
+        type: number
+      seed:
+        type: integer
+      size:
+        description: image
+        type: string
+      step:
+        type: integer
+      stop: {}
+      stream:
+        type: boolean
+      temperature:
+        type: number
+      tfz:
+        type: number
+      tokenizer:
+        description: RWKV (?)
+        type: string
+      tool_choice: {}
+      tools:
+        items:
+          $ref: '#/definitions/grammar.Tool'
+        type: array
+      top_k:
+        type: integer
+      top_p:
+        description: Common options between all the API calls, part of the OpenAI
+          spec
+        type: number
+      typical_p:
+        type: number
+      use_fast_tokenizer:
+        description: AutoGPTQ
+        type: boolean
+    required:
+    - file
+    type: object
+  schema.OpenAIResponse:
+    properties:
+      choices:
+        items:
+          $ref: '#/definitions/schema.Choice'
+        type: array
+      created:
+        type: integer
+      data:
+        items:
+          $ref: '#/definitions/schema.Item'
+        type: array
+      id:
+        type: string
+      model:
+        type: string
+      object:
+        type: string
+      usage:
+        $ref: '#/definitions/schema.OpenAIUsage'
+    type: object
+  schema.OpenAIUsage:
+    properties:
+      completion_tokens:
+        type: integer
+      prompt_tokens:
+        type: integer
+      total_tokens:
+        type: integer
+    type: object
+  schema.TTSRequest:
+    properties:
+      backend:
+        type: string
+      input:
+        type: string
+      model:
+        type: string
+      voice:
+        type: string
+    type: object
+  schema.ToolCall:
+    properties:
+      function:
+        $ref: '#/definitions/schema.FunctionCall'
+      id:
+        type: string
+      index:
+        type: integer
+      type:
+        type: string
+    type: object
+info:
+  contact:
+    name: LocalAI
+    url: https://localai.io
+  description: The LocalAI Rest API.
+  license:
+    name: MIT
+    url: https://raw.githubusercontent.com/mudler/LocalAI/master/LICENSE
+  title: LocalAI API
+  version: 2.0.0
+paths:
+  /v1/assistants:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/openai.AssistantRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/openai.Assistant'
+      summary: Create an assistant with a model and instructions.
+  /v1/audio/speech:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.TTSRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            type: string
+      summary: Generates audio from the input text.
+  /v1/audio/transcriptions:
+    post:
+      consumes:
+      - multipart/form-data
+      parameters:
+      - description: model
+        in: formData
+        name: model
+        required: true
+        type: string
+      - description: file
+        in: formData
+        name: file
+        required: true
+        type: file
+      responses:
+        "200":
+          description: Response
+          schema:
+            additionalProperties:
+              type: string
+            type: object
+      summary: Transcribes audio into the input language.
+  /v1/chat/completions:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.OpenAIRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.OpenAIResponse'
+      summary: Generate a chat completions for a given prompt and model.
+  /v1/completions:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.OpenAIRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.OpenAIResponse'
+      summary: Generate completions for a given prompt and model.
+  /v1/embeddings:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.OpenAIRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.OpenAIResponse'
+      summary: Get a vector representation of a given input that can be easily consumed
+        by machine learning models and algorithms.
+  /v1/images/generations:
+    post:
+      parameters:
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.OpenAIRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.OpenAIResponse'
+      summary: Creates an image given a prompt.
+  /v1/text-to-speech/{voice-id}:
+    post:
+      parameters:
+      - description: Account ID
+        in: path
+        name: voice-id
+        required: true
+        type: string
+      - description: query params
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/schema.TTSRequest'
+      responses:
+        "200":
+          description: Response
+          schema:
+            type: string
+      summary: Generates audio from the input text.
+securityDefinitions:
+  BearerAuth:
+    in: header
+    name: Authorization
+    type: apiKey
+swagger: "2.0"

tests/e2e-aio/e2e_suite_test.go

@@ -23,6 +23,7 @@ var containerImageTag = os.Getenv("LOCALAI_IMAGE_TAG")
 var modelsDir = os.Getenv("LOCALAI_MODELS_DIR")
 var apiPort = os.Getenv("LOCALAI_API_PORT")
 var apiEndpoint = os.Getenv("LOCALAI_API_ENDPOINT")
+var apiKey = os.Getenv("LOCALAI_API_KEY")
 
 func TestLocalAI(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -38,11 +39,11 @@ var _ = BeforeSuite(func() {
 	var defaultConfig openai.ClientConfig
 	if apiEndpoint == "" {
 		startDockerImage()
-		defaultConfig = openai.DefaultConfig("")
+		defaultConfig = openai.DefaultConfig(apiKey)
 		defaultConfig.BaseURL = "http://localhost:" + apiPort + "/v1"
 	} else {
 		fmt.Println("Default ", apiEndpoint)
-		defaultConfig = openai.DefaultConfig("")
+		defaultConfig = openai.DefaultConfig(apiKey)
 		defaultConfig.BaseURL = apiEndpoint
 	}
 

tests/e2e-aio/e2e_test.go

@@ -2,6 +2,7 @@ package e2e_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -9,8 +10,8 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
 	"github.com/sashabaranov/go-openai"
+	"github.com/sashabaranov/go-openai/jsonschema"
 )
 
 var _ = Describe("E2E test", func() {
@@ -40,6 +41,82 @@ var _ = Describe("E2E test", func() {
 				Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("4"), ContainSubstring("four")), fmt.Sprint(resp.Choices[0].Message.Content))
 			})
 		})
+
+		Context("function calls", func() {
+			It("correctly invoke", func() {
+				params := jsonschema.Definition{
+					Type: jsonschema.Object,
+					Properties: map[string]jsonschema.Definition{
+						"location": {
+							Type:        jsonschema.String,
+							Description: "The city and state, e.g. San Francisco, CA",
+						},
+						"unit": {
+							Type: jsonschema.String,
+							Enum: []string{"celsius", "fahrenheit"},
+						},
+					},
+					Required: []string{"location"},
+				}
+
+				f := openai.FunctionDefinition{
+					Name:        "get_current_weather",
+					Description: "Get the current weather in a given location",
+					Parameters:  params,
+				}
+				t := openai.Tool{
+					Type:     openai.ToolTypeFunction,
+					Function: &f,
+				}
+
+				dialogue := []openai.ChatCompletionMessage{
+					{Role: openai.ChatMessageRoleUser, Content: "What is the weather in Boston today?"},
+				}
+				resp, err := client.CreateChatCompletion(context.TODO(),
+					openai.ChatCompletionRequest{
+						Model:    openai.GPT4,
+						Messages: dialogue,
+						Tools:    []openai.Tool{t},
+					},
+				)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp))
+
+				msg := resp.Choices[0].Message
+				Expect(len(msg.ToolCalls)).To(Equal(1), fmt.Sprint(msg.ToolCalls))
+				Expect(msg.ToolCalls[0].Function.Name).To(Equal("get_current_weather"), fmt.Sprint(msg.ToolCalls[0].Function.Name))
+				Expect(msg.ToolCalls[0].Function.Arguments).To(ContainSubstring("Boston"), fmt.Sprint(msg.ToolCalls[0].Function.Arguments))
+			})
+		})
+		Context("json", func() {
+			It("correctly", func() {
+				model := "gpt-4"
+
+				req := openai.ChatCompletionRequest{
+					ResponseFormat: &openai.ChatCompletionResponseFormat{Type: openai.ChatCompletionResponseFormatTypeJSONObject},
+					Model:          model,
+					Messages: []openai.ChatCompletionMessage{
+						{
+
+							Role:    "user",
+							Content: "An animal with 'name', 'gender' and 'legs' fields",
+						},
+					},
+				}
+
+				resp, err := client.CreateChatCompletion(context.TODO(), req)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp))
+
+				var i map[string]interface{}
+				err = json.Unmarshal([]byte(resp.Choices[0].Message.Content), &i)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(i).To(HaveKey("name"))
+				Expect(i).To(HaveKey("gender"))
+				Expect(i).To(HaveKey("legs"))
+			})
+		})
+
 		Context("images", func() {
 			It("correctly", func() {
 				resp, err := client.CreateImage(context.TODO(),