build with cmake

wip shell
2025-12-25 00:30:56 -05:00 · 2023-08-21 18:36:31 -07:00 · 2023-08-21 09:24:22 -07:00
151 changed files with 13800 additions and 29580 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,7 @@
+build
+llama/build
+.venv
 .vscode
 ollama
 app
-dist
-scripts
-llm/llama.cpp/ggml
-llm/llama.cpp/gguf
-.env
+web
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@
 .swp
 dist
 ollama
-ggml-metal.metal
+/ggml-metal.metal
+build
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,10 +0,0 @@
-[submodule "llm/llama.cpp/ggml"]
-    path = llm/llama.cpp/ggml
-    url = https://github.com/ggerganov/llama.cpp.git
-    ignore = dirty
-    shallow = true
-[submodule "llm/llama.cpp/gguf"]
-    path = llm/llama.cpp/gguf
-    url = https://github.com/ggerganov/llama.cpp.git
-    ignore = dirty
-    shallow = true
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -0,0 +1,40 @@
+cmake_minimum_required(VERSION 3.14)  # 3.11 or later for FetchContent, but some features might require newer versions
+
+project(llama_cpp)
+
+include(FetchContent)
+
+FetchContent_Declare(
+    llama_cpp_gguf
+    GIT_REPOSITORY    https://github.com/ggerganov/llama.cpp.git
+    GIT_TAG           6381d4e
+)
+
+FetchContent_Declare(
+    llama_cpp_ggml
+    GIT_REPOSITORY    https://github.com/ggerganov/llama.cpp.git
+    GIT_TAG           dadbed9
+)
+
+FetchContent_MakeAvailable(llama_cpp_ggml)
+
+add_subdirectory(${llama_cpp_ggml_SOURCE_DIR}/examples EXCLUDE_FROM_ALL)
+add_executable(llama_cpp ${llama_cpp_ggml_SOURCE_DIR}/examples/server/server.cpp)
+include_directories(${llama_cpp_ggml_SOURCE_DIR})
+include_directories(${llama_cpp_ggml_SOURCE_DIR}/examples)
+target_compile_features(llama_cpp PRIVATE cxx_std_11)
+target_link_libraries(llama_cpp PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+
+if (APPLE)
+    add_executable(llama_cpp_metal ${llama_cpp_ggml_SOURCE_DIR}/examples/server/server.cpp)
+    target_compile_options(llama_cpp_metal PRIVATE -DLLAMA_STATIC=ON -DLLAMA_METAL=ON -DGGML_USE_METAL=1)
+    target_compile_features(llama_cpp_metal PRIVATE cxx_std_11)
+    target_link_libraries(llama_cpp_metal PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+    configure_file(${llama_cpp_SOURCE_DIR}/ggml-metal.metal ${CMAKE_BINARY_DIR}/ggml-metal.metal COPYONLY)
+else()
+    add_executable(llama_cpp_cublas ${llama_cpp_ggml_SOURCE_DIR}/examples/server/server.cpp)
+    target_compile_definitions(llama_cpp_cublas PRIVATE -DLLAMA_STATIC=ON -DLLAMA_CUBLAS=ON)
+    target_compile_options(llama_cpp_cublas PRIVATE -DLLAMA_CUBLAS=ON -DLLAMA_STATIC=ON)
+    target_compile_features(llama_cpp_cublas PRIVATE cxx_std_11)
+    target_link_libraries(llama_cpp_cublas PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+endif()
--- a/24
+++ b/24
@@ -1,23 +1,15 @@
-FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
-
-ARG TARGETARCH
-ARG GOFLAGS="'-ldflags=-w -s'"
-
+FROM golang:1.20
 WORKDIR /go/src/github.com/jmorganca/ollama
-RUN apt-get update && apt-get install -y git build-essential cmake
-ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
-RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
-
 COPY . .
-ENV GOARCH=$TARGETARCH
-ENV GOFLAGS=$GOFLAGS
-RUN /usr/local/go/bin/go generate ./... \
-    && /usr/local/go/bin/go build .
+RUN CGO_ENABLED=1 go build -ldflags '-linkmode external -extldflags "-static"' .

-FROM ubuntu:22.04
-RUN apt-get update && apt-get install -y ca-certificates
+FROM alpine
 COPY --from=0 /go/src/github.com/jmorganca/ollama/ollama /bin/ollama
 EXPOSE 11434
-ENV OLLAMA_HOST 0.0.0.0
+ARG USER=ollama
+ARG GROUP=ollama
+RUN addgroup -g 1000 $GROUP && adduser -u 1000 -DG $GROUP $USER
+USER $USER:$GROUP
 ENTRYPOINT ["/bin/ollama"]
+ENV OLLAMA_HOST 0.0.0.0
 CMD ["serve"]
--- a/Dockerfile.build
+++ b/Dockerfile.build
@@ -1,31 +0,0 @@
-# centos7 amd64 dependencies
-FROM --platform=linux/amd64 nvidia/cuda:11.3.1-devel-centos7 AS base-amd64
-RUN yum install -y https://repo.ius.io/ius-release-el7.rpm centos-release-scl && \
-    yum update -y && \
-    yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ git236 wget
-RUN wget "https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-x86_64.sh" -O cmake-installer.sh && chmod +x cmake-installer.sh && ./cmake-installer.sh --skip-license --prefix=/usr/local
-ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
-
-# centos8 arm64 dependencies
-FROM --platform=linux/arm64 nvidia/cuda-arm64:11.3.1-devel-centos8 AS base-arm64
-RUN sed -i -e 's/mirrorlist/#mirrorlist/g' -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-RUN yum install -y git cmake
-
-FROM base-${TARGETARCH}
-ARG TARGETARCH
-ARG GOFLAGS="'-ldflags -w -s'"
-
-# install go
-ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
-RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
-
-# build the final binary
-WORKDIR /go/src/github.com/jmorganca/ollama
-COPY . .
-
-ENV GOOS=linux
-ENV GOARCH=$TARGETARCH
-ENV GOFLAGS=$GOFLAGS
-
-RUN /usr/local/go/bin/go generate ./... && \
-    /usr/local/go/bin/go build .
--- a/README.md
+++ b/README.md
@@ -9,31 +9,19 @@

 [![Discord](https://dcbadge.vercel.app/api/server/ollama?style=flat&compact=true)](https://discord.gg/ollama)

-Get up and running with large language models locally.
+Run, create, and share large language models (LLMs).

-### macOS
+> Note: Ollama is in early preview. Please report any issues you find.

-[Download](https://ollama.ai/download/Ollama-darwin.zip)
+## Download

-### Windows
-
-Coming soon!
-
-### Linux & WSL2
-
-```
-curl https://ollama.ai/install.sh | sh
-```
-
-[Manual install instructions](https://github.com/jmorganca/ollama/blob/main/docs/linux.md)
-
-### Docker
-
-See the official [Docker image](https://hub.docker.com/r/ollama/ollama).
+- [Download](https://ollama.ai/download) for macOS
+- Download for Windows and Linux (coming soon)
+- Build [from source](#building)

 ## Quickstart

-To run and chat with [Llama 2](https://ollama.ai/library/llama2):
+To run and chat with [Llama 2](https://ai.meta.com/llama), the new model by Meta:

 ```
 ollama run llama2
@@ -41,59 +29,53 @@ ollama run llama2

 ## Model library

-Ollama supports a list of open-source models available on [ollama.ai/library](https://ollama.ai/library 'ollama model library')
+Ollama supports a list of open-source models available on [ollama.ai/library](https://ollama.ai/library "ollama model library") 

-Here are some example open-source models that can be downloaded:
+Here are some example open-source models that can be downloaded: 

-| Model              | Parameters | Size  | Download                       |
-| ------------------ | ---------- | ----- | ------------------------------ |
-| Mistral            | 7B         | 4.1GB | `ollama run mistral`           |
-| Llama 2            | 7B         | 3.8GB | `ollama run llama2`            |
-| Code Llama         | 7B         | 3.8GB | `ollama run codellama`         |
-| Llama 2 Uncensored | 7B         | 3.8GB | `ollama run llama2-uncensored` |
-| Llama 2 13B        | 13B        | 7.3GB | `ollama run llama2:13b`        |
-| Llama 2 70B        | 70B        | 39GB  | `ollama run llama2:70b`        |
-| Orca Mini          | 3B         | 1.9GB | `ollama run orca-mini`         |
-| Vicuna             | 7B         | 3.8GB | `ollama run vicuna`            |
+| Model                    | Parameters | Size  | Download                        |
+| ------------------------ | ---------- | ----- | ------------------------------- |
+| Llama2                   | 7B         | 3.8GB | `ollama pull llama2`            |
+| Llama2 13B               | 13B        | 7.3GB | `ollama pull llama2:13b`        |
+| Llama2 70B               | 70B        | 39GB  | `ollama pull llama2:70b`        |
+| Llama2 Uncensored        | 7B         | 3.8GB | `ollama pull llama2-uncensored` |
+| Orca Mini                | 3B         | 1.9GB | `ollama pull orca-mini`         |
+| Vicuna                   | 7B         | 3.8GB | `ollama pull vicuna`            |
+| Nous-Hermes              | 7B         | 3.8GB | `ollama pull nous-hermes`       |
+| Nous-Hermes 13B          | 13B        | 7.3GB | `ollama pull nous-hermes:13b`   |
+| Wizard Vicuna Uncensored | 13B        | 7.3GB | `ollama pull wizard-vicuna`     |

 > Note: You should have at least 8 GB of RAM to run the 3B models, 16 GB to run the 7B models, and 32 GB to run the 13B models.

-## Customize your own model
+## Examples

-### Import from GGUF
+### Run a model

-Ollama supports importing GGUF models in the Modelfile:
+```
+ollama run llama2
+>>> hi
+Hello! How can I help you today?
+```

-1. Create a file named `Modelfile`, with a `FROM` instruction with the local filepath to the model you want to import.
+For multiline input, you can wrap text with `"""`:

-   ```
-   FROM ./vicuna-33b.Q4_0.gguf
-   ```
+```
+>>> """Hello,
+... world!
+... """
+I'm a basic program that prints the famous "Hello, world!" message to the console.
+```

-2. Create the model in Ollama
+### Create a custom model

-   ```
-   ollama create example -f Modelfile
-   ```
-
-3. Run the model
-
-   ```
-   ollama run example
-   ```
-
-### Import from PyTorch or Safetensors
-
-See the [guide](docs/import.md) on importing models for more information.
-
-### Customize a prompt
-
-Models from the Ollama library can be customized with a prompt. The example
+Pull a base model:

 ```
 ollama pull llama2
 ```

+> To update a model to the latest version, run `ollama pull llama2` again. The model will be updated (if necessary).
+
 Create a `Modelfile`:

 ```
@@ -117,85 +99,44 @@ ollama run mario
 Hello! It's your friend Mario.
 ```

-For more examples, see the [examples](examples) directory. For more information on working with a Modelfile, see the [Modelfile](docs/modelfile.md) documentation.
+For more examples, see the [examples](./examples) directory. For more information on creating a Modelfile, see the [Modelfile](./docs/modelfile.md) documentation.

-## CLI Reference
-
-### Create a model
-
-`ollama create` is used to create a model from a Modelfile.
-
-### Pull a model
+### Pull a model from the registry

 ```
-ollama pull llama2
+ollama pull orca
 ```

-> This command can also be used to update a local model. Only the diff will be pulled.
-
-### Remove a model
-
-```
-ollama rm llama2
-```
-
-### Copy a model
-
-```
-ollama cp llama2 my-llama2
-```
-
-### Multiline input
-
-For multiline input, you can wrap text with `"""`:
-
-```
->>> """Hello,
-... world!
-... """
-I'm a basic program that prints the famous "Hello, world!" message to the console.
-```
-
-### Pass in prompt as arguments
-
-```
-$ ollama run llama2 "summarize this file:" "$(cat README.md)"
- Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
-```
-
-### List models on your computer
+### Listing local models

 ```
 ollama list
 ```

-### Start Ollama
+## Model packages

-`ollama serve` is used when you want to start ollama without running the desktop application.
+### Overview
+
+Ollama bundles model weights, configuration, and data into a single package, defined by a [Modelfile](./docs/modelfile.md).
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" height="480" srcset="https://github.com/jmorganca/ollama/assets/251292/2fd96b5f-191b-45c1-9668-941cfad4eb70">
+  <img alt="logo" height="480" src="https://github.com/jmorganca/ollama/assets/251292/2fd96b5f-191b-45c1-9668-941cfad4eb70">
+</picture>

 ## Building

-Install `cmake` and `go`:
-
 ```
-brew install cmake
-brew install go
-```
-
-Then generate dependencies and build:
-
-```
-go generate ./...
 go build .
 ```

-Next, start the server:
+To run it start the server:

 ```
-./ollama serve
+./ollama serve &
 ```

-Finally, in a separate shell, run a model:
+Finally, run a model!

 ```
 ./ollama run llama2
@@ -203,7 +144,7 @@ Finally, in a separate shell, run a model:

 ## REST API

-See the [API documentation](docs/api.md) for all endpoints.
+> See the [API documentation](./docs/api.md) for all endpoints.

 Ollama has an API for running and managing models. For example to generate text from a model:

@@ -214,22 +155,12 @@ curl -X POST http://localhost:11434/api/generate -d '{
 }'
 ```

-## Community Integrations
+## Tools using Ollama

- [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa)
- [LlamaIndex](https://gpt-index.readthedocs.io/en/stable/examples/llm/ollama.html)
- [Raycast extension](https://github.com/MassimilianoPasquini97/raycast_ollama)
- [Discollama](https://github.com/mxyng/discollama) (Discord bot inside the Ollama discord channel)
- [Continue](https://github.com/continuedev/continue)
- [Obsidian Ollama plugin](https://github.com/hinterdupfinger/obsidian-ollama)
- [Dagger Chatbot](https://github.com/samalba/dagger-chatbot)
- [LiteLLM](https://github.com/BerriAI/litellm)
- [Discord AI Bot](https://github.com/mekb-turtle/discord-ai-bot)
- [Chatbot UI](https://github.com/ivanfioravanti/chatbot-ollama)
- [HTML UI](https://github.com/rtcfirefly/ollama-ui)
- [Typescript UI](https://github.com/ollama-interface/Ollama-Gui?tab=readme-ov-file)
- [Dumbar](https://github.com/JerrySievert/Dumbar)
- [Emacs client](https://github.com/zweifisch/ollama)
- [oterm](https://github.com/ggozad/oterm)
- [Ellama Emacs client](https://github.com/s-kostyaev/ellama)
- [OllamaSharp for .NET](https://github.com/awaescher/OllamaSharp)
+- [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with a question-answering [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa).
+- [Continue](https://github.com/continuedev/continue) - embeds Ollama inside Visual Studio Code. The extension lets you highlight code to add to the prompt, ask questions in the sidebar, and generate code inline.
+- [LiteLLM](https://github.com/BerriAI/litellm) a lightweight python package to simplify LLM API calls
+- [Discord AI Bot](https://github.com/mekb-turtle/discord-ai-bot) - interact with Ollama as a chatbot on Discord.
+- [Raycast Ollama](https://github.com/MassimilianoPasquini97/raycast_ollama) - Raycast extension to use Ollama for local llama inference on Raycast.
+- [Simple HTML UI for Ollama](https://github.com/rtcfirefly/ollama-ui)
+- [Emacs client](https://github.com/zweifisch/ollama) for Ollama
--- a/api/client.go
+++ b/api/client.go
@@ -7,28 +7,26 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"runtime"
 	"strings"
-
-	"github.com/jmorganca/ollama/format"
-	"github.com/jmorganca/ollama/version"
 )

-const DefaultHost = "127.0.0.1:11434"
+const DefaultHost = "localhost:11434"

-var envHost = os.Getenv("OLLAMA_HOST")
+var (
+	envHost = os.Getenv("OLLAMA_HOST")
+)

 type Client struct {
-	base *url.URL
-	http http.Client
+	Base    url.URL
+	HTTP    http.Client
+	Headers http.Header
 }

 func checkError(resp *http.Response, body []byte) error {
-	if resp.StatusCode < http.StatusBadRequest {
+	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
 		return nil
 	}

@@ -43,46 +41,34 @@ func checkError(resp *http.Response, body []byte) error {
 	return apiError
 }

-func ClientFromEnvironment() (*Client, error) {
-	scheme, hostport, ok := strings.Cut(os.Getenv("OLLAMA_HOST"), "://")
-	if !ok {
-		scheme, hostport = "http", os.Getenv("OLLAMA_HOST")
+// Host returns the default host to use for the client. It is determined in the following order:
+// 1. The OLLAMA_HOST environment variable
+// 2. The default host (localhost:11434)
+func Host() string {
+	if envHost != "" {
+		return envHost
+	}
+	return DefaultHost
+}
+
+// FromEnv creates a new client using Host() as the host. An error is returns
+// if the host is invalid.
+func FromEnv() (*Client, error) {
+	h := Host()
+	if !strings.HasPrefix(h, "http://") && !strings.HasPrefix(h, "https://") {
+		h = "http://" + h
 	}

-	host, port, err := net.SplitHostPort(hostport)
+	u, err := url.Parse(h)
 	if err != nil {
-		host, port = "127.0.0.1", "11434"
-		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
-			host = ip.String()
-		} else if hostport != "" {
-			host = hostport
-		}
+		return nil, fmt.Errorf("could not parse host: %w", err)
 	}

-	client := Client{
-		base: &url.URL{
-			Scheme: scheme,
-			Host:   net.JoinHostPort(host, port),
-		},
+	if u.Port() == "" {
+		u.Host += ":11434"
 	}

-	mockRequest, err := http.NewRequest("HEAD", client.base.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	proxyURL, err := http.ProxyFromEnvironment(mockRequest)
-	if err != nil {
-		return nil, err
-	}
-
-	client.http = http.Client{
-		Transport: &http.Transport{
-			Proxy: http.ProxyURL(proxyURL),
-		},
-	}
-
-	return &client, nil
+	return &Client{Base: *u, HTTP: http.Client{}}, nil
 }

 func (c *Client) do(ctx context.Context, method, path string, reqData, respData any) error {
@@ -97,17 +83,21 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData
 		reqBody = bytes.NewReader(data)
 	}

-	requestURL := c.base.JoinPath(path)
-	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), reqBody)
+	url := c.Base.JoinPath(path).String()
+
+	req, err := http.NewRequestWithContext(ctx, method, url, reqBody)
 	if err != nil {
 		return err
 	}

-	request.Header.Set("Content-Type", "application/json")
-	request.Header.Set("Accept", "application/json")
-	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")

-	respObj, err := c.http.Do(request)
+	for k, v := range c.Headers {
+		req.Header[k] = v
+	}
+
+	respObj, err := c.HTTP.Do(req)
 	if err != nil {
 		return err
 	}
@@ -130,8 +120,6 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData
 	return nil
 }

-const maxBufferSize = 512 * format.KiloByte
-
 func (c *Client) stream(ctx context.Context, method, path string, data any, fn func([]byte) error) error {
 	var buf *bytes.Buffer
 	if data != nil {
@@ -143,26 +131,21 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
 		buf = bytes.NewBuffer(bts)
 	}

-	requestURL := c.base.JoinPath(path)
-	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), buf)
+	request, err := http.NewRequestWithContext(ctx, method, c.Base.JoinPath(path).String(), buf)
 	if err != nil {
 		return err
 	}

 	request.Header.Set("Content-Type", "application/json")
-	request.Header.Set("Accept", "application/x-ndjson")
-	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
+	request.Header.Set("Accept", "application/json")

-	response, err := c.http.Do(request)
+	response, err := http.DefaultClient.Do(request)
 	if err != nil {
 		return err
 	}
 	defer response.Body.Close()

 	scanner := bufio.NewScanner(response.Body)
-	// increase the buffer size to avoid running out of space
-	scanBuf := make([]byte, 0, maxBufferSize)
-	scanner.Buffer(scanBuf, maxBufferSize)
 	for scanner.Scan() {
 		var errorResponse struct {
 			Error string `json:"error,omitempty"`
@@ -177,7 +160,7 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
 			return fmt.Errorf(errorResponse.Error)
 		}

-		if response.StatusCode >= http.StatusBadRequest {
+		if response.StatusCode >= 400 {
 			return StatusError{
 				StatusCode:   response.StatusCode,
 				Status:       response.Status,
@@ -267,14 +250,6 @@ func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
 	return nil
 }

-func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, error) {
-	var resp ShowResponse
-	if err := c.do(ctx, http.MethodPost, "/api/show", req, &resp); err != nil {
-		return nil, err
-	}
-	return &resp, nil
-}
-
 func (c *Client) Heartbeat(ctx context.Context) error {
 	if err := c.do(ctx, http.MethodHead, "/", nil, nil); err != nil {
 		return err
--- a/api/client.py
+++ b/api/client.py
@@ -1,225 +0,0 @@
-import os
-import json
-import requests
-
-BASE_URL = os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
-
-# Generate a response for a given prompt with a provided model. This is a streaming endpoint, so will be a series of responses.
-# The final response object will include statistics and additional data from the request. Use the callback function to override
-# the default handler.
-def generate(model_name, prompt, system=None, template=None, context=None, options=None, callback=None):
-    try:
-        url = f"{BASE_URL}/api/generate"
-        payload = {
-            "model": model_name, 
-            "prompt": prompt, 
-            "system": system, 
-            "template": template, 
-            "context": context, 
-            "options": options
-        }
-        
-        # Remove keys with None values
-        payload = {k: v for k, v in payload.items() if v is not None}
-        
-        with requests.post(url, json=payload, stream=True) as response:
-            response.raise_for_status()
-            
-            # Creating a variable to hold the context history of the final chunk
-            final_context = None
-            
-            # Variable to hold concatenated response strings if no callback is provided
-            full_response = ""
-
-            # Iterating over the response line by line and displaying the details
-            for line in response.iter_lines():
-                if line:
-                    # Parsing each line (JSON chunk) and extracting the details
-                    chunk = json.loads(line)
-                    
-                    # If a callback function is provided, call it with the chunk
-                    if callback:
-                        callback(chunk)
-                    else:
-                        # If this is not the last chunk, add the "response" field value to full_response and print it
-                        if not chunk.get("done"):
-                            response_piece = chunk.get("response", "")
-                            full_response += response_piece
-                            print(response_piece, end="", flush=True)
-                    
-                    # Check if it's the last chunk (done is true)
-                    if chunk.get("done"):
-                        final_context = chunk.get("context")
-            
-            # Return the full response and the final context
-            return full_response, final_context
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return None, None
-
-# Create a model from a Modelfile. Use the callback function to override the default handler.
-def create(model_name, model_path, callback=None):
-    try:
-        url = f"{BASE_URL}/api/create"
-        payload = {"name": model_name, "path": model_path}
-        
-        # Making a POST request with the stream parameter set to True to handle streaming responses
-        with requests.post(url, json=payload, stream=True) as response:
-            response.raise_for_status()
-
-            # Iterating over the response line by line and displaying the status
-            for line in response.iter_lines():
-                if line:
-                    # Parsing each line (JSON chunk) and extracting the status
-                    chunk = json.loads(line)
-
-                    if callback:
-                        callback(chunk)
-                    else:
-                        print(f"Status: {chunk.get('status')}")
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-
-# Pull a model from a the model registry. Cancelled pulls are resumed from where they left off, and multiple
-# calls to will share the same download progress. Use the callback function to override the default handler.
-def pull(model_name, insecure=False, callback=None):
-    try:
-        url = f"{BASE_URL}/api/pull"
-        payload = {
-            "name": model_name,
-            "insecure": insecure
-        }
-
-        # Making a POST request with the stream parameter set to True to handle streaming responses
-        with requests.post(url, json=payload, stream=True) as response:
-            response.raise_for_status()
-
-            # Iterating over the response line by line and displaying the details
-            for line in response.iter_lines():
-                if line:
-                    # Parsing each line (JSON chunk) and extracting the details
-                    chunk = json.loads(line)
-
-                    # If a callback function is provided, call it with the chunk
-                    if callback:
-                        callback(chunk)
-                    else:
-                        # Print the status message directly to the console
-                        print(chunk.get('status', ''), end='', flush=True)
-                    
-                    # If there's layer data, you might also want to print that (adjust as necessary)
-                    if 'digest' in chunk:
-                        print(f" - Digest: {chunk['digest']}", end='', flush=True)
-                        print(f" - Total: {chunk['total']}", end='', flush=True)
-                        print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
-                    else:
-                        print()
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-
-# Push a model to the model registry. Use the callback function to override the default handler.
-def push(model_name, insecure=False, callback=None):
-    try:
-        url = f"{BASE_URL}/api/push"
-        payload = {
-            "name": model_name,
-            "insecure": insecure
-        }
-
-        # Making a POST request with the stream parameter set to True to handle streaming responses
-        with requests.post(url, json=payload, stream=True) as response:
-            response.raise_for_status()
-
-            # Iterating over the response line by line and displaying the details
-            for line in response.iter_lines():
-                if line:
-                    # Parsing each line (JSON chunk) and extracting the details
-                    chunk = json.loads(line)
-
-                    # If a callback function is provided, call it with the chunk
-                    if callback:
-                        callback(chunk)
-                    else:
-                        # Print the status message directly to the console
-                        print(chunk.get('status', ''), end='', flush=True)
-                    
-                    # If there's layer data, you might also want to print that (adjust as necessary)
-                    if 'digest' in chunk:
-                        print(f" - Digest: {chunk['digest']}", end='', flush=True)
-                        print(f" - Total: {chunk['total']}", end='', flush=True)
-                        print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
-                    else:
-                        print()
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-
-# List models that are available locally.
-def list():
-    try:
-        response = requests.get(f"{BASE_URL}/api/tags")
-        response.raise_for_status()
-        data = response.json()
-        models = data.get('models', [])
-        return models
-
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return None
-
-# Copy a model. Creates a model with another name from an existing model.
-def copy(source, destination):
-    try:
-        # Create the JSON payload
-        payload = {
-            "source": source,
-            "destination": destination
-        }
-        
-        response = requests.post(f"{BASE_URL}/api/copy", json=payload)
-        response.raise_for_status()
-        
-        # If the request was successful, return a message indicating that the copy was successful
-        return "Copy successful"
-
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return None
-
-# Delete a model and its data.
-def delete(model_name):
-    try:
-        url = f"{BASE_URL}/api/delete"
-        payload = {"name": model_name}
-        response = requests.delete(url, json=payload)
-        response.raise_for_status()
-        return "Delete successful"
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return None
-
-# Show info about a model.
-def show(model_name):
-    try:
-        url = f"{BASE_URL}/api/show"
-        payload = {"name": model_name}
-        response = requests.post(url, json=payload)
-        response.raise_for_status()
-        
-        # Parse the JSON response and return it
-        data = response.json()
-        return data
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return None
-
-def heartbeat():
-    try:
-        url = f"{BASE_URL}/"
-        response = requests.head(url)
-        response.raise_for_status()
-        return "Ollama is running"
-    except requests.exceptions.RequestException as e:
-        print(f"An error occurred: {e}")
-        return "Ollama is not running"
-
-
--- a/api/types.go
+++ b/api/types.go
@@ -3,9 +3,11 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"math"
 	"os"
 	"reflect"
+	"runtime"
 	"strings"
 	"time"
 )
@@ -36,7 +38,6 @@ type GenerateRequest struct {
 	System   string `json:"system"`
 	Template string `json:"template"`
 	Context  []int  `json:"context,omitempty"`
-	Stream   *bool  `json:"stream,omitempty"`

 	Options map[string]interface{} `json:"options"`
 }
@@ -53,27 +54,14 @@ type EmbeddingResponse struct {
 }

 type CreateRequest struct {
-	Name   string `json:"name"`
-	Path   string `json:"path"`
-	Stream *bool  `json:"stream,omitempty"`
+	Name string `json:"name"`
+	Path string `json:"path"`
 }

 type DeleteRequest struct {
 	Name string `json:"name"`
 }

-type ShowRequest struct {
-	Name string `json:"name"`
-}
-
-type ShowResponse struct {
-	License    string `json:"license,omitempty"`
-	Modelfile  string `json:"modelfile,omitempty"`
-	Parameters string `json:"parameters,omitempty"`
-	Template   string `json:"template,omitempty"`
-	System     string `json:"system,omitempty"`
-}
-
 type CopyRequest struct {
 	Source      string `json:"source"`
 	Destination string `json:"destination"`
@@ -84,14 +72,13 @@ type PullRequest struct {
 	Insecure bool   `json:"insecure,omitempty"`
 	Username string `json:"username"`
 	Password string `json:"password"`
-	Stream   *bool  `json:"stream,omitempty"`
 }

 type ProgressResponse struct {
 	Status    string `json:"status"`
 	Digest    string `json:"digest,omitempty"`
-	Total     int64  `json:"total,omitempty"`
-	Completed int64  `json:"completed,omitempty"`
+	Total     int    `json:"total,omitempty"`
+	Completed int    `json:"completed,omitempty"`
 }

 type PushRequest struct {
@@ -99,18 +86,16 @@ type PushRequest struct {
 	Insecure bool   `json:"insecure,omitempty"`
 	Username string `json:"username"`
 	Password string `json:"password"`
-	Stream   *bool  `json:"stream,omitempty"`
 }

 type ListResponse struct {
-	Models []ModelResponse `json:"models"`
+	Models []ListResponseModel `json:"models"`
 }

-type ModelResponse struct {
+type ListResponseModel struct {
 	Name       string    `json:"name"`
 	ModifiedAt time.Time `json:"modified_at"`
-	Size       int64     `json:"size"`
-	Digest     string    `json:"digest"`
+	Size       int       `json:"size"`
 }

 type TokenResponse struct {
@@ -120,13 +105,15 @@ type TokenResponse struct {
 type GenerateResponse struct {
 	Model     string    `json:"model"`
 	CreatedAt time.Time `json:"created_at"`
-	Response  string    `json:"response"`
+	Response  string    `json:"response,omitempty"`

 	Done    bool  `json:"done"`
 	Context []int `json:"context,omitempty"`

 	TotalDuration      time.Duration `json:"total_duration,omitempty"`
 	LoadDuration       time.Duration `json:"load_duration,omitempty"`
+	SampleCount        int           `json:"sample_count,omitempty"`
+	SampleDuration     time.Duration `json:"sample_duration,omitempty"`
 	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
 	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
 	EvalCount          int           `json:"eval_count,omitempty"`
@@ -142,6 +129,15 @@ func (r *GenerateResponse) Summary() {
 		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
 	}

+	if r.SampleCount > 0 {
+		fmt.Fprintf(os.Stderr, "sample count:         %d token(s)\n", r.SampleCount)
+	}
+
+	if r.SampleDuration > 0 {
+		fmt.Fprintf(os.Stderr, "sample duration:      %s\n", r.SampleDuration)
+		fmt.Fprintf(os.Stderr, "sample rate:          %.2f tokens/s\n", float64(r.SampleCount)/r.SampleDuration.Seconds())
+	}
+
 	if r.PromptEvalCount > 0 {
 		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
 	}
@@ -161,10 +157,15 @@ func (r *GenerateResponse) Summary() {
 	}
 }

-// Runner options which must be set when the model is loaded into memory
-type Runner struct {
-	UseNUMA            bool    `json:"numa,omitempty"`
+type Options struct {
+	Seed int `json:"seed,omitempty"`
+
+	// Backend options
+	UseNUMA bool `json:"numa,omitempty"`
+
+	// Model options
 	NumCtx             int     `json:"num_ctx,omitempty"`
+	NumKeep            int     `json:"num_keep,omitempty"`
 	NumBatch           int     `json:"num_batch,omitempty"`
 	NumGQA             int     `json:"num_gqa,omitempty"`
 	NumGPU             int     `json:"num_gpu,omitempty"`
@@ -178,33 +179,25 @@ type Runner struct {
 	EmbeddingOnly      bool    `json:"embedding_only,omitempty"`
 	RopeFrequencyBase  float32 `json:"rope_frequency_base,omitempty"`
 	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
-	NumThread          int     `json:"num_thread,omitempty"`
-}

-type Options struct {
-	Runner
-
-	// Predict options used at runtime
-	NumKeep          int      `json:"num_keep,omitempty"`
-	Seed             int      `json:"seed,omitempty"`
-	NumPredict       int      `json:"num_predict,omitempty"`
+	// Predict options
+	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
+	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
+	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
+	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
+	Temperature      float32  `json:"temperature,omitempty"`
 	TopK             int      `json:"top_k,omitempty"`
 	TopP             float32  `json:"top_p,omitempty"`
 	TFSZ             float32  `json:"tfs_z,omitempty"`
 	TypicalP         float32  `json:"typical_p,omitempty"`
-	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
-	Temperature      float32  `json:"temperature,omitempty"`
-	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
-	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
-	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
 	Mirostat         int      `json:"mirostat,omitempty"`
 	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
 	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
 	PenalizeNewline  bool     `json:"penalize_newline,omitempty"`
 	Stop             []string `json:"stop,omitempty"`
-}

-var ErrInvalidOpts = fmt.Errorf("invalid options")
+	NumThread int `json:"num_thread,omitempty"`
+}

 func (opts *Options) FromMap(m map[string]interface{}) error {
 	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
@@ -219,7 +212,6 @@ func (opts *Options) FromMap(m map[string]interface{}) error {
 		}
 	}

-	invalidOpts := []string{}
 	for key, val := range m {
 		if opt, ok := jsonOpts[key]; ok {
 			field := valueOpts.FieldByName(opt.Name)
@@ -237,39 +229,44 @@ func (opts *Options) FromMap(m map[string]interface{}) error {
 						// when JSON unmarshals numbers, it uses float64, not int
 						field.SetInt(int64(t))
 					default:
-						return fmt.Errorf("option %q must be of type integer", key)
+						log.Printf("could not convert model parameter %v to int, skipped", key)
 					}
 				case reflect.Bool:
 					val, ok := val.(bool)
 					if !ok {
-						return fmt.Errorf("option %q must be of type boolean", key)
+						log.Printf("could not convert model parameter %v to bool, skipped", key)
+						continue
 					}
 					field.SetBool(val)
 				case reflect.Float32:
 					// JSON unmarshals to float64
 					val, ok := val.(float64)
 					if !ok {
-						return fmt.Errorf("option %q must be of type float32", key)
+						log.Printf("could not convert model parameter %v to float32, skipped", key)
+						continue
 					}
 					field.SetFloat(val)
 				case reflect.String:
 					val, ok := val.(string)
 					if !ok {
-						return fmt.Errorf("option %q must be of type string", key)
+						log.Printf("could not convert model parameter %v to string, skipped", key)
+						continue
 					}
 					field.SetString(val)
 				case reflect.Slice:
 					// JSON unmarshals to []interface{}, not []string
 					val, ok := val.([]interface{})
 					if !ok {
-						return fmt.Errorf("option %q must be of type array", key)
+						log.Printf("could not convert model parameter %v to slice, skipped", key)
+						continue
 					}
 					// convert []interface{} to []string
 					slice := make([]string, len(val))
 					for i, item := range val {
 						str, ok := item.(string)
 						if !ok {
-							return fmt.Errorf("option %q must be of an array of strings", key)
+							log.Printf("could not convert model parameter %v to slice of strings, skipped", key)
+							continue
 						}
 						slice[i] = str
 					}
@@ -278,53 +275,45 @@ func (opts *Options) FromMap(m map[string]interface{}) error {
 					return fmt.Errorf("unknown type loading config params: %v", field.Kind())
 				}
 			}
-		} else {
-			invalidOpts = append(invalidOpts, key)
 		}
 	}
-
-	if len(invalidOpts) > 0 {
-		return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", "))
-	}
 	return nil
 }

 func DefaultOptions() Options {
 	return Options{
-		// options set on request to runner
-		NumPredict:       -1,
-		NumKeep:          -1,
+		Seed: -1,
+
+		UseNUMA: false,
+
+		NumCtx:             2048,
+		NumKeep:            -1,
+		NumBatch:           512,
+		NumGPU:             1,
+		NumGQA:             1,
+		LowVRAM:            false,
+		F16KV:              true,
+		UseMMap:            true,
+		UseMLock:           false,
+		RopeFrequencyBase:  10000.0,
+		RopeFrequencyScale: 1.0,
+		EmbeddingOnly:      true,
+
+		RepeatLastN:      64,
+		RepeatPenalty:    1.1,
+		FrequencyPenalty: 0.0,
+		PresencePenalty:  0.0,
 		Temperature:      0.8,
 		TopK:             40,
 		TopP:             0.9,
 		TFSZ:             1.0,
 		TypicalP:         1.0,
-		RepeatLastN:      64,
-		RepeatPenalty:    1.1,
-		PresencePenalty:  0.0,
-		FrequencyPenalty: 0.0,
 		Mirostat:         0,
 		MirostatTau:      5.0,
 		MirostatEta:      0.1,
 		PenalizeNewline:  true,
-		Seed:             -1,

-		Runner: Runner{
-			// options set when the model is loaded
-			NumCtx:             2048,
-			RopeFrequencyBase:  10000.0,
-			RopeFrequencyScale: 1.0,
-			NumBatch:           512,
-			NumGPU:             -1, // -1 here indicates that NumGPU should be set dynamically
-			NumGQA:             1,
-			NumThread:          0, // let the runtime decide
-			LowVRAM:            false,
-			F16KV:              true,
-			UseMLock:           false,
-			UseMMap:            true,
-			UseNUMA:            false,
-			EmbeddingOnly:      true,
-		},
+		NumThread: runtime.NumCPU(),
 	}
 }

--- a/app/forge.config.ts
+++ b/app/forge.config.ts
@@ -27,6 +27,7 @@ const config: ForgeConfig = {
      path.join(__dirname, './assets/iconDarkTemplate@2x.png'),
      path.join(__dirname, './assets/iconDarkUpdateTemplate.png'),
      path.join(__dirname, './assets/iconDarkUpdateTemplate@2x.png'),
+      ...(process.platform === 'darwin' ? ['../llm/ggml-metal.metal'] : []),
    ],
    ...(process.env.SIGN
      ? {
@@ -47,6 +48,16 @@ const config: ForgeConfig = {
  },
  rebuildConfig: {},
  makers: [new MakerSquirrel({}), new MakerZIP({}, ['darwin'])],
+  publishers: [
+    new PublisherGithub({
+      repository: {
+        name: 'ollama',
+        owner: 'jmorganca',
+      },
+      draft: false,
+      prerelease: true,
+    }),
+  ],
  hooks: {
    readPackageJson: async (_, packageJson) => {
      return { ...packageJson, version: process.env.VERSION || packageJson.version }
--- a/app/package-lock.json
+++ b/app/package-lock.json
--- a/app/package.json
+++ b/app/package.json
@@ -46,7 +46,7 @@
    "chmodr": "^1.2.0",
    "copy-webpack-plugin": "^11.0.0",
    "css-loader": "^6.8.1",
-    "electron": "25.9.2",
+    "electron": "25.2.0",
    "eslint": "^8.43.0",
    "eslint-plugin-import": "^2.27.5",
    "fork-ts-checker-webpack-plugin": "^7.3.0",
--- a/app/src/index.ts
+++ b/app/src/index.ts
@@ -5,7 +5,7 @@ import winston from 'winston'
 import 'winston-daily-rotate-file'
 import * as path from 'path'

-import { v4 as uuidv4 } from 'uuid'
+import { analytics, id } from './telemetry'
 import { installed } from './install'

 require('@electron/remote/main').initialize()
@@ -158,60 +158,17 @@ function restart() {
 app.on('before-quit', () => {
  if (proc) {
    proc.off('exit', restart)
-    proc.kill('SIGINT') // send SIGINT signal to the server, which also stops any loaded llms
+    proc.kill()
  }
 })

-const updateURL = `https://ollama.ai/api/update?os=${process.platform}&arch=${
-  process.arch
-}&version=${app.getVersion()}&id=${id()}`
-
-let latest = ''
-async function isNewReleaseAvailable() {
-  try {
-    const response = await fetch(updateURL)
-
-    if (!response.ok) {
-      return false
-    }
-
-    if (response.status === 204) {
-      return false
-    }
-
-    const data = await response.json()
-
-    const url = data?.url
-    if (!url) {
-      return false
-    }
-
-    if (latest === url) {
-      return false
-    }
-
-    latest = url
-
-    return true
-  } catch (error) {
-    logger.error(`update check failed - ${error}`)
-    return false
-  }
-}
-
-async function checkUpdate() {
-  const available = await isNewReleaseAvailable()
-  if (available) {
-    logger.info('checking for update')
-    autoUpdater.checkForUpdates()
-  }
-}
-
 function init() {
  if (app.isPackaged) {
-    checkUpdate()
+    heartbeat()
+    autoUpdater.checkForUpdates()
    setInterval(() => {
-      checkUpdate()
+      heartbeat()
+      autoUpdater.checkForUpdates()
    }, 60 * 60 * 1000)
  }

@@ -277,22 +234,28 @@ app.on('window-all-closed', () => {
  }
 })

-function id(): string {
-  const id = store.get('id') as string
+// In this file you can include the rest of your app's specific main process
+// code. You can also put them in separate files and import them here.
+let aid = ''
+try {
+  aid = id()
+} catch (e) {}

-  if (id) {
-    return id
-  }
+autoUpdater.setFeedURL({
+  url: `https://ollama.ai/api/update?os=${process.platform}&arch=${process.arch}&version=${app.getVersion()}&id=${aid}`,
+})

-  const uuid = uuidv4()
-  store.set('id', uuid)
-  return uuid
+async function heartbeat() {
+  analytics.track({
+    anonymousId: aid,
+    event: 'heartbeat',
+    properties: {
+      version: app.getVersion(),
+    },
+  })
 }

-autoUpdater.setFeedURL({ url: updateURL })
-
 autoUpdater.on('error', e => {
-  logger.error(`update check failed - ${e.message}`)
  console.error(`update check failed - ${e.message}`)
 })

--- a/app/src/telemetry.ts
+++ b/app/src/telemetry.ts
@@ -0,0 +1,19 @@
+import { Analytics } from '@segment/analytics-node'
+import { v4 as uuidv4 } from 'uuid'
+import Store from 'electron-store'
+
+const store = new Store()
+
+export const analytics = new Analytics({ writeKey: process.env.TELEMETRY_WRITE_KEY || '<empty>' })
+
+export function id(): string {
+  const id = store.get('id') as string
+
+  if (id) {
+    return id
+  }
+
+  const uuid = uuidv4()
+  store.set('id', uuid)
+  return uuid
+}
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -11,49 +11,27 @@ import (
 	"io"
 	"log"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
-	"os/signal"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
-	"syscall"
 	"time"

+	"github.com/chzyer/readline"
 	"github.com/dustin/go-humanize"
 	"github.com/olekukonko/tablewriter"
-	"github.com/pdevine/readline"
 	"github.com/spf13/cobra"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/term"

 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/format"
 	"github.com/jmorganca/ollama/progressbar"
 	"github.com/jmorganca/ollama/server"
-	"github.com/jmorganca/ollama/version"
 )

-type Painter struct {
-	IsMultiLine bool
-}
-
-func (p Painter) Paint(line []rune, _ int) []rune {
-	termType := os.Getenv("TERM")
-	if termType == "xterm-256color" && len(line) == 0 {
-		var prompt string
-		if p.IsMultiLine {
-			prompt = "Use \"\"\" to end multi-line input"
-		} else {
-			prompt = "Send a message (/? for help)"
-		}
-		return []rune(fmt.Sprintf("\033[38;5;245m%s\033[%dD\033[0m", prompt, len(prompt)))
-	}
-	// add a space and a backspace to prevent the cursor from walking up the screen
-	line = append(line, []rune(" \b")...)
-	return line
-}
-
 func CreateHandler(cmd *cobra.Command, args []string) error {
 	filename, _ := cmd.Flags().GetString("file")
 	filename, err := filepath.Abs(filename)
@@ -61,7 +39,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 		return err
 	}

-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -78,14 +56,20 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 				spinner.Stop()
 			}
 			currentDigest = resp.Digest
-			// pulling
-			bar = progressbar.DefaultBytes(
-				resp.Total,
-				resp.Status,
-			)
-			bar.Set64(resp.Completed)
+			switch {
+			case strings.Contains(resp.Status, "embeddings"):
+				bar = progressbar.Default(int64(resp.Total), resp.Status)
+				bar.Set(resp.Completed)
+			default:
+				// pulling
+				bar = progressbar.DefaultBytes(
+					int64(resp.Total),
+					resp.Status,
+				)
+				bar.Set(resp.Completed)
+			}
 		} else if resp.Digest == currentDigest && resp.Digest != "" {
-			bar.Set64(resp.Completed)
+			bar.Set(resp.Completed)
 		} else {
 			currentDigest = ""
 			if spinner != nil {
@@ -113,24 +97,26 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 }

 func RunHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
+	mp := server.ParseModelPath(args[0])
+	fp, err := mp.GetManifestPath(false)
 	if err != nil {
 		return err
 	}

-	models, err := client.List(context.Background())
-	if err != nil {
-		return err
-	}
+	_, err = os.Stat(fp)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		if err := pull(args[0], false); err != nil {
+			var apiStatusError api.StatusError
+			if !errors.As(err, &apiStatusError) {
+				return err
+			}

-	canonicalModelPath := server.ParseModelPath(args[0])
-	for _, model := range models.Models {
-		if model.Name == canonicalModelPath.GetShortTagname() {
-			return RunGenerate(cmd, args)
+			if apiStatusError.StatusCode != http.StatusBadGateway {
+				return err
+			}
 		}
-	}
-
-	if err := PullHandler(cmd, args); err != nil {
+	case err != nil:
 		return err
 	}

@@ -138,7 +124,7 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 }

 func PushHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -156,13 +142,13 @@ func PushHandler(cmd *cobra.Command, args []string) error {
 		if resp.Digest != currentDigest && resp.Digest != "" {
 			currentDigest = resp.Digest
 			bar = progressbar.DefaultBytes(
-				resp.Total,
+				int64(resp.Total),
 				fmt.Sprintf("pushing %s...", resp.Digest[7:19]),
 			)

-			bar.Set64(resp.Completed)
+			bar.Set(resp.Completed)
 		} else if resp.Digest == currentDigest && resp.Digest != "" {
-			bar.Set64(resp.Completed)
+			bar.Set(resp.Completed)
 		} else {
 			currentDigest = ""
 			fmt.Println(resp.Status)
@@ -182,7 +168,7 @@ func PushHandler(cmd *cobra.Command, args []string) error {
 }

 func ListHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -196,12 +182,12 @@ func ListHandler(cmd *cobra.Command, args []string) error {

 	for _, m := range models.Models {
 		if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
-			data = append(data, []string{m.Name, m.Digest[:12], humanize.Bytes(uint64(m.Size)), format.HumanTime(m.ModifiedAt, "Never")})
+			data = append(data, []string{m.Name, humanize.Bytes(uint64(m.Size)), format.HumanTime(m.ModifiedAt, "Never")})
 		}
 	}

 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"NAME", "ID", "SIZE", "MODIFIED"})
+	table.SetHeader([]string{"NAME", "SIZE", "MODIFIED"})
 	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
 	table.SetAlignment(tablewriter.ALIGN_LEFT)
 	table.SetHeaderLine(false)
@@ -215,101 +201,21 @@ func ListHandler(cmd *cobra.Command, args []string) error {
 }

 func DeleteHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}

-	for _, name := range args {
-		req := api.DeleteRequest{Name: name}
-		if err := client.Delete(context.Background(), &req); err != nil {
-			return err
-		}
-		fmt.Printf("deleted '%s'\n", name)
-	}
-	return nil
-}
-
-func ShowHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
-	if err != nil {
+	req := api.DeleteRequest{Name: args[0]}
+	if err := client.Delete(context.Background(), &req); err != nil {
 		return err
 	}
-
-	if len(args) != 1 {
-		return errors.New("missing model name")
-	}
-
-	license, errLicense := cmd.Flags().GetBool("license")
-	modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
-	parameters, errParams := cmd.Flags().GetBool("parameters")
-	system, errSystem := cmd.Flags().GetBool("system")
-	template, errTemplate := cmd.Flags().GetBool("template")
-
-	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
-		if boolErr != nil {
-			return errors.New("error retrieving flags")
-		}
-	}
-
-	flagsSet := 0
-	showType := ""
-
-	if license {
-		flagsSet++
-		showType = "license"
-	}
-
-	if modelfile {
-		flagsSet++
-		showType = "modelfile"
-	}
-
-	if parameters {
-		flagsSet++
-		showType = "parameters"
-	}
-
-	if system {
-		flagsSet++
-		showType = "system"
-	}
-
-	if template {
-		flagsSet++
-		showType = "template"
-	}
-
-	if flagsSet > 1 {
-		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
-	} else if flagsSet == 0 {
-		return errors.New("one of '--license', '--modelfile', '--parameters', '--system', or '--template' must be specified")
-	}
-
-	req := api.ShowRequest{Name: args[0]}
-	resp, err := client.Show(context.Background(), &req)
-	if err != nil {
-		return err
-	}
-
-	switch showType {
-	case "license":
-		fmt.Println(resp.License)
-	case "modelfile":
-		fmt.Println(resp.Modelfile)
-	case "parameters":
-		fmt.Println(resp.Parameters)
-	case "system":
-		fmt.Println(resp.System)
-	case "template":
-		fmt.Println(resp.Template)
-	}
-
+	fmt.Printf("deleted '%s'\n", args[0])
 	return nil
 }

 func CopyHandler(cmd *cobra.Command, args []string) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -332,7 +238,7 @@ func PullHandler(cmd *cobra.Command, args []string) error {
 }

 func pull(model string, insecure bool) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -345,13 +251,13 @@ func pull(model string, insecure bool) error {
 		if resp.Digest != currentDigest && resp.Digest != "" {
 			currentDigest = resp.Digest
 			bar = progressbar.DefaultBytes(
-				resp.Total,
+				int64(resp.Total),
 				fmt.Sprintf("pulling %s...", resp.Digest[7:19]),
 			)

-			bar.Set64(resp.Completed)
+			bar.Set(resp.Completed)
 		} else if resp.Digest == currentDigest && resp.Digest != "" {
-			bar.Set64(resp.Completed)
+			bar.Set(resp.Completed)
 		} else {
 			currentDigest = ""
 			fmt.Println(resp.Status)
@@ -374,20 +280,7 @@ func pull(model string, insecure bool) error {
 func RunGenerate(cmd *cobra.Command, args []string) error {
 	if len(args) > 1 {
 		// join all args into a single prompt
-		wordWrap := false
-		if term.IsTerminal(int(os.Stdout.Fd())) {
-			wordWrap = true
-		}
-
-		nowrap, err := cmd.Flags().GetBool("nowordwrap")
-		if err != nil {
-			return err
-		}
-		if nowrap {
-			wordWrap = false
-		}
-
-		return generate(cmd, args[0], strings.Join(args[1:], " "), wordWrap)
+		return generate(cmd, args[0], strings.Join(args[1:], " "))
 	}

 	if readline.IsTerminal(int(os.Stdin.Fd())) {
@@ -399,140 +292,111 @@ func RunGenerate(cmd *cobra.Command, args []string) error {

 type generateContextKey string

-func generate(cmd *cobra.Command, model, prompt string, wordWrap bool) error {
-	client, err := api.ClientFromEnvironment()
-	if err != nil {
-		return err
-	}
-
-	spinner := NewSpinner("")
-	go spinner.Spin(60 * time.Millisecond)
-
-	var latest api.GenerateResponse
-
-	generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int)
-	if !ok {
-		generateContext = []int{}
-	}
-
-	termWidth, _, err := term.GetSize(int(0))
-	if err != nil {
-		wordWrap = false
-	}
-
-	cancelCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT)
-	var abort bool
-
-	go func() {
-		<-sigChan
-		cancel()
-		abort = true
-	}()
-
-	var currentLineLength int
-	var wordBuffer string
-
-	request := api.GenerateRequest{Model: model, Prompt: prompt, Context: generateContext}
-	fn := func(response api.GenerateResponse) error {
-		if !spinner.IsFinished() {
-			spinner.Finish()
+func generate(cmd *cobra.Command, model, prompt string) error {
+	if len(strings.TrimSpace(prompt)) > 0 {
+		client, err := api.FromEnv()
+		if err != nil {
+			return err
 		}

-		latest = response
+		spinner := NewSpinner("")
+		go spinner.Spin(60 * time.Millisecond)

-		if wordWrap {
-			for _, ch := range response.Response {
-				if currentLineLength+1 > termWidth-5 {
-					// backtrack the length of the last word and clear to the end of the line
-					fmt.Printf("\x1b[%dD\x1b[K\n", len(wordBuffer))
-					fmt.Printf("%s%c", wordBuffer, ch)
-					currentLineLength = len(wordBuffer) + 1
-				} else {
-					fmt.Print(string(ch))
-					currentLineLength += 1
+		var latest api.GenerateResponse

-					switch ch {
-					case ' ':
-						wordBuffer = ""
-					case '\n':
-						currentLineLength = 0
-					default:
-						wordBuffer += string(ch)
-					}
+		generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int)
+		if !ok {
+			generateContext = []int{}
+		}
+
+		request := api.GenerateRequest{Model: model, Prompt: prompt, Context: generateContext}
+		fn := func(response api.GenerateResponse) error {
+			if !spinner.IsFinished() {
+				spinner.Finish()
+			}
+
+			latest = response
+
+			fmt.Print(response.Response)
+			return nil
+		}
+
+		if err := client.Generate(context.Background(), &request, fn); err != nil {
+			if strings.Contains(err.Error(), "failed to load model") {
+				// tell the user to check the server log, if it exists locally
+				home, nestedErr := os.UserHomeDir()
+				if nestedErr != nil {
+					// return the original error
+					return err
+				}
+				logPath := filepath.Join(home, ".ollama", "logs", "server.log")
+				if _, nestedErr := os.Stat(logPath); nestedErr == nil {
+					err = fmt.Errorf("%w\nFor more details, check the error logs at %s", err, logPath)
 				}
 			}
-		} else {
-			fmt.Print(response.Response)
+			return err
 		}

-		return nil
-	}
-
-	if err := client.Generate(cancelCtx, &request, fn); err != nil {
-		if strings.Contains(err.Error(), "context canceled") && abort {
-			spinner.Finish()
-			return nil
-		}
-		return err
-	}
-	if prompt != "" {
 		fmt.Println()
 		fmt.Println()
-	}

-	if !latest.Done {
-		if abort {
-			return nil
+		if !latest.Done {
+			return errors.New("unexpected end of response")
 		}
-		return errors.New("unexpected end of response")
-	}

-	verbose, err := cmd.Flags().GetBool("verbose")
-	if err != nil {
-		return err
-	}
+		verbose, err := cmd.Flags().GetBool("verbose")
+		if err != nil {
+			return err
+		}

-	if verbose {
-		latest.Summary()
-	}
+		if verbose {
+			latest.Summary()
+		}

-	ctx := cmd.Context()
-	ctx = context.WithValue(ctx, generateContextKey("context"), latest.Context)
-	cmd.SetContext(ctx)
+		ctx := cmd.Context()
+		ctx = context.WithValue(ctx, generateContextKey("context"), latest.Context)
+		cmd.SetContext(ctx)
+	}

 	return nil
 }

+func showLayer(l *server.Layer) {
+	filename, err := server.GetBlobsPath(l.Digest)
+	if err != nil {
+		fmt.Println("Couldn't get layer's path")
+		return
+	}
+	bts, err := os.ReadFile(filename)
+	if err != nil {
+		fmt.Println("Couldn't read layer")
+		return
+	}
+	fmt.Println(string(bts))
+}
+
 func generateInteractive(cmd *cobra.Command, model string) error {
 	home, err := os.UserHomeDir()
 	if err != nil {
 		return err
 	}

-	// load the model
-	if err := generate(cmd, model, "", false); err != nil {
-		return err
-	}
-
 	completer := readline.NewPrefixCompleter(
 		readline.PcItem("/help"),
 		readline.PcItem("/list"),
 		readline.PcItem("/set",
 			readline.PcItem("history"),
 			readline.PcItem("nohistory"),
-			readline.PcItem("wordwrap"),
-			readline.PcItem("nowordwrap"),
 			readline.PcItem("verbose"),
 			readline.PcItem("quiet"),
+			readline.PcItem("mode",
+				readline.PcItem("vim"),
+				readline.PcItem("emacs"),
+				readline.PcItem("default"),
+			),
 		),
 		readline.PcItem("/show",
 			readline.PcItem("license"),
-			readline.PcItem("modelfile"),
-			readline.PcItem("parameters"),
 			readline.PcItem("system"),
 			readline.PcItem("template"),
 		),
@@ -541,41 +405,11 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 	)

 	usage := func() {
-		fmt.Fprintln(os.Stderr, "Available Commands:")
-		fmt.Fprintln(os.Stderr, "  /set         Set session variables")
-		fmt.Fprintln(os.Stderr, "  /show        Show model information")
-		fmt.Fprintln(os.Stderr, "  /bye         Exit")
-		fmt.Fprintln(os.Stderr, "  /?, /help    Help for a command")
-		fmt.Fprintln(os.Stderr, "")
-		fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
-		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "commands:")
+		fmt.Fprintln(os.Stderr, completer.Tree("  "))
 	}

-	usageSet := func() {
-		fmt.Fprintln(os.Stderr, "Available Commands:")
-		fmt.Fprintln(os.Stderr, "  /set history      Enable history")
-		fmt.Fprintln(os.Stderr, "  /set nohistory    Disable history")
-		fmt.Fprintln(os.Stderr, "  /set wordwrap     Enable wordwrap")
-		fmt.Fprintln(os.Stderr, "  /set nowordwrap   Disable wordwrap")
-		fmt.Fprintln(os.Stderr, "  /set verbose      Show LLM stats")
-		fmt.Fprintln(os.Stderr, "  /set quiet        Disable LLM stats")
-		fmt.Fprintln(os.Stderr, "")
-	}
-
-	usageShow := func() {
-		fmt.Fprintln(os.Stderr, "Available Commands:")
-		fmt.Fprintln(os.Stderr, "  /show license      Show model license")
-		fmt.Fprintln(os.Stderr, "  /show modelfile    Show Modelfile for this model")
-		fmt.Fprintln(os.Stderr, "  /show parameters   Show parameters for this model")
-		fmt.Fprintln(os.Stderr, "  /show system       Show system prompt")
-		fmt.Fprintln(os.Stderr, "  /show template     Show prompt template")
-		fmt.Fprintln(os.Stderr, "")
-	}
-
-	var painter Painter
-
 	config := readline.Config{
-		Painter:      &painter,
 		Prompt:       ">>> ",
 		HistoryFile:  filepath.Join(home, ".ollama", "history"),
 		AutoComplete: completer,
@@ -587,21 +421,6 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 	}
 	defer scanner.Close()

-	var wordWrap bool
-	termType := os.Getenv("TERM")
-	if termType == "xterm-256color" {
-		wordWrap = true
-	}
-
-	// override wrapping if the user turned it off
-	nowrap, err := cmd.Flags().GetBool("nowordwrap")
-	if err != nil {
-		return err
-	}
-	if nowrap {
-		wordWrap = false
-	}
-
 	var multiLineBuffer string
 	var isMultiLine bool

@@ -612,7 +431,7 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 			return nil
 		case errors.Is(err, readline.ErrInterrupt):
 			if line == "" {
-				fmt.Println("Use Ctrl-D or /bye to exit.")
+				return nil
 			}

 			continue
@@ -626,7 +445,6 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 		case isMultiLine:
 			if strings.HasSuffix(line, `"""`) {
 				isMultiLine = false
-				painter.IsMultiLine = isMultiLine
 				multiLineBuffer += strings.TrimSuffix(line, `"""`)
 				line = multiLineBuffer
 				multiLineBuffer = ""
@@ -637,7 +455,6 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 			}
 		case strings.HasPrefix(line, `"""`):
 			isMultiLine = true
-			painter.IsMultiLine = isMultiLine
 			multiLineBuffer = strings.TrimPrefix(line, `"""`) + " "
 			scanner.SetPrompt("... ")
 			continue
@@ -646,115 +463,94 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 			if err := ListHandler(cmd, args[1:]); err != nil {
 				return err
 			}
+
+			continue
 		case strings.HasPrefix(line, "/set"):
 			args := strings.Fields(line)
 			if len(args) > 1 {
 				switch args[1] {
 				case "history":
 					scanner.HistoryEnable()
+					continue
 				case "nohistory":
 					scanner.HistoryDisable()
-				case "wordwrap":
-					wordWrap = true
-					fmt.Println("Set 'wordwrap' mode.")
-				case "nowordwrap":
-					wordWrap = false
-					fmt.Println("Set 'nowordwrap' mode.")
+					continue
 				case "verbose":
 					cmd.Flags().Set("verbose", "true")
-					fmt.Println("Set 'verbose' mode.")
+					continue
 				case "quiet":
 					cmd.Flags().Set("verbose", "false")
-					fmt.Println("Set 'quiet' mode.")
+					continue
 				case "mode":
 					if len(args) > 2 {
 						switch args[2] {
 						case "vim":
 							scanner.SetVimMode(true)
+							continue
 						case "emacs", "default":
 							scanner.SetVimMode(false)
+							continue
 						default:
 							usage()
+							continue
 						}
 					} else {
 						usage()
+						continue
 					}
-				default:
-					fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
 				}
 			} else {
-				usageSet()
+				usage()
+				continue
 			}
 		case strings.HasPrefix(line, "/show"):
 			args := strings.Fields(line)
 			if len(args) > 1 {
-				client, err := api.ClientFromEnvironment()
+				mp := server.ParseModelPath(model)
+				manifest, err := server.GetManifest(mp)
 				if err != nil {
-					fmt.Println("error: couldn't connect to ollama server")
-					return err
+					fmt.Println("error: couldn't get a manifest for this model")
+					continue
 				}
-				resp, err := client.Show(cmd.Context(), &api.ShowRequest{Name: model})
-				if err != nil {
-					fmt.Println("error: couldn't get model")
-					return err
-				}
-
 				switch args[1] {
 				case "license":
-					if resp.License == "" {
-						fmt.Print("No license was specified for this model.\n\n")
-					} else {
-						fmt.Println(resp.License)
-					}
-				case "modelfile":
-					fmt.Println(resp.Modelfile)
-				case "parameters":
-					if resp.Parameters == "" {
-						fmt.Print("No parameters were specified for this model.\n\n")
-					} else {
-						fmt.Println(resp.Parameters)
+					for _, l := range manifest.Layers {
+						if l.MediaType == "application/vnd.ollama.image.license" {
+							showLayer(l)
+						}
 					}
+					continue
 				case "system":
-					if resp.System == "" {
-						fmt.Print("No system prompt was specified for this model.\n\n")
-					} else {
-						fmt.Println(resp.System)
+					for _, l := range manifest.Layers {
+						if l.MediaType == "application/vnd.ollama.image.system" {
+							showLayer(l)
+						}
 					}
+					continue
 				case "template":
-					if resp.Template == "" {
-						fmt.Print("No prompt template was specified for this model.\n\n")
-					} else {
-						fmt.Println(resp.Template)
+					for _, l := range manifest.Layers {
+						if l.MediaType == "application/vnd.ollama.image.template" {
+							showLayer(l)
+						}
 					}
+					continue
 				default:
-					fmt.Printf("Unknown command '/show %s'. Type /? for help\n", args[1])
-				}
-			} else {
-				usageShow()
-			}
-		case strings.HasPrefix(line, "/help"), strings.HasPrefix(line, "/?"):
-			args := strings.Fields(line)
-			if len(args) > 1 {
-				switch args[1] {
-				case "set", "/set":
-					usageSet()
-				case "show", "/show":
-					usageShow()
+					usage()
+					continue
 				}
 			} else {
 				usage()
+				continue
 			}
+		case line == "/help", line == "/?":
+			usage()
+			continue
 		case line == "/exit", line == "/bye":
 			return nil
-		case strings.HasPrefix(line, "/"):
-			args := strings.Fields(line)
-			fmt.Printf("Unknown command '%s'. Type /? for help\n", args[0])
 		}

-		if len(line) > 0 && line[0] != '/' {
-			if err := generate(cmd, model, line, wordWrap); err != nil {
-				return err
-			}
+		if err := generate(cmd, model, line); err != nil {
+			return err
 		}
 	}
 }
@@ -764,7 +560,7 @@ func generateBatch(cmd *cobra.Command, model string) error {
 	for scanner.Scan() {
 		prompt := scanner.Text()
 		fmt.Printf(">>> %s\n", prompt)
-		if err := generate(cmd, model, prompt, false); err != nil {
+		if err := generate(cmd, model, prompt); err != nil {
 			return err
 		}
 	}
@@ -773,19 +569,28 @@ func generateBatch(cmd *cobra.Command, model string) error {
 }

 func RunServer(cmd *cobra.Command, _ []string) error {
-	host, port, err := net.SplitHostPort(os.Getenv("OLLAMA_HOST"))
-	if err != nil {
-		host, port = "127.0.0.1", "11434"
-		if ip := net.ParseIP(strings.Trim(os.Getenv("OLLAMA_HOST"), "[]")); ip != nil {
-			host = ip.String()
-		}
+	var host, port = "127.0.0.1", "11434"
+
+	parts := strings.Split(os.Getenv("OLLAMA_HOST"), ":")
+	if ip := net.ParseIP(parts[0]); ip != nil {
+		host = ip.String()
 	}

-	if err := initializeKeypair(); err != nil {
+	if len(parts) > 1 {
+		port = parts[1]
+	}
+
+	// deprecated: include port in OLLAMA_HOST
+	if p := os.Getenv("OLLAMA_PORT"); p != "" {
+		port = p
+	}
+
+	err := initializeKeypair()
+	if err != nil {
 		return err
 	}

-	ln, err := net.Listen("tcp", net.JoinHostPort(host, port))
+	ln, err := net.Listen("tcp", fmt.Sprintf("%s:%s", host, port))
 	if err != nil {
 		return err
 	}
@@ -795,21 +600,6 @@ func RunServer(cmd *cobra.Command, _ []string) error {
 		origins = strings.Split(o, ",")
 	}

-	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
-		if err := server.PruneLayers(); err != nil {
-			return err
-		}
-
-		manifestsPath, err := server.GetManifestPath()
-		if err != nil {
-			return err
-		}
-
-		if err := server.PruneDirectory(manifestsPath); err != nil {
-			return err
-		}
-	}
-
 	return server.Serve(ln, origins)
 }

@@ -835,12 +625,12 @@ func initializeKeypair() error {
 			return err
 		}

-		err = os.MkdirAll(filepath.Dir(privKeyPath), 0o755)
+		err = os.MkdirAll(path.Dir(privKeyPath), 0o700)
 		if err != nil {
 			return fmt.Errorf("could not create directory %w", err)
 		}

-		err = os.WriteFile(privKeyPath, pem.EncodeToMemory(privKeyBytes), 0o600)
+		err = os.WriteFile(privKeyPath, pem.EncodeToMemory(privKeyBytes), 0600)
 		if err != nil {
 			return err
 		}
@@ -852,7 +642,7 @@ func initializeKeypair() error {

 		pubKeyData := ssh.MarshalAuthorizedKey(sshPrivateKey.PublicKey())

-		err = os.WriteFile(pubKeyPath, pubKeyData, 0o644)
+		err = os.WriteFile(pubKeyPath, pubKeyData, 0644)
 		if err != nil {
 			return err
 		}
@@ -894,7 +684,7 @@ func startMacApp(client *api.Client) error {
 }

 func checkServerHeartbeat(_ *cobra.Command, _ []string) error {
-	client, err := api.ClientFromEnvironment()
+	client, err := api.FromEnv()
 	if err != nil {
 		return err
 	}
@@ -924,7 +714,6 @@ func NewCLI() *cobra.Command {
 		CompletionOptions: cobra.CompletionOptions{
 			DisableDefaultCmd: true,
 		},
-		Version: version.Version,
 	}

 	cobra.EnableCommandSorting = false
@@ -932,27 +721,13 @@ func NewCLI() *cobra.Command {
 	createCmd := &cobra.Command{
 		Use:     "create MODEL",
 		Short:   "Create a model from a Modelfile",
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.MinimumNArgs(1),
 		PreRunE: checkServerHeartbeat,
 		RunE:    CreateHandler,
 	}

 	createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile (default \"Modelfile\")")

-	showCmd := &cobra.Command{
-		Use:     "show MODEL",
-		Short:   "Show information for a model",
-		Args:    cobra.ExactArgs(1),
-		PreRunE: checkServerHeartbeat,
-		RunE:    ShowHandler,
-	}
-
-	showCmd.Flags().Bool("license", false, "Show license of a model")
-	showCmd.Flags().Bool("modelfile", false, "Show Modelfile of a model")
-	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
-	showCmd.Flags().Bool("template", false, "Show template of a model")
-	showCmd.Flags().Bool("system", false, "Show system prompt of a model")
-
 	runCmd := &cobra.Command{
 		Use:     "run MODEL [PROMPT]",
 		Short:   "Run a model",
@@ -962,21 +737,18 @@ func NewCLI() *cobra.Command {
 	}

 	runCmd.Flags().Bool("verbose", false, "Show timings for response")
-	runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
-	runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")

 	serveCmd := &cobra.Command{
 		Use:     "serve",
 		Aliases: []string{"start"},
 		Short:   "Start ollama",
-		Args:    cobra.ExactArgs(0),
 		RunE:    RunServer,
 	}

 	pullCmd := &cobra.Command{
 		Use:     "pull MODEL",
 		Short:   "Pull a model from a registry",
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.MinimumNArgs(1),
 		PreRunE: checkServerHeartbeat,
 		RunE:    PullHandler,
 	}
@@ -986,7 +758,7 @@ func NewCLI() *cobra.Command {
 	pushCmd := &cobra.Command{
 		Use:     "push MODEL",
 		Short:   "Push a model to a registry",
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.MinimumNArgs(1),
 		PreRunE: checkServerHeartbeat,
 		RunE:    PushHandler,
 	}
@@ -1002,15 +774,15 @@ func NewCLI() *cobra.Command {
 	}

 	copyCmd := &cobra.Command{
-		Use:     "cp SOURCE TARGET",
+		Use:     "cp",
 		Short:   "Copy a model",
-		Args:    cobra.ExactArgs(2),
+		Args:    cobra.MinimumNArgs(2),
 		PreRunE: checkServerHeartbeat,
 		RunE:    CopyHandler,
 	}

 	deleteCmd := &cobra.Command{
-		Use:     "rm MODEL [MODEL...]",
+		Use:     "rm",
 		Short:   "Remove a model",
 		Args:    cobra.MinimumNArgs(1),
 		PreRunE: checkServerHeartbeat,
@@ -1020,7 +792,6 @@ func NewCLI() *cobra.Command {
 	rootCmd.AddCommand(
 		serveCmd,
 		createCmd,
-		showCmd,
 		runCmd,
 		pullCmd,
 		pushCmd,
--- a/deps.sh
+++ b/deps.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cmake -B build
+make -C build
--- a/docs/api.md
+++ b/docs/api.md
@@ -3,32 +3,26 @@
 ## Endpoints

 - [Generate a completion](#generate-a-completion)
- [Create a Model](#create-a-model)
- [List Local Models](#list-local-models)
- [Show Model Information](#show-model-information)
- [Copy a Model](#copy-a-model)
- [Delete a Model](#delete-a-model)
- [Pull a Model](#pull-a-model)
- [Push a Model](#push-a-model)
- [Generate Embeddings](#generate-embeddings)
+- [Create a model](#create-a-model)
+- [List local models](#list-local-models)
+- [Copy a model](#copy-a-model)
+- [Delete a model](#delete-a-model)
+- [Pull a model](#pull-a-model)
+- [Generate embeddings](#generate-embeddings)

 ## Conventions

 ### Model names

-Model names follow a `model:tag` format. Some examples are `orca-mini:3b-q4_1` and `llama2:70b`. The tag is optional and, if not provided, will default to `latest`. The tag is used to identify a specific version.
+Model names follow a `model:tag` format. Some examples are `orca:3b-q4_1` and `llama2:70b`. The tag is optional and if not provided will default to `latest`. The tag is used to identify a specific version.

 ### Durations

 All durations are returned in nanoseconds.

-### Streaming responses
-
-Certain endpoints stream responses as JSON objects delineated with the newline (`\n`) character.
-
 ## Generate a completion

-```shell
+```
 POST /api/generate
 ```

@@ -39,17 +33,16 @@ Generate a response for a given prompt with a provided model. This is a streamin
 - `model`: (required) the [model name](#model-names)
 - `prompt`: the prompt to generate a response for

-Advanced parameters (optional):
+Advanced parameters:

 - `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
 - `system`: system prompt to (overrides what is defined in the `Modelfile`)
 - `template`: the full prompt or prompt template (overrides what is defined in the `Modelfile`)
 - `context`: the context parameter returned from a previous request to `/generate`, this can be used to keep a short conversational memory
- `stream`: if `false` the response will be be returned as a single response object, rather than a stream of objects

 ### Request

-```shell
+```
 curl -X POST http://localhost:11434/api/generate -d '{
  "model": "llama2:7b",
  "prompt": "Why is the sky blue?"
@@ -80,7 +73,6 @@ The final response in the stream also includes additional data about the generat
 - `eval_count`: number of tokens the response
 - `eval_duration`: time in nanoseconds spent generating the response
 - `context`: an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memory
- `response`: empty if the response was streamed, if not streamed, this will contain the full response

 To calculate how fast the response is generated in tokens per second (token/s), divide `eval_count` / `eval_duration`.

@@ -88,7 +80,6 @@ To calculate how fast the response is generated in tokens per second (token/s),
 {
  "model": "llama2:7b",
  "created_at": "2023-08-04T19:22:45.499127Z",
-  "response": "",
  "context": [1, 2, 3],
  "done": true,
  "total_duration": 5589157167,
@@ -104,7 +95,7 @@ To calculate how fast the response is generated in tokens per second (token/s),

 ## Create a Model

-```shell
+```
 POST /api/create
 ```

@@ -114,11 +105,10 @@ Create a model from a [`Modelfile`](./modelfile.md)

 - `name`: name of the model to create
 - `path`: path to the Modelfile
- `stream`: (optional) if `false` the response will be be returned as a single response object, rather than a stream of objects

 ### Request

-```shell
+```
 curl -X POST http://localhost:11434/api/create -d '{
  "name": "mario",
  "path": "~/Modelfile"
@@ -127,7 +117,7 @@ curl -X POST http://localhost:11434/api/create -d '{

 ### Response

-A stream of JSON objects. When finished, `status` is `success`.
+A stream of JSON objects. When finished, `status` is `success`

 ```json
 {
@@ -137,7 +127,7 @@ A stream of JSON objects. When finished, `status` is `success`.

 ## List Local Models

-```shell
+```
 GET /api/tags
 ```

@@ -145,7 +135,7 @@ List models that are available locally.

 ### Request

-```shell
+```
 curl http://localhost:11434/api/tags
 ```

@@ -168,40 +158,9 @@ curl http://localhost:11434/api/tags
 }
 ```

-## Show Model Information
-
-```shell
-POST /api/show
-```
-
-Show details about a model including modelfile, template, parameters, license, and system prompt.
-
-### Parameters
-
- `name`: name of the model to show
-
-### Request
-
-```shell
-curl http://localhost:11434/api/show -d '{
-  "name": "llama2:7b"
-}'
-```
-
-### Response
-
-```json
-{
-  "license": "<contents of license block>",
-  "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llama2:latest\n\nFROM /Users/username/.ollama/models/blobs/sha256:8daa9615cce30c259a9555b1cc250d461d1bc69980a274b44d7eda0be78076d8\nTEMPLATE \"\"\"[INST] {{ if and .First .System }}<<SYS>>{{ .System }}<</SYS>>\n\n{{ end }}{{ .Prompt }} [/INST] \"\"\"\nSYSTEM \"\"\"\"\"\"\nPARAMETER stop [INST]\nPARAMETER stop [/INST]\nPARAMETER stop <<SYS>>\nPARAMETER stop <</SYS>>\n",
-  "parameters": "stop                           [INST]\nstop                           [/INST]\nstop                           <<SYS>>\nstop                           <</SYS>>",
-  "template": "[INST] {{ if and .First .System }}<<SYS>>{{ .System }}<</SYS>>\n\n{{ end }}{{ .Prompt }} [/INST] "
-}
-```
-
 ## Copy a Model

-```shell
+```
 POST /api/copy
 ```

@@ -209,7 +168,7 @@ Copy a model. Creates a model with another name from an existing model.

 ### Request

-```shell
+```
 curl http://localhost:11434/api/copy -d '{
  "source": "llama2:7b",
  "destination": "llama2-backup"
@@ -218,7 +177,7 @@ curl http://localhost:11434/api/copy -d '{

 ## Delete a Model

-```shell
+```
 DELETE /api/delete
 ```

@@ -230,7 +189,7 @@ Delete a model and its data.

 ### Request

-```shell
+```
 curl -X DELETE http://localhost:11434/api/delete -d '{
  "name": "llama2:13b"
 }'
@@ -238,21 +197,19 @@ curl -X DELETE http://localhost:11434/api/delete -d '{

 ## Pull a Model

-```shell
+```
 POST /api/pull
 ```

-Download a model from the ollama library. Cancelled pulls are resumed from where they left off, and multiple calls will share the same download progress.
+Download a model from a the model registry. Cancelled pulls are resumed from where they left off, and multiple calls to will share the same download progress.

 ### Parameters

 - `name`: name of the model to pull
- `insecure`: (optional) allow insecure connections to the library. Only use this if you are pulling from your own library during development.
- `stream`: (optional) if `false` the response will be be returned as a single response object, rather than a stream of objects

 ### Request

-```shell
+```
 curl -X POST http://localhost:11434/api/pull -d '{
  "name": "llama2:7b"
 }'
@@ -268,66 +225,9 @@ curl -X POST http://localhost:11434/api/pull -d '{
 }
 ```

-## Push a Model
-
-```shell
-POST /api/push
-```
-
-Upload a model to a model library. Requires registering for ollama.ai and adding a public key first.
-
-### Parameters
-
- `name`: name of the model to push in the form of `<namespace>/<model>:<tag>`
- `insecure`: (optional) allow insecure connections to the library. Only use this if you are pushing to your library during development.
- `stream`: (optional) if `false` the response will be be returned as a single response object, rather than a stream of objects
-
-### Request
-
-```shell
-curl -X POST http://localhost:11434/api/push -d '{
-  "name": "mattw/pygmalion:latest"
-}'
-```
-
-### Response
-
-Streaming response that starts with:
-
-```json
-{ "status": "retrieving manifest" }
-```
-
-and then:
-
-```json
-{
-  "status": "starting upload",
-  "digest": "sha256:bc07c81de745696fdf5afca05e065818a8149fb0c77266fb584d9b2cba3711ab",
-  "total": 1928429856
-}
-```
-
-Then there is a series of uploading responses:
-
-```json
-{
-  "status": "starting upload",
-  "digest": "sha256:bc07c81de745696fdf5afca05e065818a8149fb0c77266fb584d9b2cba3711ab",
-  "total": 1928429856
-}
-```
-
-Finally, when the upload is complete:
-
-```json
-{"status":"pushing manifest"}
-{"status":"success"}
-```
-
 ## Generate Embeddings

-```shell
+```
 POST /api/embeddings
 ```

@@ -338,13 +238,9 @@ Generate embeddings from a model
 - `model`: name of model to generate embeddings from
 - `prompt`: text to generate embeddings for

-Advanced parameters:
-
- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
-
 ### Request

-```shell
+```
 curl -X POST http://localhost:11434/api/embeddings -d '{
  "model": "llama2:7b",
  "prompt": "Here is an article about llamas..."
@@ -355,7 +251,7 @@ curl -X POST http://localhost:11434/api/embeddings -d '{

 ```json
 {
-  "embedding": [
+  "embeddings": [
    0.5670403838157654, 0.009260174818336964, 0.23178744316101074, -0.2916173040866852, -0.8924556970596313,
    0.8785552978515625, -0.34576427936553955, 0.5742510557174683, -0.04222835972905159, -0.137906014919281
  ]
--- a/docs/development.md
+++ b/docs/development.md
@@ -1,39 +1,44 @@
 # Development

- Install cmake or (optionally, required tools for GPUs)
- run `go generate ./...`
- run `go build .`
-
 Install required tools:

- cmake version 3.24 or higher
- go version 1.20 or higher
- gcc version 11.4.0 or higher
-
-```bash
-brew install go cmake gcc
+```
+brew install go
 ```

-Get the required libraries:
+Enable CGO:

-```bash
-go generate ./...
 ```
+export CGO_ENABLED=1
+```
+
+You will also need a C/C++ compiler such as GCC for MacOS and Linux or Mingw-w64 GCC for Windows.

 Then build ollama:

-```bash
+```
 go build .
 ```

 Now you can run `ollama`:

-```bash
+```
 ./ollama
 ```

-## Building on Linux with GPU support
+## Releasing

- Install cmake and nvidia-cuda-toolkit
- run `go generate ./...`
- run `go build .`
+To release a new version of Ollama you'll need to set some environment variables:
+
+- `GITHUB_TOKEN`: your GitHub token
+- `APPLE_IDENTITY`: the Apple signing identity (macOS only)
+- `APPLE_ID`: your Apple ID
+- `APPLE_PASSWORD`: your Apple ID app-specific password
+- `APPLE_TEAM_ID`: the Apple team ID for the signing identity
+- `TELEMETRY_WRITE_KEY`: segment write key for telemetry
+
+Then run the publish script with the target version:
+
+```
+VERSION=0.0.2 ./scripts/publish.sh
+```
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -1,34 +1,17 @@
 # FAQ

-## How can I view the logs?
-
-On macOS:
-
-```
-cat ~/.ollama/logs/server.log
-```
-
-On Linux:
-
-```
-journalctl -u ollama
-```
-
-If you're running `ollama serve` directly, the logs will be printed to the console.
-
 ## How can I expose the Ollama server?

-```bash
+```
 OLLAMA_HOST=0.0.0.0:11435 ollama serve
 ```

 By default, Ollama allows cross origin requests from `127.0.0.1` and `0.0.0.0`. To support more origins, you can use the `OLLAMA_ORIGINS` environment variable:

-```bash
+```
 OLLAMA_ORIGINS=http://192.168.1.1:*,https://example.com ollama serve
 ```

 ## Where are models stored?

- macOS: Raw model data is stored under `~/.ollama/models`.
- Linux: Raw model data is stored under `/usr/share/ollama/.ollama/models`
+Raw model data is stored under `~/.ollama/models`.
--- a/docs/import.md
+++ b/docs/import.md
@@ -1,163 +0,0 @@
-# Import a model
-
-This guide walks through importing a PyTorch, Safetensors or GGUF model.
-
-## Supported models
-
-Ollama supports a set of model architectures, with support for more coming soon:
-
- Llama & Mistral
- Falcon & RW
- GPT-NeoX
- BigCode
-
-To view a model's architecture, check the `config.json` file in its HuggingFace repo. You should see an entry under `architectures` (e.g. `LlamaForCausalLM`).
-
-## Importing
-
-### Step 1: Clone the HuggingFace repository (optional)
-
-If the model is currently hosted in a HuggingFace repository, first clone that repository to download the raw model.
-
-```
-git lfs install
-git clone https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1
-cd Mistral-7B-Instruct-v0.1
-```
-
-### Step 2: Convert and quantize to a `.bin` file (optional, for PyTorch and Safetensors)
-
-If the model is in PyTorch or Safetensors format, a [Docker image](https://hub.docker.com/r/ollama/quantize) with the tooling required to convert and quantize models is available.
-
-First, Install [Docker](https://www.docker.com/get-started/).
-
-Next, to convert and quantize your model, run:
-
-```
-docker run --rm -v .:/model ollama/quantize -q q4_0 /model
-```
-
-This will output two files into the directory:
-
- `f16.bin`: the model converted to GGUF
- `q4_0.bin` the model quantized to a 4-bit quantization (we will use this file to create the Ollama model)
-
-### Step 3: Write a `Modelfile`
-
-Next, create a `Modelfile` for your model. This file is the blueprint for your model, specifying weights, parameters, prompt templates and more.
-
-```
-FROM ./q4_0.bin
-```
-
-(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
-
-```
-FROM ./q4_0.bin
-TEMPLATE "[INST] {{ .Prompt }} [/INST]"
-```
-
-### Step 4: Create the Ollama model
-
-Finally, create a model from your `Modelfile`:
-
-```
-ollama create example -f Modelfile
-```
-
-Next, test the model with `ollama run`:
-
-```
-ollama run example "What is your favourite condiment?"
-```
-
-### Step 5: Publish your model (optional – early alpha)
-
-Publishing models is in early alpha. If you'd like to publish your model to share with others, follow these steps:
-
-1. Create [an account](https://ollama.ai/signup)
-2. Run `cat ~/.ollama/id_ed25519.pub` to view your Ollama public key. Copy this to the clipboard.
-3. Add your public key to your [Ollama account](https://ollama.ai/settings/keys)
-
-Next, copy your model to your username's namespace:
-
-```
-ollama cp example <your username>/example
-```
-
-Then push the model:
-
-```
-ollama push <your username>/example
-```
-
-After publishing, your model will be available at `https://ollama.ai/<your username>/example`.
-
-## Quantization reference
-
-The quantization options are as follow (from highest highest to lowest levels of quantization). Note: some architectures such as Falcon do not support K quants.
-
- `q2_K`
- `q3_K`
- `q3_K_S`
- `q3_K_M`
- `q3_K_L`
- `q4_0` (recommended)
- `q4_1`
- `q4_K`
- `q4_K_S`
- `q4_K_M`
- `q5_0`
- `q5_1`
- `q5_K`
- `q5_K_S`
- `q5_K_M`
- `q6_K`
- `q8_0`
-
-## Manually converting & quantizing models
-
-### Prerequisites
-
-Start by cloning the `llama.cpp` repo to your machine in another directory:
-
-```
-git clone https://github.com/ggerganov/llama.cpp.git
-cd llama.cpp
-```
-
-Next, install the Python dependencies:
-
-```
-pip install -r requirements.txt
-```
-
-Finally, build the `quantize` tool:
-
-```
-make quantize
-```
-
-### Convert the model
-
-Run the correct conversion script for your model architecture:
-
-```shell
-# LlamaForCausalLM or MistralForCausalLM
-python convert.py <path to model directory>
-
-# FalconForCausalLM
-python convert-falcon-hf-to-gguf.py <path to model directory>
-
-# GPTNeoXForCausalLM
-python convert-falcon-hf-to-gguf.py <path to model directory>
-
-# GPTBigCodeForCausalLM
-python convert-starcoder-hf-to-gguf.py <path to model directory>
-```
-
-### Quantize the model
-
-```
-quantize <path to model dir>/ggml-model-f32.bin <path to model dir>/q4_0.bin q4_0
-```
--- a/docs/linux.md
+++ b/docs/linux.md
@@ -1,82 +0,0 @@
-# Installing Ollama on Linux
-
-> Note: A one line installer for Ollama is available by running:
->
-> ```bash
-> curl https://ollama.ai/install.sh | sh
-> ```
-
-## Download the `ollama` binary
-
-Ollama is distributed as a self-contained binary. Download it to a directory in your PATH:
-
-```bash
-sudo curl -L https://ollama.ai/download/ollama-linux-amd64 -o /usr/bin/ollama
-sudo chmod +x /usr/bin/ollama
-```
-
-## Start Ollama
-
-Start Ollama by running `ollama serve`:
-
-```bash
-ollama serve
-```
-
-Once Ollama is running, run a model in another terminal session:
-
-```bash
-ollama run llama2
-```
-
-## Install CUDA drivers (optional – for Nvidia GPUs)
-
-[Download and install](https://developer.nvidia.com/cuda-downloads) CUDA.
-
-Verify that the drivers are installed by running the following command, which should print details about your GPU:
-
-```bash
-nvidia-smi
-```
-
-## Adding Ollama as a startup service (optional)
-
-Create a user for Ollama:
-
-```bash
-sudo useradd -r -s /bin/false -m -d /usr/share/ollama ollama
-```
-
-Create a service file in `/etc/systemd/system/ollama.service`:
-
-```ini
-[Unit]
-Description=Ollama Service
-After=network-online.target
-
-[Service]
-ExecStart=/usr/bin/ollama serve
-User=ollama
-Group=ollama
-Restart=always
-RestartSec=3
-Environment="HOME=/usr/share/ollama"
-
-[Install]
-WantedBy=default.target
-```
-
-Then start the service:
-
-```bash
-sudo systemctl daemon-reload
-sudo systemctl enable ollama
-```
-
-### Viewing logs
-
-To view logs of Ollama running as a startup service, run:
-
-```bash
-journalctl -u ollama
-```
--- a/docs/modelfile.md
+++ b/docs/modelfile.md
@@ -1,6 +1,6 @@
 # Ollama Model File

-> Note: this `Modelfile` syntax is in development
+> Note: this model file syntax is in development

 A model file is the blueprint to create and share models with Ollama.

@@ -12,6 +12,7 @@ A model file is the blueprint to create and share models with Ollama.
  - [FROM (Required)](#from-required)
    - [Build from llama2](#build-from-llama2)
    - [Build from a bin file](#build-from-a-bin-file)
+  - [EMBED](#embed)
  - [PARAMETER](#parameter)
    - [Valid Parameters and Values](#valid-parameters-and-values)
  - [TEMPLATE](#template)
@@ -23,7 +24,7 @@ A model file is the blueprint to create and share models with Ollama.

 ## Format

-The format of the `Modelfile`:
+The format of the Modelfile:

 ```modelfile
 # comment
@@ -41,9 +42,9 @@ INSTRUCTION arguments

 ## Examples

-An example of a `Modelfile` creating a mario blueprint:
+An example of a model file creating a mario blueprint:

-```modelfile
+```
 FROM llama2
 # sets the temperature to 1 [higher is more creative, lower is more coherent]
 PARAMETER temperature 1
@@ -56,9 +57,9 @@ SYSTEM You are Mario from super mario bros, acting as an assistant.

 To use this:

-1. Save it as a file (e.g. `Modelfile`)
-2. `ollama create choose-a-model-name -f <location of the file e.g. ./Modelfile>'`
-3. `ollama run choose-a-model-name`
+1. Save it as a file (eg. `Modelfile`)
+2. `ollama create NAME -f <location of the file eg. ./Modelfile>'`
+3. `ollama run NAME`
 4. Start using the model!

 More examples are available in the [examples directory](../examples).
@@ -67,34 +68,44 @@ More examples are available in the [examples directory](../examples).

 ### FROM (Required)

-The `FROM` instruction defines the base model to use when creating a model.
+The FROM instruction defines the base model to use when creating a model.

-```modelfile
+```
 FROM <model name>:<tag>
 ```

 #### Build from llama2

-```modelfile
+```
 FROM llama2
 ```

 A list of available base models:
 <https://github.com/jmorganca/ollama#model-library>

-#### Build from a `bin` file
+#### Build from a bin file

-```modelfile
+```
 FROM ./ollama-model.bin
 ```

-This bin file location should be specified as an absolute path or relative to the `Modelfile` location.
+This bin file location should be specified as an absolute path or relative to the Modelfile location.
+
+### EMBED
+
+The EMBED instruction is used to add embeddings of files to a model. This is useful for adding custom data that the model can reference when generating an answer. Note that currently only text files are supported, formatted with each line as one embedding.
+```
+FROM <model name>:<tag>
+EMBED <file path>.txt
+EMBED <different file path>.txt
+EMBED <path to directory>/*.txt
+```

 ### PARAMETER

 The `PARAMETER` instruction defines a parameter that can be set when the model is run.

-```modelfile
+```
 PARAMETER <parameter> <parametervalue>
 ```

@@ -107,21 +118,19 @@ PARAMETER <parameter> <parametervalue>
 | mirostat_tau   | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0)                                                                                                         | float      | mirostat_tau 5.0     |
 | num_ctx        | Sets the size of the context window used to generate the next token. (Default: 2048)                                                                                                                                                                    | int        | num_ctx 4096         |
 | num_gqa        | The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b                                                                                                                                         | int        | num_gqa 1            |
-| num_gpu        | The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable.                                                                                                                                            | int        | num_gpu 50           |
+| num_gpu        | The number of GPUs to use. On macOS it defaults to 1 to enable metal support, 0 to disable.                                                                                                                                                             | int        | num_gpu 1            |
 | num_thread     | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | int        | num_thread 8         |
 | repeat_last_n  | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx)                                                                                                                                           | int        | repeat_last_n 64     |
 | repeat_penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)                                                                     | float      | repeat_penalty 1.1   |
 | temperature    | The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8)                                                                                                                                     | float      | temperature 0.7      |
-| seed | Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. (Default: 0) | int | seed 42 |
-| stop           | Sets the stop sequences to use.                                                                                                                                                                                                                         | string     | stop "AI assistant:" |
+| stop           | Sets the stop tokens to use.                                                                                                                                                                                                                            | string     | stop "AI assistant:" |
 | tfs_z          | Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. (default: 1)                                               | float      | tfs_z 1              |
-| num_predict    | Maximum number of tokens to predict when generating text. (Default: 128, -1 = infinite generation, -2 = fill context)                                                                                                                                   | int        | num_predict 42       |
 | top_k          | Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)                                                                        | int        | top_k 40             |
 | top_p          | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)                                                                 | float      | top_p 0.9            |

 ### TEMPLATE

-`TEMPLATE` of the full prompt template to be passed into the model. It may include (optionally) a system prompt and a user's prompt. This is used to create a full custom prompt, and syntax may be model specific. You can usually find the template for a given model in the readme for that model.
+`TEMPLATE` of the full prompt template to be passed into the model. It may include (optionally) a system prompt and a user's prompt. This is used to create a full custom prompt, and syntax may be model specific.

 #### Template Variables

@@ -131,7 +140,7 @@ PARAMETER <parameter> <parametervalue>
 | `{{ .Prompt }}` | The incoming prompt, this is not specified in the model file and will be set based on input.                 |
 | `{{ .First }}`  | A boolean value used to render specific template information for the first generation of a session.          |

-```modelfile
+```
 TEMPLATE """
 {{- if .First }}
 ### System:
@@ -151,7 +160,7 @@ SYSTEM """<system message>"""

 The `SYSTEM` instruction specifies the system prompt to be used in the template, if applicable.

-```modelfile
+```
 SYSTEM """<system message>"""
 ```

@@ -159,7 +168,7 @@ SYSTEM """<system message>"""

 The `ADAPTER` instruction specifies the LoRA adapter to apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGML file format. The adapter should be tuned from the base model otherwise the behaviour is undefined.

-```modelfile
+```
 ADAPTER ./ollama-lora.bin
 ```

@@ -167,7 +176,7 @@ ADAPTER ./ollama-lora.bin

 The `LICENSE` instruction allows you to specify the legal license under which the model used with this Modelfile is shared or distributed.

-```modelfile
+```
 LICENSE """
 <license text>
 """
@@ -175,5 +184,5 @@ LICENSE """

 ## Notes

- the **`Modelfile` is not case sensitive**. In the examples, we use uppercase for instructions to make it easier to distinguish it from arguments.
+- the **modelfile is not case sensitive**. In the examples, we use uppercase for instructions to make it easier to distinguish it from arguments.
 - Instructions can be in any order. In the examples, we start with FROM instruction to keep it easily readable.
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -1,171 +0,0 @@
-node_modules
-# OSX
-.DS_STORE
-
-# Models
-models/
-
-# Local Chroma db
-.chroma/
-db/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,3 +1,15 @@
 # Examples

-This directory contains different examples of using Ollama.
+This directory contains different examples of using Ollama
+
+To create a model:
+
+```
+ollama create example -f <example file>
+```
+
+To run a model:
+
+```
+ollama run example
+```
--- a/examples/modelfile-devopsengineer/Modelfile
+++ b/examples/modelfile-devopsengineer/Modelfile
@@ -1,7 +1,7 @@
 # Modelfile for creating a devops engineer assistant
 # Run `ollama create devops-engineer -f ./Modelfile` and then `ollama run devops-engineer` and enter a topic

-FROM mistral
+FROM llama2:13b
 PARAMETER temperature 1
 SYSTEM """
 You are a senior devops engineer, acting as an assistant. You offer help with cloud technologies like: Terraform, AWS, kubernetes, python. You answer with code examples when possible
--- a/examples/python-dockerit/Modelfile
+++ b/examples/python-dockerit/Modelfile
@@ -1,6 +1,6 @@
-FROM mistral
+FROM llama2
 SYSTEM """
-You are an experienced Devops engineer focused on docker. When given specifications for a particular need or application you know the best way to host that within a docker container. For instance if someone tells you they want an nginx server to host files located at /web you will answer as follows
+You are an experience Devops engineer focused on docker. When given specifications for a particular need or application you know the best way to host that within a docker container. For instance if someone tells you they want an nginx server to host files located at /web you will answer as follows

 ---start
 FROM nginx:alpine
@@ -16,5 +16,5 @@ ENV POSTGRES_PASSWORD=abc123
 EXPOSE 5432
 ---end

-Again it's just the contents of the dockerfile and nothing else.
-"""
+Again it's just the contents of the dockerfile an nothing else.
+"""
--- a/examples/python-dockerit/README.md
+++ b/examples/python-dockerit/README.md
--- a/examples/python-dockerit/dockerit.py
+++ b/examples/python-dockerit/dockerit.py
--- a/examples/python-dockerit/requirements.txt
+++ b/examples/python-dockerit/requirements.txt
--- a/examples/golang-simplegenerate/README.md
+++ b/examples/golang-simplegenerate/README.md
--- a/examples/golang-simplegenerate/main.go
+++ b/examples/golang-simplegenerate/main.go
@@ -1,27 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-)
-
-func main() {
-	body := []byte(`{"model":"mistral"}`)
-	resp, err := http.Post("http://localhost:11434/api/generate", "application/json", bytes.NewBuffer(body))
-
-	if err != nil {
-		fmt.Print(err.Error())
-		os.Exit(1)
-	}
-
-	responseData, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(string(responseData))
-
-}
--- a/examples/langchain-python-rag-document/README.md
+++ b/examples/langchain-python-rag-document/README.md
--- a/examples/langchain-python-rag-document/main.py
+++ b/examples/langchain-python-rag-document/main.py
--- a/examples/langchain-python-rag-document/requirements.txt
+++ b/examples/langchain-python-rag-document/requirements.txt
--- a/examples/langchain-typescript-simple/README.md
+++ b/examples/langchain-typescript-simple/README.md
@@ -1,21 +0,0 @@
-# LangChain
-
-This example is a basic "hello world" of using LangChain with Ollama using Node.js and Typescript.
-
-## Setup
-
-```shell
-npm install
-```
-
-## Run
-
-```shell
-ts-node main.ts
-```
-
-Running this example will print the response for "hello":
-
-```plaintext
-Hello! It's nice to meet you. hopefully you are having a great day! Is there something I can help you with or would you like to chat?
-```
--- a/examples/langchain-typescript-simple/main.ts
+++ b/examples/langchain-typescript-simple/main.ts
@@ -1,15 +0,0 @@
-import { Ollama} from 'langchain/llms/ollama';
-
-async function main() {
-  const ollama = new Ollama({
-    model: 'mistral'    
-    // other parameters can be found at https://js.langchain.com/docs/api/llms_ollama/classes/Ollama
-  })
-  const stream = await ollama.stream("Hello");
-
-  for await (const chunk of stream) {
-    process.stdout.write(chunk);
-  }
-}
-
-main();
--- a/examples/langchain-typescript-simple/package-lock.json
+++ b/examples/langchain-typescript-simple/package-lock.json
@@ -1,997 +0,0 @@
-{
-  "name": "with-langchain-typescript-simplegenerate",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "dependencies": {
-        "langchain": "^0.0.165"
-      },
-      "devDependencies": {
-        "typescript": "^5.2.2"
-      }
-    },
-    "node_modules/@anthropic-ai/sdk": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.6.2.tgz",
-      "integrity": "sha512-fB9PUj9RFT+XjkL+E9Ol864ZIJi+1P8WnbHspN3N3/GK2uSzjd0cbVIKTGgf4v3N8MwaQu+UWnU7C4BG/fap/g==",
-      "dependencies": {
-        "@types/node": "^18.11.18",
-        "@types/node-fetch": "^2.6.4",
-        "abort-controller": "^3.0.0",
-        "agentkeepalive": "^4.2.1",
-        "digest-fetch": "^1.3.0",
-        "form-data-encoder": "1.7.2",
-        "formdata-node": "^4.3.2",
-        "node-fetch": "^2.6.7"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "18.18.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.4.tgz",
-      "integrity": "sha512-t3rNFBgJRugIhackit2mVcLfF6IRc0JE4oeizPQL8Zrm8n2WY/0wOdpOPhdtG0V9Q2TlW/axbF1MJ6z+Yj/kKQ=="
-    },
-    "node_modules/@types/node-fetch": {
-      "version": "2.6.6",
-      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz",
-      "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==",
-      "dependencies": {
-        "@types/node": "*",
-        "form-data": "^4.0.0"
-      }
-    },
-    "node_modules/@types/retry": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="
-    },
-    "node_modules/@types/uuid": {
-      "version": "9.0.5",
-      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz",
-      "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ=="
-    },
-    "node_modules/abort-controller": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
-      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
-      "dependencies": {
-        "event-target-shim": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=6.5"
-      }
-    },
-    "node_modules/agentkeepalive": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
-      "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
-      "dependencies": {
-        "humanize-ms": "^1.2.1"
-      },
-      "engines": {
-        "node": ">= 8.0.0"
-      }
-    },
-    "node_modules/ansi-styles": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
-      "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-      }
-    },
-    "node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
-    },
-    "node_modules/asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-    },
-    "node_modules/base-64": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
-      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
-    },
-    "node_modules/base64-js": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
-    "node_modules/binary-extensions": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
-      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/binary-search": {
-      "version": "1.3.6",
-      "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz",
-      "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA=="
-    },
-    "node_modules/camelcase": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
-      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/charenc": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
-      "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dependencies": {
-        "delayed-stream": "~1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/commander": {
-      "version": "10.0.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz",
-      "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
-    "node_modules/crypt": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
-      "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/decamelize": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
-      "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
-    "node_modules/digest-fetch": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
-      "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
-      "dependencies": {
-        "base-64": "^0.1.0",
-        "md5": "^2.3.0"
-      }
-    },
-    "node_modules/event-target-shim": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
-      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-      "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/eventemitter3": {
-      "version": "4.0.7",
-      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
-      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
-    },
-    "node_modules/expr-eval": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz",
-      "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg=="
-    },
-    "node_modules/flat": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
-      "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
-      "bin": {
-        "flat": "cli.js"
-      }
-    },
-    "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/form-data-encoder": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
-      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
-    },
-    "node_modules/formdata-node": {
-      "version": "4.4.1",
-      "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
-      "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
-      "dependencies": {
-        "node-domexception": "1.0.0",
-        "web-streams-polyfill": "4.0.0-beta.3"
-      },
-      "engines": {
-        "node": ">= 12.20"
-      }
-    },
-    "node_modules/humanize-ms": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
-      "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
-      "dependencies": {
-        "ms": "^2.0.0"
-      }
-    },
-    "node_modules/is-any-array": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz",
-      "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ=="
-    },
-    "node_modules/is-buffer": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
-    },
-    "node_modules/js-tiktoken": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz",
-      "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==",
-      "dependencies": {
-        "base64-js": "^1.5.1"
-      }
-    },
-    "node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
-      "dependencies": {
-        "argparse": "^2.0.1"
-      },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
-      }
-    },
-    "node_modules/jsonpointer": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz",
-      "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/langchain": {
-      "version": "0.0.165",
-      "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.165.tgz",
-      "integrity": "sha512-CpbNpjwaE+9lzjdw+pZz0VgnRrFivEgr7CVp9dDaAb5JpaJAA4V2v6uQ9ZPN+TSqupTQ79HFn2sfyZVEl2EG7Q==",
-      "dependencies": {
-        "@anthropic-ai/sdk": "^0.6.2",
-        "ansi-styles": "^5.0.0",
-        "binary-extensions": "^2.2.0",
-        "camelcase": "6",
-        "decamelize": "^1.2.0",
-        "expr-eval": "^2.0.2",
-        "flat": "^5.0.2",
-        "js-tiktoken": "^1.0.7",
-        "js-yaml": "^4.1.0",
-        "jsonpointer": "^5.0.1",
-        "langchainhub": "~0.0.6",
-        "langsmith": "~0.0.31",
-        "ml-distance": "^4.0.0",
-        "object-hash": "^3.0.0",
-        "openai": "~4.4.0",
-        "openapi-types": "^12.1.3",
-        "p-queue": "^6.6.2",
-        "p-retry": "4",
-        "uuid": "^9.0.0",
-        "yaml": "^2.2.1",
-        "zod": "^3.22.3",
-        "zod-to-json-schema": "^3.20.4"
-      },
-      "engines": {
-        "node": ">=18"
-      },
-      "peerDependencies": {
-        "@aws-crypto/sha256-js": "^5.0.0",
-        "@aws-sdk/client-bedrock-runtime": "^3.422.0",
-        "@aws-sdk/client-dynamodb": "^3.310.0",
-        "@aws-sdk/client-kendra": "^3.352.0",
-        "@aws-sdk/client-lambda": "^3.310.0",
-        "@aws-sdk/client-s3": "^3.310.0",
-        "@aws-sdk/client-sagemaker-runtime": "^3.310.0",
-        "@aws-sdk/client-sfn": "^3.310.0",
-        "@aws-sdk/credential-provider-node": "^3.388.0",
-        "@azure/storage-blob": "^12.15.0",
-        "@clickhouse/client": "^0.0.14",
-        "@cloudflare/ai": "^1.0.12",
-        "@elastic/elasticsearch": "^8.4.0",
-        "@getmetal/metal-sdk": "*",
-        "@getzep/zep-js": "^0.7.0",
-        "@gomomento/sdk": "^1.23.0",
-        "@google-ai/generativelanguage": "^0.2.1",
-        "@google-cloud/storage": "^6.10.1",
-        "@huggingface/inference": "^1.5.1",
-        "@mozilla/readability": "*",
-        "@notionhq/client": "^2.2.10",
-        "@opensearch-project/opensearch": "*",
-        "@pinecone-database/pinecone": "^1.1.0",
-        "@planetscale/database": "^1.8.0",
-        "@qdrant/js-client-rest": "^1.2.0",
-        "@raycast/api": "^1.55.2",
-        "@smithy/eventstream-codec": "^2.0.5",
-        "@smithy/protocol-http": "^3.0.6",
-        "@smithy/signature-v4": "^2.0.10",
-        "@smithy/util-utf8": "^2.0.0",
-        "@supabase/postgrest-js": "^1.1.1",
-        "@supabase/supabase-js": "^2.10.0",
-        "@tensorflow-models/universal-sentence-encoder": "*",
-        "@tensorflow/tfjs-converter": "*",
-        "@tensorflow/tfjs-core": "*",
-        "@upstash/redis": "^1.20.6",
-        "@vercel/postgres": "^0.5.0",
-        "@writerai/writer-sdk": "^0.40.2",
-        "@xata.io/client": "^0.25.1",
-        "@xenova/transformers": "^2.5.4",
-        "@zilliz/milvus2-sdk-node": ">=2.2.7",
-        "apify-client": "^2.7.1",
-        "axios": "*",
-        "cassandra-driver": "^4.6.4",
-        "cheerio": "^1.0.0-rc.12",
-        "chromadb": "*",
-        "cohere-ai": ">=6.0.0",
-        "d3-dsv": "^2.0.0",
-        "epub2": "^3.0.1",
-        "faiss-node": "^0.3.0",
-        "fast-xml-parser": "^4.2.7",
-        "firebase-admin": "^11.9.0",
-        "google-auth-library": "^8.9.0",
-        "googleapis": "^126.0.1",
-        "hnswlib-node": "^1.4.2",
-        "html-to-text": "^9.0.5",
-        "ignore": "^5.2.0",
-        "ioredis": "^5.3.2",
-        "jsdom": "*",
-        "llmonitor": "*",
-        "lodash": "^4.17.21",
-        "mammoth": "*",
-        "mongodb": "^5.2.0",
-        "mysql2": "^3.3.3",
-        "neo4j-driver": "*",
-        "node-llama-cpp": "*",
-        "notion-to-md": "^3.1.0",
-        "pdf-parse": "1.1.1",
-        "peggy": "^3.0.2",
-        "pg": "^8.11.0",
-        "pg-copy-streams": "^6.0.5",
-        "pickleparser": "^0.1.0",
-        "playwright": "^1.32.1",
-        "portkey-ai": "^0.1.11",
-        "puppeteer": "^19.7.2",
-        "redis": "^4.6.4",
-        "replicate": "^0.18.0",
-        "sonix-speech-recognition": "^2.1.1",
-        "srt-parser-2": "^1.2.2",
-        "typeorm": "^0.3.12",
-        "typesense": "^1.5.3",
-        "usearch": "^1.1.1",
-        "vectordb": "^0.1.4",
-        "voy-search": "0.6.2",
-        "weaviate-ts-client": "^1.4.0",
-        "web-auth-library": "^1.0.3",
-        "youtube-transcript": "^1.0.6",
-        "youtubei.js": "^5.8.0"
-      },
-      "peerDependenciesMeta": {
-        "@aws-crypto/sha256-js": {
-          "optional": true
-        },
-        "@aws-sdk/client-bedrock-runtime": {
-          "optional": true
-        },
-        "@aws-sdk/client-dynamodb": {
-          "optional": true
-        },
-        "@aws-sdk/client-kendra": {
-          "optional": true
-        },
-        "@aws-sdk/client-lambda": {
-          "optional": true
-        },
-        "@aws-sdk/client-s3": {
-          "optional": true
-        },
-        "@aws-sdk/client-sagemaker-runtime": {
-          "optional": true
-        },
-        "@aws-sdk/client-sfn": {
-          "optional": true
-        },
-        "@aws-sdk/credential-provider-node": {
-          "optional": true
-        },
-        "@azure/storage-blob": {
-          "optional": true
-        },
-        "@clickhouse/client": {
-          "optional": true
-        },
-        "@cloudflare/ai": {
-          "optional": true
-        },
-        "@elastic/elasticsearch": {
-          "optional": true
-        },
-        "@getmetal/metal-sdk": {
-          "optional": true
-        },
-        "@getzep/zep-js": {
-          "optional": true
-        },
-        "@gomomento/sdk": {
-          "optional": true
-        },
-        "@google-ai/generativelanguage": {
-          "optional": true
-        },
-        "@google-cloud/storage": {
-          "optional": true
-        },
-        "@huggingface/inference": {
-          "optional": true
-        },
-        "@mozilla/readability": {
-          "optional": true
-        },
-        "@notionhq/client": {
-          "optional": true
-        },
-        "@opensearch-project/opensearch": {
-          "optional": true
-        },
-        "@pinecone-database/pinecone": {
-          "optional": true
-        },
-        "@planetscale/database": {
-          "optional": true
-        },
-        "@qdrant/js-client-rest": {
-          "optional": true
-        },
-        "@raycast/api": {
-          "optional": true
-        },
-        "@smithy/eventstream-codec": {
-          "optional": true
-        },
-        "@smithy/protocol-http": {
-          "optional": true
-        },
-        "@smithy/signature-v4": {
-          "optional": true
-        },
-        "@smithy/util-utf8": {
-          "optional": true
-        },
-        "@supabase/postgrest-js": {
-          "optional": true
-        },
-        "@supabase/supabase-js": {
-          "optional": true
-        },
-        "@tensorflow-models/universal-sentence-encoder": {
-          "optional": true
-        },
-        "@tensorflow/tfjs-converter": {
-          "optional": true
-        },
-        "@tensorflow/tfjs-core": {
-          "optional": true
-        },
-        "@upstash/redis": {
-          "optional": true
-        },
-        "@vercel/postgres": {
-          "optional": true
-        },
-        "@writerai/writer-sdk": {
-          "optional": true
-        },
-        "@xata.io/client": {
-          "optional": true
-        },
-        "@xenova/transformers": {
-          "optional": true
-        },
-        "@zilliz/milvus2-sdk-node": {
-          "optional": true
-        },
-        "apify-client": {
-          "optional": true
-        },
-        "axios": {
-          "optional": true
-        },
-        "cassandra-driver": {
-          "optional": true
-        },
-        "cheerio": {
-          "optional": true
-        },
-        "chromadb": {
-          "optional": true
-        },
-        "cohere-ai": {
-          "optional": true
-        },
-        "d3-dsv": {
-          "optional": true
-        },
-        "epub2": {
-          "optional": true
-        },
-        "faiss-node": {
-          "optional": true
-        },
-        "fast-xml-parser": {
-          "optional": true
-        },
-        "firebase-admin": {
-          "optional": true
-        },
-        "google-auth-library": {
-          "optional": true
-        },
-        "googleapis": {
-          "optional": true
-        },
-        "hnswlib-node": {
-          "optional": true
-        },
-        "html-to-text": {
-          "optional": true
-        },
-        "ignore": {
-          "optional": true
-        },
-        "ioredis": {
-          "optional": true
-        },
-        "jsdom": {
-          "optional": true
-        },
-        "llmonitor": {
-          "optional": true
-        },
-        "lodash": {
-          "optional": true
-        },
-        "mammoth": {
-          "optional": true
-        },
-        "mongodb": {
-          "optional": true
-        },
-        "mysql2": {
-          "optional": true
-        },
-        "neo4j-driver": {
-          "optional": true
-        },
-        "node-llama-cpp": {
-          "optional": true
-        },
-        "notion-to-md": {
-          "optional": true
-        },
-        "pdf-parse": {
-          "optional": true
-        },
-        "peggy": {
-          "optional": true
-        },
-        "pg": {
-          "optional": true
-        },
-        "pg-copy-streams": {
-          "optional": true
-        },
-        "pickleparser": {
-          "optional": true
-        },
-        "playwright": {
-          "optional": true
-        },
-        "portkey-ai": {
-          "optional": true
-        },
-        "puppeteer": {
-          "optional": true
-        },
-        "redis": {
-          "optional": true
-        },
-        "replicate": {
-          "optional": true
-        },
-        "sonix-speech-recognition": {
-          "optional": true
-        },
-        "srt-parser-2": {
-          "optional": true
-        },
-        "typeorm": {
-          "optional": true
-        },
-        "typesense": {
-          "optional": true
-        },
-        "usearch": {
-          "optional": true
-        },
-        "vectordb": {
-          "optional": true
-        },
-        "voy-search": {
-          "optional": true
-        },
-        "weaviate-ts-client": {
-          "optional": true
-        },
-        "web-auth-library": {
-          "optional": true
-        },
-        "youtube-transcript": {
-          "optional": true
-        },
-        "youtubei.js": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/langchainhub": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/langchainhub/-/langchainhub-0.0.6.tgz",
-      "integrity": "sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w=="
-    },
-    "node_modules/langsmith": {
-      "version": "0.0.42",
-      "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.0.42.tgz",
-      "integrity": "sha512-sFuN+e7E+pPBIRaRgFqZh/BRBWNHTZNAwi6uj4kydQawooCZYoJmM5snOkiQrhVSvAhgu6xFhLvmfvkPcKzD7w==",
-      "dependencies": {
-        "@types/uuid": "^9.0.1",
-        "commander": "^10.0.1",
-        "p-queue": "^6.6.2",
-        "p-retry": "4",
-        "uuid": "^9.0.0"
-      },
-      "bin": {
-        "langsmith": "dist/cli/main.cjs"
-      }
-    },
-    "node_modules/md5": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
-      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
-      "dependencies": {
-        "charenc": "0.0.2",
-        "crypt": "0.0.2",
-        "is-buffer": "~1.1.6"
-      }
-    },
-    "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/ml-array-mean": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz",
-      "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==",
-      "dependencies": {
-        "ml-array-sum": "^1.1.6"
-      }
-    },
-    "node_modules/ml-array-sum": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/ml-array-sum/-/ml-array-sum-1.1.6.tgz",
-      "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==",
-      "dependencies": {
-        "is-any-array": "^2.0.0"
-      }
-    },
-    "node_modules/ml-distance": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.1.tgz",
-      "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==",
-      "dependencies": {
-        "ml-array-mean": "^1.1.6",
-        "ml-distance-euclidean": "^2.0.0",
-        "ml-tree-similarity": "^1.0.0"
-      }
-    },
-    "node_modules/ml-distance-euclidean": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz",
-      "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q=="
-    },
-    "node_modules/ml-tree-similarity": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz",
-      "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==",
-      "dependencies": {
-        "binary-search": "^1.3.5",
-        "num-sort": "^2.0.0"
-      }
-    },
-    "node_modules/ms": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
-    },
-    "node_modules/node-domexception": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
-      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/jimmywarting"
-        },
-        {
-          "type": "github",
-          "url": "https://paypal.me/jimmywarting"
-        }
-      ],
-      "engines": {
-        "node": ">=10.5.0"
-      }
-    },
-    "node_modules/node-fetch": {
-      "version": "2.7.0",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
-      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-      "dependencies": {
-        "whatwg-url": "^5.0.0"
-      },
-      "engines": {
-        "node": "4.x || >=6.0.0"
-      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/num-sort": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz",
-      "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==",
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/object-hash": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
-      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
-      "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/openai": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.4.0.tgz",
-      "integrity": "sha512-JN0t628Kh95T0IrXl0HdBqnlJg+4Vq0Bnh55tio+dfCnyzHvMLiWyCM9m726MAJD2YkDU4/8RQB6rNbEq9ct2w==",
-      "dependencies": {
-        "@types/node": "^18.11.18",
-        "@types/node-fetch": "^2.6.4",
-        "abort-controller": "^3.0.0",
-        "agentkeepalive": "^4.2.1",
-        "digest-fetch": "^1.3.0",
-        "form-data-encoder": "1.7.2",
-        "formdata-node": "^4.3.2",
-        "node-fetch": "^2.6.7"
-      },
-      "bin": {
-        "openai": "bin/cli"
-      }
-    },
-    "node_modules/openapi-types": {
-      "version": "12.1.3",
-      "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
-      "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="
-    },
-    "node_modules/p-finally": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
-      "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==",
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/p-queue": {
-      "version": "6.6.2",
-      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
-      "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
-      "dependencies": {
-        "eventemitter3": "^4.0.4",
-        "p-timeout": "^3.2.0"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/p-retry": {
-      "version": "4.6.2",
-      "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",
-      "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",
-      "dependencies": {
-        "@types/retry": "0.12.0",
-        "retry": "^0.13.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/p-timeout": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
-      "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
-      "dependencies": {
-        "p-finally": "^1.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/retry": {
-      "version": "0.13.1",
-      "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
-      "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
-      "engines": {
-        "node": ">= 4"
-      }
-    },
-    "node_modules/tr46": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
-    },
-    "node_modules/typescript": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
-      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=14.17"
-      }
-    },
-    "node_modules/uuid": {
-      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
-      "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
-      "funding": [
-        "https://github.com/sponsors/broofa",
-        "https://github.com/sponsors/ctavan"
-      ],
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    },
-    "node_modules/web-streams-polyfill": {
-      "version": "4.0.0-beta.3",
-      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
-      "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
-      "engines": {
-        "node": ">= 14"
-      }
-    },
-    "node_modules/webidl-conversions": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
-    },
-    "node_modules/whatwg-url": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-      "dependencies": {
-        "tr46": "~0.0.3",
-        "webidl-conversions": "^3.0.0"
-      }
-    },
-    "node_modules/yaml": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz",
-      "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==",
-      "engines": {
-        "node": ">= 14"
-      }
-    },
-    "node_modules/zod": {
-      "version": "3.22.4",
-      "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz",
-      "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==",
-      "funding": {
-        "url": "https://github.com/sponsors/colinhacks"
-      }
-    },
-    "node_modules/zod-to-json-schema": {
-      "version": "3.21.4",
-      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.21.4.tgz",
-      "integrity": "sha512-fjUZh4nQ1s6HMccgIeE0VP4QG/YRGPmyjO9sAh890aQKPEk3nqbfUXhMFaC+Dr5KvYBm8BCyvfpZf2jY9aGSsw==",
-      "peerDependencies": {
-        "zod": "^3.21.4"
-      }
-    }
-  }
-}
--- a/examples/langchain-typescript-simple/package.json
+++ b/examples/langchain-typescript-simple/package.json
@@ -1,8 +0,0 @@
-{
-  "devDependencies": {
-    "typescript": "^5.2.2"
-  },
-  "dependencies": {
-    "langchain": "^0.0.165"
-  }
-}
--- a/examples/langchain-python-rag-websummary/README.md
+++ b/examples/langchain-python-rag-websummary/README.md
--- a/examples/langchain-python-rag-websummary/main.py
+++ b/examples/langchain-python-rag-websummary/main.py
--- a/examples/langchain-python-rag-websummary/requirements.txt
+++ b/examples/langchain-python-rag-websummary/requirements.txt
--- a/examples/langchain-python-simple/README.md
+++ b/examples/langchain-python-simple/README.md
--- a/examples/langchain-python-simple/main.py
+++ b/examples/langchain-python-simple/main.py
--- a/examples/langchain-python-simple/requirements.txt
+++ b/examples/langchain-python-simple/requirements.txt
--- a/examples/modelfile-mario/Modelfile
+++ b/examples/modelfile-mario/Modelfile
--- a/examples/modelfile-mario/logo.png
+++ b/examples/modelfile-mario/logo.png
--- a/examples/modelfile-mario/readme.md
+++ b/examples/modelfile-mario/readme.md
--- a/examples/midjourney-prompter/Modelfile
+++ b/examples/midjourney-prompter/Modelfile
@@ -0,0 +1,8 @@
+# Modelfile for creating a Midjourney prompts from a topic
+# This prompt was adapted from the original at https://www.greataiprompts.com/guide/midjourney/best-chatgpt-prompt-for-midjourney/
+# Run `ollama create mj -f ./Modelfile` and then `ollama run mj` and enter a topic
+
+FROM nous-hermes
+SYSTEM """
+Embrace your role as an AI-powered creative assistant, employing Midjourney to manifest compelling AI-generated art. I will outline a specific image concept, and in response, you must produce an exhaustive, multifaceted prompt for Midjourney, ensuring every detail of the original concept is represented in your instructions. Midjourney doesn't do well with text, so after the prompt, give me instructions that I can use to create the titles in a image editor.
+"""
--- a/examples/modelfile-10tweets/Modelfile
+++ b/examples/modelfile-10tweets/Modelfile
@@ -1,7 +0,0 @@
-# Modelfile for creating a list of ten tweets from a topic
-# Run `ollama create 10tweets -f ./Modelfile` and then `ollama run 10tweets` and enter a topic
-
-FROM llama2
-SYSTEM """
-You are a content marketer who needs to come up with 10 short but succinct tweets. The answer should be a list of ten tweets. Each tweet can have a maximum of 280 characters and should include hashtags. Each user input will be a subject and you should expand it in ten creative ways. Never stop after just one tweet. Always include ten. 
-"""
--- a/examples/modelfile-10tweets/README.md
+++ b/examples/modelfile-10tweets/README.md
@@ -1,23 +0,0 @@
-# Ten Tweets Modelfile
-
-This is a simple modelfile that generates ten tweets based off any topic.
-
-```bash
-ollama create tentweets
-
-ollama run tentweets
->>> underwater basketweaving
- Great! Here are ten creative tweets about underwater basketweaving:
-
-1. "Just discovered the ultimate stress-reliever: Underwater basketweaving! 🌊🧵 #UnderwaterBasketweaving #StressRelief"
-2. "Who needs meditation when you can do underwater basketweaving? 😴👀 #PeacefulDistraction #UnderwaterBasketweaving"
-3. "Just spent an hour in the pool and still managed to knot my basket. Goal: untangle it before next session. 💪🏽 #ChallengeAccepted #UnderwaterBasketweaving"
-4. "When life gives you lemons, make underwater basketweaving! 🍋🧵 #LemonadeLife #UnderwaterBasketweaving"
-5. "Just realized my underwater basketweaving skills could come in handy during a zombie apocalypse. 😂🧡 #SurvivalTips #UnderwaterBasketweaving"
-6. "I'm not lazy, I'm just conserving energy for my next underwater basketweaving session. 😴💤 #LazyDay #UnderwaterBasketweaving"
-7. "Just found my inner peace while doing underwater basketweaving. It's like meditation, but with knots! 🙏🧵 #Mindfulness #UnderwaterBasketweaving"
-8. "Why study for exams when you can do underwater basketweaving and forget all your worries? 😜🧵 #ProcrastinationStation #UnderwaterBasketweaving"
-9. "Just had to cut my underwater basketweaving session short due to a sudden urge to breathe. 🤯🌊 #AquaticAdventures #UnderwaterBasketweaving"
-10. "I'm not sure what's more impressive: my underwater basketweaving skills or the fact that I didn't drown trying to make this tweet. 😅🧵 #Accomplishment 
-#UnderwaterBasketweaving"
-```
--- a/examples/modelfile-devopsengineer/README.md
+++ b/examples/modelfile-devopsengineer/README.md
@@ -1,40 +0,0 @@
-# DevOps Engineer Modelfile
-
-This is a simple modelfile that takes will respond to any questions that a senior DevOps engineer should know how to solve.
-
-````text
-ollama create devopsengineer
-
-ollama run devopsengineer
->>> how can i proxy all connections from inside the office through a single server that caches large files. Provide a config file for nginx to achieve this?
-
-Here is an example configuration file for Nginx that can be used to proxy all connections from inside the office network to a centralized cache server:
-```perl
-http {
-    upstream cache_server {
-        server <cache_server_ip>;
-        server <cache_server_ip>;
-        server <cache_server_ip>;
-    }
-    
-    server {
-        listen 80;
-        server_name localhost;
-        
-        location / {
-            proxy_pass http://cache_server;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        }
-    }
-}
-```
-In this configuration, we define an `upstream` block that lists the IP addresses of the cache servers. We then define a server block that listens on port 80 and routes all 
-traffic to the `/` location to the cache servers using the `proxy_pass` directive. The `proxy_set_header` directives are used to preserve the source IP address of the client
-request when forwarding it to the cache server.
-
-To use this configuration, you would need to replace the placeholder `<cache_server_ip>` with the actual IP addresses of your cache servers. You would also need to make sure
-that the cache servers are configured to accept incoming connections from the Nginx server and handle requests for files.
-
-````
--- a/examples/modelfile-midjourney/Modelfile
+++ b/examples/modelfile-midjourney/Modelfile
@@ -1,11 +0,0 @@
-# Modelfile for creating a Midjourney prompts from a topic
-# This prompt was adapted from the original at https://www.greataiprompts.com/guide/midjourney/best-chatgpt-prompt-for-midjourney/
-# Run `ollama create mj -f ./Modelfile` and then `ollama run mj` and enter a topic
-
-FROM zephyr
-PARAMETER temperature 0.8
-PARAMETER top_k 500
-PARAMETER top_p 0.9
-SYSTEM """
-Embrace your role as a creative illustrator. Based on a concept provided, you must produce a single paragraph with a multifaceted description of an image, ensuring significant details of the concept and more is represented in your instructions. You do not need to write complete sentences but rather short concepts with the following information: the level of detail that should be represented, an artistic style and maybe a specific name of a painter or illustrator, the ideal color pallete, lighting, mood, perspective, the setting, time of day, weather, the season, the time period, location, materials, the textures, patterns, lines, brushstrokes, techniques, the medium, the genre, the rendering style. Don't include everything and keep the description length under 250 words. 
-"""
--- a/examples/modelfile-midjourney/README.md
+++ b/examples/modelfile-midjourney/README.md
@@ -1,11 +0,0 @@
-# Midjourney Prompt Generator Modelfile
-
-This simple modelfile will help create a prompt to feed to Midjourney.
-
-```text
-ollama create midjourney
-
-ollama run midjourney
->>> a sports car in the mountains. 
-A sleek, high-performance automobile cuts through a serpentine mountain landscape. The concept is a classic illustration of speed and power, depicted in the style of pop art by Andy Warhol. The color palette is dominated by bold, primary hues of red, blue, and yellow, with striking accent colors of white, black, and metallic shades. The lighting is bright and focused, casting sharp shadows on the rugged terrain. A sense of excitement and anticipation permeates throughout the scene, as the car navigates a treacherous course through the winding road. The perspective is low, allowing for a full view of the vehicle's sleek lines and intricate details. The setting takes place in the afternoon during a sunny day in autumn, as evidenced by the vibrant foliage on the mountainside. The time period is modern, with nods to classic car design. The materials are primarily digital, allowing for smooth curves and sharp contrasts. The textures are sleek and polished, with meticulously detailed lines and brushstrokes that accentuate the car's aerodynamic design. The patterns consist of geometric shapes and bold stripes, adding to the car's dynamic appeal. The genre is modern realism, with a focus on precision and detail. The rendering style is highly technical, capturing the nuances and subtleties of the vehicle and its surroundings in breathtaking detail.
-```
--- a/examples/modelfile-recipemaker/README.md
+++ b/examples/modelfile-recipemaker/README.md
@@ -1,20 +0,0 @@
-# Recipe Maker Modelfile 
-
-Simple modelfile to generate a recipe from a short list of ingredients.
-
-```
-ollama create recipemaker
-
-ollama run recipemaker
->>> chilli pepper, white chocolate, kale
- Ingredients:
- 1 small chili pepper
- 4 squares of white chocolate
- handful of kale leaves
-
-Instructions:
-1. In a blender or food processor, puree the chilies and white chocolate until smooth.
-2. Add the chopped kale leaves to the blender and pulse until well combined.
-3. Serve immediately as a dip for crackers or use it as an ingredient in your favorite recipe. The mixture of spicy chili pepper with sweet white chocolate and nutritious 
-kale will make your taste buds dance with delight!
-```
--- a/examples/modelfile-sentiments/Modelfile
+++ b/examples/modelfile-sentiments/Modelfile
@@ -1,28 +0,0 @@
-# Modelfile for creating a sentiment analyzer. 
-# Run `ollama create sentiments -f pathtofile` and then `ollama run sentiments` and enter a topic
-
-FROM orca
-TEMPLATE """
-{{- if .First }}
-### System:
-{{ .System }}
-{{- end }}
-### User: 
-I hate it when my phone dies
-### Response: 
-NEGATIVE
-### User: 
-He is awesome
-### Response: 
-POSITIVE
-### User: 
-This is the link to the article
-### Response: 
-NEUTRAL
-### User:
-{{ .Prompt }}
-
-### Response:
-"""
-
-SYSTEM """You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text."""
--- a/examples/modelfile-sentiments/Readme.md
+++ b/examples/modelfile-sentiments/Readme.md
@@ -1,25 +0,0 @@
-# Sentiments Modelfile
-
-This is a simple sentiments analyzer using the Orca model. When you pull Orca from the registry, it has a Template already defined that looks like this:
-
-```Modelfile
-{{- if .First }}
-### System:
-{{ .System }}
-{{- end }}
-
-### User:
-{{ .Prompt }}
-
-### Response:
-```
-
-If we just wanted to have the text:
-
-```Plaintext
-You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text.
-```
-
-then we could have put this in a SYSTEM block. But we want to provide examples which require updating the full Template. Any Modelfile you create will inherit all the settings from the source model. But in this example, we are overriding the Template.
-
-When providing examples for the input and output, you should include the way the model usually provides information. Since the Orca model expects a user prompt to appear after ### User: and the response is after ### Response, we should format our examples like that as well. If we were using the Llama 2 model, the format would be a bit different.
--- a/examples/langchain-python-rag-privategpt/.gitignore
+++ b/examples/langchain-python-rag-privategpt/.gitignore
--- a/examples/langchain-python-rag-privategpt/LICENSE
+++ b/examples/langchain-python-rag-privategpt/LICENSE
--- a/examples/langchain-python-rag-privategpt/README.md
+++ b/examples/langchain-python-rag-privategpt/README.md
--- a/examples/langchain-python-rag-privategpt/constants.py
+++ b/examples/langchain-python-rag-privategpt/constants.py
--- a/examples/langchain-python-rag-privategpt/ingest.py
+++ b/examples/langchain-python-rag-privategpt/ingest.py
--- a/examples/langchain-python-rag-privategpt/poetry.lock
+++ b/examples/langchain-python-rag-privategpt/poetry.lock
--- a/examples/langchain-python-rag-privategpt/privateGPT.py
+++ b/examples/langchain-python-rag-privategpt/privateGPT.py
--- a/examples/langchain-python-rag-privategpt/pyproject.toml
+++ b/examples/langchain-python-rag-privategpt/pyproject.toml
--- a/examples/langchain-python-rag-privategpt/requirements.txt
+++ b/examples/langchain-python-rag-privategpt/requirements.txt
--- a/examples/python-rag-newssummary/README.md
+++ b/examples/python-rag-newssummary/README.md
@@ -1,22 +0,0 @@
-# News Summarizer
-
-This example goes through a series of steps:
-
-  1. You choose a topic area (e.g., "news", "NVidia", "music", etc.).
-  2. Gets the most recent articles on that topic from various sources.
-  3. Uses Ollama to summarize each article.
-  4. Creates chunks of sentences from each article.
-  5. Uses Sentence Transformers to generate embeddings for each of those chunks.
-  6. You enter a question regarding the summaries shown.
-  7. Uses Sentence Transformers to generate an embedding for that question.
-  8. Uses the embedded question to find the most similar chunks.
-  9. Feeds all that to Ollama to generate a good answer to your question based on these news articles.
-
-This example lets you pick from a few different topic areas, then summarize the most recent x articles for that topic. It then creates chunks of sentences from each article and then generates embeddings for each of those chunks.
-
-You can run the example like this:
-
-```bash
-pip install -r requirements.txt
-python summ.py
-```
--- a/examples/python-rag-newssummary/requirements.txt
+++ b/examples/python-rag-newssummary/requirements.txt
@@ -1,9 +0,0 @@
-beautifulsoup4==4.12.2
-feedparser==6.0.10
-mattsollamatools==0.0.8
-newspaper3k==0.2.8
-nltk==3.8.1
-numpy==1.24.3
-Requests==2.31.0
-scikit_learn==1.3.0
-sentence_transformers==2.2.2
--- a/examples/python-rag-newssummary/summ.py
+++ b/examples/python-rag-newssummary/summ.py
@@ -1,86 +0,0 @@
-import curses
-import json
-from utils import get_url_for_topic, topic_urls, menu, getUrls, get_summary, getArticleText, knn_search
-import requests
-from sentence_transformers import SentenceTransformer
-from mattsollamatools import chunker
-
-if __name__ == "__main__":
-    chosen_topic = curses.wrapper(menu)
-    print("Here is your news summary:\n")
-    urls = getUrls(chosen_topic, n=5)
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    allEmbeddings = []
-
-    for url in urls:
-      article={}
-      article['embeddings'] = []
-      article['url'] = url
-      text = getArticleText(url)
-      summary = get_summary(text)
-      chunks = chunker(text)  # Use the chunk_text function from web_utils
-      embeddings = model.encode(chunks)
-      for (chunk, embedding) in zip(chunks, embeddings):
-        item = {}
-        item['source'] = chunk
-        item['embedding'] = embedding.tolist()  # Convert NumPy array to list
-        item['sourcelength'] = len(chunk)
-        article['embeddings'].append(item)
-    
-      allEmbeddings.append(article)
-
-      print(f"{summary}\n")
-
-    
-    while True:
-      context = []
-      # Input a question from the user
-      question = input("Enter your question about the news, or type quit: ")
-
-      if question.lower() == 'quit':
-        break
-
-      # Embed the user's question
-      question_embedding = model.encode([question])
-
-      # Perform KNN search to find the best matches (indices and source text)
-      best_matches = knn_search(question_embedding, allEmbeddings, k=10)
-
-
-      sourcetext=""
-      for i, (index, source_text) in enumerate(best_matches, start=1):
-          sourcetext += f"{i}. Index: {index}, Source Text: {source_text}"
-
-      systemPrompt = f"Only use the following information to answer the question. Do not use anything else: {sourcetext}"
-
-      url = "http://localhost:11434/api/generate"
-
-      payload = {
-      "model": "mistral-openorca",
-      "prompt": question, 
-      "system": systemPrompt,
-      "stream": False, 
-      "context": context
-      }
-
-      # Convert the payload to a JSON string
-      payload_json = json.dumps(payload)
-
-      # Set the headers to specify JSON content
-      headers = {
-          "Content-Type": "application/json"
-      }
-
-      # Send the POST request
-      response = requests.post(url, data=payload_json, headers=headers)
-
-      # Check the response
-      if response.status_code == 200:
-          output = json.loads(response.text)
-          context = output['context']
-          print(output['response']+ "\n")
-          
-
-      else:
-          print(f"Request failed with status code {response.status_code}")
-
--- a/examples/python-rag-newssummary/utils.py
+++ b/examples/python-rag-newssummary/utils.py
@@ -1,108 +0,0 @@
-import curses
-import feedparser
-import requests
-import unicodedata
-import json
-from newspaper import Article
-from bs4 import BeautifulSoup
-from nltk.tokenize import sent_tokenize, word_tokenize
-import numpy as np
-from sklearn.neighbors import NearestNeighbors
-from mattsollamatools import chunker
-
-# Create a dictionary to store topics and their URLs
-topic_urls = {
-    "Mac": "https://9to5mac.com/guides/mac/feed",
-    "News": "http://www.npr.org/rss/rss.php?id=1001",
-    "Nvidia": "https://nvidianews.nvidia.com/releases.xml",
-    "Raspberry Pi": "https://www.raspberrypi.com/news/feed/", 
-    "Music": "https://www.billboard.com/c/music/music-news/feed/"
-}
-
-# Use curses to create a menu of topics
-def menu(stdscr):
-    chosen_topic = get_url_for_topic(stdscr)  
-    url = topic_urls[chosen_topic] if chosen_topic in topic_urls else "Topic not found"
-    
-    stdscr.addstr(len(topic_urls) + 3, 0, f"Selected URL for {chosen_topic}: {url}")
-    stdscr.refresh()
-    
-    return chosen_topic
-
-# You have chosen a topic. Now return the url for that topic
-def get_url_for_topic(stdscr):
-    curses.curs_set(0)  # Hide the cursor
-    stdscr.clear()
-
-    stdscr.addstr(0, 0, "Choose a topic using the arrow keys (Press Enter to select):")
-
-    # Create a list of topics
-    topics = list(topic_urls.keys())
-    current_topic = 0
-
-    while True:
-        for i, topic in enumerate(topics):
-            if i == current_topic:
-                stdscr.addstr(i + 2, 2, f"> {topic}")
-            else:
-                stdscr.addstr(i + 2, 2, f"  {topic}")
-
-        stdscr.refresh()
-
-        key = stdscr.getch()
-
-        if key == curses.KEY_DOWN and current_topic < len(topics) - 1:
-            current_topic += 1
-        elif key == curses.KEY_UP and current_topic > 0:
-            current_topic -= 1
-        elif key == 10:  # Enter key
-            return topic_urls[topics[current_topic]]
-
-# Get the last N URLs from an RSS feed
-def getUrls(feed_url, n=20):
-    feed = feedparser.parse(feed_url)
-    entries = feed.entries[-n:]
-    urls = [entry.link for entry in entries]
-    return urls
-
-# Often there are a bunch of ads and menus on pages for a news article. This uses newspaper3k to get just the text of just the article.
-def getArticleText(url):
-  article = Article(url)
-  article.download()
-  article.parse()
-  return article.text
-
-def get_summary(text):
-  systemPrompt = "Write a concise summary of the text, return your responses with 5 lines that cover the key points of the text given."
-  prompt = text
-  
-  url = "http://localhost:11434/api/generate"
-
-  payload = {
-    "model": "mistral-openorca",
-    "prompt": prompt, 
-    "system": systemPrompt,
-    "stream": False
-  }
-  payload_json = json.dumps(payload)
-  headers = {"Content-Type": "application/json"}
-  response = requests.post(url, data=payload_json, headers=headers)
-
-  return json.loads(response.text)["response"]
-
-# Perform K-nearest neighbors (KNN) search
-def knn_search(question_embedding, embeddings, k=5):
-    X = np.array([item['embedding'] for article in embeddings for item in article['embeddings']])
-    source_texts = [item['source'] for article in embeddings for item in article['embeddings']]
-    
-    # Fit a KNN model on the embeddings
-    knn = NearestNeighbors(n_neighbors=k, metric='cosine')
-    knn.fit(X)
-    
-    # Find the indices and distances of the k-nearest neighbors
-    distances, indices = knn.kneighbors(question_embedding, n_neighbors=k)
-    
-    # Get the indices and source texts of the best matches
-    best_matches = [(indices[0][i], source_texts[indices[0][i]]) for i in range(k)]
-    
-    return best_matches
--- a/examples/python-simplegenerate/client.py
+++ b/examples/python-simplegenerate/client.py
--- a/examples/modelfile-recipemaker/Modelfile
+++ b/examples/modelfile-recipemaker/Modelfile
--- a/examples/modelfile-tweetwriter/Modelfile
+++ b/examples/modelfile-tweetwriter/Modelfile
--- a/examples/typescript-mentors/.gitignore
+++ b/examples/typescript-mentors/.gitignore
@@ -1,2 +0,0 @@
-node_modules
-package-lock.json
--- a/examples/typescript-mentors/README.md
+++ b/examples/typescript-mentors/README.md
@@ -1,21 +0,0 @@
-# Ask the Mentors
-
-This example demonstrates how one would create a set of 'mentors' you can have a conversation with. The mentors are generated using the `character-generator.ts` file. This will use **Stable Beluga 70b** to create a bio and list of verbal ticks and common phrases used by each person. Then `mentors.ts` will take a question, and choose three of the 'mentors' and start a conversation with them. Occasionally, they will talk to each other, and other times they will just deliver a set of monologues. It's fun to see what they do and say.
-
-## Usage
-
-```bash
-ts-node ./character-generator.ts "Lorne Greene"
-```
-
-This will create `lornegreene/Modelfile`. Now you can create a model with this command:
-
-```bash
-ollama create lornegreene -f lornegreene/Modelfile
-```
-
-If you want to add your own mentors, you will have to update the code to look at your namespace instead of **mattw**. Also set the list of mentors to include yours.
-
-```bash
-ts-node ./mentors.ts "What is a Jackalope?"
-```
--- a/examples/typescript-mentors/character-generator.ts
+++ b/examples/typescript-mentors/character-generator.ts
@@ -1,26 +0,0 @@
-import { Ollama } from 'ollama-node'
-import fs from 'fs';
-import path from 'path';
-
-async function characterGenerator() {
-  const character = process.argv[2];
-  console.log(`You are creating a character for ${character}.`);
-  const foldername = character.replace(/\s/g, '').toLowerCase();
-  const directory = path.join(__dirname, foldername);
-  if (!fs.existsSync(directory)) {
-    fs.mkdirSync(directory, { recursive: true });
-  }
-
-  const ollama = new Ollama();
-  ollama.setModel("stablebeluga2:70b-q4_K_M");
-  const bio = await ollama.generate(`create a bio of ${character} in a single long paragraph. Instead of saying '${character} is...' or '${character} was...' use language like 'You are...' or 'You were...'. Then create a paragraph describing the speaking mannerisms and style of ${character}. Don't include anything about how ${character} looked or what they sounded like, just focus on the words they said. Instead of saying '${character} would say...' use language like 'You should say...'. If you use quotes, always use single quotes instead of double quotes. If there are any specific words or phrases you used a lot, show how you used them. `);
-
-  const thecontents = `FROM llama2\nSYSTEM """\n${bio.response.replace(/(\r\n|\n|\r)/gm, " ").replace('would', 'should')} All answers to questions should be related back to what you are most known for.\n"""`;
-
-  fs.writeFile(path.join(directory, 'Modelfile'), thecontents, (err: any) => {
-    if (err) throw err;
-    console.log('The file has been saved!');
-  });
-}
-
-characterGenerator();
--- a/examples/typescript-mentors/mentors.ts
+++ b/examples/typescript-mentors/mentors.ts
@@ -1,59 +0,0 @@
-import { Ollama } from 'ollama-node';
-
-const mentorCount = 3;
-const ollama = new Ollama();
-
-function getMentors(): string[] {
-  const mentors = ['Gary Vaynerchuk', 'Kanye West', 'Martha Stewart', 'Neil deGrasse Tyson', 'Owen Wilson', 'Ronald Reagan', 'Donald Trump', 'Barack Obama', 'Jeff Bezos'];
-  const chosenMentors: string[] = [];
-  for (let i = 0; i < mentorCount; i++) {
-    const mentor = mentors[Math.floor(Math.random() * mentors.length)];
-    chosenMentors.push(mentor);
-    mentors.splice(mentors.indexOf(mentor), 1);
-  }
-  return chosenMentors;
-}
-
-function getMentorFileName(mentor: string): string {
-  const model = mentor.toLowerCase().replace(/\s/g, '');
-  return `mattw/${model}`;
-}
-
-async function getSystemPrompt(mentor: string, isLast: boolean, question: string): Promise<string> {
-  ollama.setModel(getMentorFileName(mentor));
-  const info = await ollama.showModelInfo()
-  let SystemPrompt = info.system || '';
-  SystemPrompt += ` You should continue the conversation as if you were ${mentor} and acknowledge the people before you in the conversation. You should adopt their mannerisms and tone, but also not use language they wouldn't use. If they are not known to know about the concept in the question, don't offer an answer. Your answer should be no longer than 1 paragraph. And definitely try not to sound like anyone else. Don't repeat any slang or phrases already used. And if it is a question the original ${mentor} wouldn't have know the answer to, just say that you don't know, in the style of ${mentor}. And think about the time the person lived. Don't use terminology that they wouldn't have used.`
-
-  if (isLast) {
-    SystemPrompt += ` End your answer with something like I hope our answers help you out`;
-  } else {
-    SystemPrompt += ` Remember, this is a conversation, so you don't need a conclusion, but end your answer with a question related to the first question: "${question}".`;
-  }
-  return SystemPrompt;
-}
-
-async function main() {
-  const mentors = getMentors();
-  const question = process.argv[2];
-  let theConversation = `Here is the conversation so far.\nYou: ${question}\n`
-
-  for await (const mentor of mentors) {
-    const SystemPrompt = await getSystemPrompt(mentor, mentor === mentors[mentorCount - 1], question);
-    ollama.setModel(getMentorFileName(mentor));
-    ollama.setSystemPrompt(SystemPrompt);
-    let output = '';
-    process.stdout.write(`\n${mentor}: `);
-    for await (const chunk of ollama.streamingGenerate(theConversation + `Continue the conversation as if you were ${mentor} on the question "${question}".`)) {
-      if (chunk.response) {
-        output += chunk.response;
-        process.stdout.write(chunk.response);
-      } else {
-        process.stdout.write('\n');
-      }
-    }
-    theConversation += `${mentor}: ${output}\n\n`
-  }
-}
-
-main();
--- a/examples/typescript-mentors/package.json
+++ b/examples/typescript-mentors/package.json
@@ -1,7 +0,0 @@
-{
-  "dependencies": {
-    "fs": "^0.0.1-security",
-    "ollama-node": "^0.0.3",
-    "path": "^0.12.7"
-  }
-}
--- a/format/bytes.go
+++ b/format/bytes.go
@@ -1,23 +0,0 @@
-package format
-
-import "fmt"
-
-const (
-	Byte     = 1
-	KiloByte = Byte * 1000
-	MegaByte = KiloByte * 1000
-	GigaByte = MegaByte * 1000
-)
-
-func HumanBytes(b int64) string {
-	switch {
-	case b > GigaByte:
-		return fmt.Sprintf("%d GB", b/GigaByte)
-	case b > MegaByte:
-		return fmt.Sprintf("%d MB", b/MegaByte)
-	case b > KiloByte:
-		return fmt.Sprintf("%d KB", b/KiloByte)
-	default:
-		return fmt.Sprintf("%d B", b)
-	}
-}
--- a/format/openssh.go
+++ b/format/openssh.go
@@ -10,11 +10,15 @@ package format

 import (
 	"crypto"
+	"crypto/ecdsa"
 	"crypto/ed25519"
+	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/rsa"
 	"encoding/binary"
 	"encoding/pem"
 	"fmt"
+	"math/big"

 	"golang.org/x/crypto/ssh"
 )
@@ -37,6 +41,25 @@ type openSSHPrivateKey struct {
 	Rest    []byte `ssh:"rest"`
 }

+type openSSHRSAPrivateKey struct {
+	N       *big.Int
+	E       *big.Int
+	D       *big.Int
+	Iqmp    *big.Int
+	P       *big.Int
+	Q       *big.Int
+	Comment string
+	Pad     []byte `ssh:"rest"`
+}
+
+type openSSHECDSAPrivateKey struct {
+	Curve   string
+	Pub     []byte
+	D       *big.Int
+	Comment string
+	Pad     []byte `ssh:"rest"`
+}
+
 type openSSHEd25519PrivateKey struct {
 	Pub     []byte
 	Priv    []byte
@@ -62,6 +85,64 @@ func OpenSSHPrivateKey(key crypto.PrivateKey, comment string) (*pem.Block, error
 	}

 	switch k := key.(type) {
+	case *rsa.PrivateKey:
+		e := new(big.Int).SetInt64(int64(k.E))
+
+		key := openSSHRSAPrivateKey{
+			N:       k.N,
+			E:       e,
+			D:       k.D,
+			Iqmp:    k.Precomputed.Qinv,
+			P:       k.Primes[0],
+			Q:       k.Primes[1],
+			Comment: comment,
+		}
+
+		pk1.Keytype = ssh.KeyAlgoRSA
+		pk1.Rest = ssh.Marshal(key)
+
+		w.PubKey = ssh.Marshal(struct {
+			KeyType string
+			E       *big.Int
+			N       *big.Int
+		}{
+			ssh.KeyAlgoRSA, e, k.N,
+		})
+	case *ecdsa.PrivateKey:
+		var curve, keytype string
+		switch name := k.Curve.Params().Name; name {
+		case "P-256":
+			curve = "nistp256"
+			keytype = ssh.KeyAlgoECDSA256
+		case "P-384":
+			curve = "nistp384"
+			keytype = ssh.KeyAlgoECDSA384
+		case "P-521":
+			curve = "nistp521"
+			keytype = ssh.KeyAlgoECDSA521
+		default:
+			return nil, fmt.Errorf("ssh: unknown curve %q", name)
+		}
+
+		pub := elliptic.Marshal(k.Curve, k.X, k.Y)
+
+		key := openSSHECDSAPrivateKey{
+			Curve:   curve,
+			Pub:     pub,
+			D:       k.D,
+			Comment: comment,
+		}
+
+		pk1.Keytype = keytype
+		pk1.Rest = ssh.Marshal(key)
+
+		w.PubKey = ssh.Marshal(struct {
+			KeyType string
+			Curve   string
+			Pub     []byte
+		}{
+			keytype, curve, pub,
+		})
 	case ed25519.PrivateKey:
 		pub, priv := k[32:], k
 		key := openSSHEd25519PrivateKey{
--- a/format/time.go
+++ b/format/time.go
@@ -7,14 +7,26 @@ import (
 	"time"
 )

-// humanDuration returns a human-readable approximation of a
-// duration (eg. "About a minute", "4 hours ago", etc.).
-func humanDuration(d time.Duration) string {
+// HumanDuration returns a human-readable approximation of a duration
+// (eg. "About a minute", "4 hours ago", etc.).
+// Modified version of github.com/docker/go-units.HumanDuration
+func HumanDuration(d time.Duration) string {
+	return HumanDurationWithCase(d, true)
+}
+
+// HumanDurationWithCase returns a human-readable approximation of a
+// duration (eg. "About a minute", "4 hours ago", etc.). but allows
+// you to specify whether the first word should be capitalized
+// (eg. "About" vs. "about")
+func HumanDurationWithCase(d time.Duration, useCaps bool) string {
 	seconds := int(d.Seconds())

 	switch {
 	case seconds < 1:
-		return "Less than a second"
+		if useCaps {
+			return "Less than a second"
+		}
+		return "less than a second"
 	case seconds == 1:
 		return "1 second"
 	case seconds < 60:
@@ -24,7 +36,10 @@ func humanDuration(d time.Duration) string {
 	minutes := int(d.Minutes())
 	switch {
 	case minutes == 1:
-		return "About a minute"
+		if useCaps {
+			return "About a minute"
+		}
+		return "about a minute"
 	case minutes < 60:
 		return fmt.Sprintf("%d minutes", minutes)
 	}
@@ -32,7 +47,10 @@ func humanDuration(d time.Duration) string {
 	hours := int(math.Round(d.Hours()))
 	switch {
 	case hours == 1:
-		return "About an hour"
+		if useCaps {
+			return "About an hour"
+		}
+		return "about an hour"
 	case hours < 48:
 		return fmt.Sprintf("%d hours", hours)
 	case hours < 24*7*2:
@@ -47,22 +65,77 @@ func humanDuration(d time.Duration) string {
 }

 func HumanTime(t time.Time, zeroValue string) string {
-	return humanTime(t, zeroValue)
+	return humanTimeWithCase(t, zeroValue, true)
 }

 func HumanTimeLower(t time.Time, zeroValue string) string {
-	return strings.ToLower(humanTime(t, zeroValue))
+	return humanTimeWithCase(t, zeroValue, false)
 }

-func humanTime(t time.Time, zeroValue string) string {
+func humanTimeWithCase(t time.Time, zeroValue string, useCaps bool) string {
 	if t.IsZero() {
 		return zeroValue
 	}

 	delta := time.Since(t)
 	if delta < 0 {
-		return humanDuration(-delta) + " from now"
+		return HumanDurationWithCase(-delta, useCaps) + " from now"
+	}
+	return HumanDurationWithCase(delta, useCaps) + " ago"
+}
+
+// ExcatDuration returns a human readable hours/minutes/seconds or milliseconds format of a duration
+// the most precise level of duration is milliseconds
+func ExactDuration(d time.Duration) string {
+	if d.Seconds() < 1 {
+		if d.Milliseconds() == 1 {
+			return fmt.Sprintf("%d millisecond", d.Milliseconds())
+		}
+		return fmt.Sprintf("%d milliseconds", d.Milliseconds())
 	}

-	return humanDuration(delta) + " ago"
+	var readableDur strings.Builder
+
+	dur := d.String()
+
+	// split the default duration string format of 0h0m0s into something nicer to read
+	h := strings.Split(dur, "h")
+	if len(h) > 1 {
+		hours := h[0]
+		if hours == "1" {
+			readableDur.WriteString(fmt.Sprintf("%s hour ", hours))
+		} else {
+			readableDur.WriteString(fmt.Sprintf("%s hours ", hours))
+		}
+		dur = h[1]
+	}
+
+	m := strings.Split(dur, "m")
+	if len(m) > 1 {
+		mins := m[0]
+		switch mins {
+		case "0":
+			// skip
+		case "1":
+			readableDur.WriteString(fmt.Sprintf("%s minute ", mins))
+		default:
+			readableDur.WriteString(fmt.Sprintf("%s minutes ", mins))
+		}
+		dur = m[1]
+	}
+
+	s := strings.Split(dur, "s")
+	if len(s) > 0 {
+		sec := s[0]
+		switch sec {
+		case "0":
+			// skip
+		case "1":
+			readableDur.WriteString(fmt.Sprintf("%s second ", sec))
+		default:
+			readableDur.WriteString(fmt.Sprintf("%s seconds ", sec))
+		}
+	}
+
+	return strings.TrimSpace(readableDur.String())
 }
--- a/format/time_test.go
+++ b/format/time_test.go
@@ -11,25 +11,92 @@ func assertEqual(t *testing.T, a interface{}, b interface{}) {
 	}
 }

+func TestHumanDuration(t *testing.T) {
+	day := 24 * time.Hour
+	week := 7 * day
+	month := 30 * day
+	year := 365 * day
+
+	assertEqual(t, "Less than a second", HumanDuration(450*time.Millisecond))
+	assertEqual(t, "Less than a second", HumanDurationWithCase(450*time.Millisecond, true))
+	assertEqual(t, "less than a second", HumanDurationWithCase(450*time.Millisecond, false))
+	assertEqual(t, "1 second", HumanDuration(1*time.Second))
+	assertEqual(t, "45 seconds", HumanDuration(45*time.Second))
+	assertEqual(t, "46 seconds", HumanDuration(46*time.Second))
+	assertEqual(t, "59 seconds", HumanDuration(59*time.Second))
+	assertEqual(t, "About a minute", HumanDuration(60*time.Second))
+	assertEqual(t, "About a minute", HumanDurationWithCase(1*time.Minute, true))
+	assertEqual(t, "about a minute", HumanDurationWithCase(1*time.Minute, false))
+	assertEqual(t, "3 minutes", HumanDuration(3*time.Minute))
+	assertEqual(t, "35 minutes", HumanDuration(35*time.Minute))
+	assertEqual(t, "35 minutes", HumanDuration(35*time.Minute+40*time.Second))
+	assertEqual(t, "45 minutes", HumanDuration(45*time.Minute))
+	assertEqual(t, "45 minutes", HumanDuration(45*time.Minute+40*time.Second))
+	assertEqual(t, "46 minutes", HumanDuration(46*time.Minute))
+	assertEqual(t, "59 minutes", HumanDuration(59*time.Minute))
+	assertEqual(t, "About an hour", HumanDuration(1*time.Hour))
+	assertEqual(t, "About an hour", HumanDurationWithCase(1*time.Hour+29*time.Minute, true))
+	assertEqual(t, "about an hour", HumanDurationWithCase(1*time.Hour+29*time.Minute, false))
+	assertEqual(t, "2 hours", HumanDuration(1*time.Hour+31*time.Minute))
+	assertEqual(t, "2 hours", HumanDuration(1*time.Hour+59*time.Minute))
+	assertEqual(t, "3 hours", HumanDuration(3*time.Hour))
+	assertEqual(t, "3 hours", HumanDuration(3*time.Hour+29*time.Minute))
+	assertEqual(t, "4 hours", HumanDuration(3*time.Hour+31*time.Minute))
+	assertEqual(t, "4 hours", HumanDuration(3*time.Hour+59*time.Minute))
+	assertEqual(t, "4 hours", HumanDuration(3*time.Hour+60*time.Minute))
+	assertEqual(t, "24 hours", HumanDuration(24*time.Hour))
+	assertEqual(t, "36 hours", HumanDuration(1*day+12*time.Hour))
+	assertEqual(t, "2 days", HumanDuration(2*day))
+	assertEqual(t, "7 days", HumanDuration(7*day))
+	assertEqual(t, "13 days", HumanDuration(13*day+5*time.Hour))
+	assertEqual(t, "2 weeks", HumanDuration(2*week))
+	assertEqual(t, "2 weeks", HumanDuration(2*week+4*day))
+	assertEqual(t, "3 weeks", HumanDuration(3*week))
+	assertEqual(t, "4 weeks", HumanDuration(4*week))
+	assertEqual(t, "4 weeks", HumanDuration(4*week+3*day))
+	assertEqual(t, "4 weeks", HumanDuration(1*month))
+	assertEqual(t, "6 weeks", HumanDuration(1*month+2*week))
+	assertEqual(t, "2 months", HumanDuration(2*month))
+	assertEqual(t, "2 months", HumanDuration(2*month+2*week))
+	assertEqual(t, "3 months", HumanDuration(3*month))
+	assertEqual(t, "3 months", HumanDuration(3*month+1*week))
+	assertEqual(t, "5 months", HumanDuration(5*month+2*week))
+	assertEqual(t, "13 months", HumanDuration(13*month))
+	assertEqual(t, "23 months", HumanDuration(23*month))
+	assertEqual(t, "24 months", HumanDuration(24*month))
+	assertEqual(t, "2 years", HumanDuration(24*month+2*week))
+	assertEqual(t, "3 years", HumanDuration(3*year+2*month))
+}
+
 func TestHumanTime(t *testing.T) {
 	now := time.Now()

 	t.Run("zero value", func(t *testing.T) {
 		assertEqual(t, HumanTime(time.Time{}, "never"), "never")
 	})
-
 	t.Run("time in the future", func(t *testing.T) {
 		v := now.Add(48 * time.Hour)
 		assertEqual(t, HumanTime(v, ""), "2 days from now")
 	})
-
 	t.Run("time in the past", func(t *testing.T) {
 		v := now.Add(-48 * time.Hour)
 		assertEqual(t, HumanTime(v, ""), "2 days ago")
 	})
-
-	t.Run("soon", func(t *testing.T) {
-		v := now.Add(800 * time.Millisecond)
-		assertEqual(t, HumanTime(v, ""), "Less than a second from now")
-	})
+}
+
+func TestExactDuration(t *testing.T) {
+	assertEqual(t, "1 millisecond", ExactDuration(1*time.Millisecond))
+	assertEqual(t, "10 milliseconds", ExactDuration(10*time.Millisecond))
+	assertEqual(t, "1 second", ExactDuration(1*time.Second))
+	assertEqual(t, "10 seconds", ExactDuration(10*time.Second))
+	assertEqual(t, "1 minute", ExactDuration(1*time.Minute))
+	assertEqual(t, "10 minutes", ExactDuration(10*time.Minute))
+	assertEqual(t, "1 hour", ExactDuration(1*time.Hour))
+	assertEqual(t, "10 hours", ExactDuration(10*time.Hour))
+	assertEqual(t, "1 hour 1 second", ExactDuration(1*time.Hour+1*time.Second))
+	assertEqual(t, "1 hour 10 seconds", ExactDuration(1*time.Hour+10*time.Second))
+	assertEqual(t, "1 hour 1 minute", ExactDuration(1*time.Hour+1*time.Minute))
+	assertEqual(t, "1 hour 10 minutes", ExactDuration(1*time.Hour+10*time.Minute))
+	assertEqual(t, "1 hour 1 minute 1 second", ExactDuration(1*time.Hour+1*time.Minute+1*time.Second))
+	assertEqual(t, "10 hours 10 minutes 10 seconds", ExactDuration(10*time.Hour+10*time.Minute+10*time.Second))
 }
--- a/go.mod
+++ b/go.mod
@@ -8,9 +8,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.14
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pdevine/readline v1.5.2
 	github.com/spf13/cobra v1.7.0
-	golang.org/x/sync v0.3.0
 )

 require github.com/rivo/uniseg v0.2.0 // indirect
@@ -18,6 +16,7 @@ require github.com/rivo/uniseg v0.2.0 // indirect
 require (
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/chzyer/readline v1.5.1
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -39,12 +38,12 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.10.0
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
+	golang.org/x/crypto v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/term v0.10.0
 	golang.org/x/text v0.10.0 // indirect
+	gonum.org/v1/gonum v0.13.0
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhD
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM=
 github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
 github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
 github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -78,8 +80,6 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/pdevine/readline v1.5.2 h1:oz6Y5GdTmhPG+08hhxcAvtHitSANWuA2100Sppb38xI=
-github.com/pdevine/readline v1.5.2/go.mod h1:na/LbuE5PYwxI7GyopWdIs3U8HVe89lYlNTFTXH3wOw=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
@@ -120,13 +120,9 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
 golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
-golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
-golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -134,8 +130,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
@@ -145,6 +141,8 @@ golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
 golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM=
+gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
--- a/llm/falcon.go
+++ b/llm/falcon.go
@@ -1,20 +0,0 @@
-package llm
-
-const (
-	falconModelType7B   = 32
-	falconModelType40B  = 60
-	falconModelType180B = 80
-)
-
-func falconModelType(numLayer uint32) string {
-	switch numLayer {
-	case 32:
-		return "7B"
-	case 60:
-		return "40B"
-	case 80:
-		return "180B"
-	default:
-		return "unknown"
-	}
-}
--- a/runner/ggml-alloc.c
+++ b/runner/ggml-alloc.c
@@ -1,5 +1,5 @@
 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
@@ -25,7 +25,6 @@
 */

 #include "ggml-alloc.h"
-#include "ggml-backend.h"
 #include "ggml.h"
 #include <assert.h>
 #include <stdarg.h>
@@ -33,10 +32,8 @@
 #include <stdlib.h>
 #include <string.h>

-
 #define UNUSED(x) (void)(x)
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
-#define GGML_MAX_CONCUR (2*GGML_MAX_NODES)

 //#define GGML_ALLOCATOR_DEBUG

@@ -85,20 +82,17 @@ struct free_block {
    size_t size;
 };

-#define MAX_FREE_BLOCKS 256
+#define MAX_FREE_BLOCKS 128

 struct ggml_allocr {
-    struct ggml_backend_buffer * buffer;
-    bool buffer_owned;
    void * data;
+    size_t size;
    size_t alignment;
    int n_free_blocks;
    struct free_block free_blocks[MAX_FREE_BLOCKS];
    struct hash_node hash_table[GGML_GRAPH_HASHTABLE_SIZE];
    size_t max_size;
    bool measure;
-    int parse_seq[GGML_MAX_CONCUR];
-    int parse_seq_len;

 #ifdef GGML_ALLOCATOR_DEBUG
    struct ggml_tensor * allocated_tensors[1024];
@@ -106,7 +100,7 @@ struct ggml_allocr {
 };

 #ifdef GGML_ALLOCATOR_DEBUG
-static void add_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
+static void add_allocated_tensor(struct ggml_allocator * alloc, struct ggml_tensor * tensor) {
    for (int i = 0; i < 1024; i++) {
        if (alloc->allocated_tensors[i] == NULL) {
            alloc->allocated_tensors[i] = tensor;
@@ -115,7 +109,7 @@ static void add_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor
    }
    GGML_ASSERT(!"out of allocated_tensors");
 }
-static void remove_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
+static void remove_allocated_tensor(struct ggml_allocator * alloc, struct ggml_tensor * tensor) {
    for (int i = 0; i < 1024; i++) {
        if (alloc->allocated_tensors[i] == tensor ||
            (alloc->allocated_tensors[i] != NULL && alloc->allocated_tensors[i]->data == tensor->data)) {
@@ -128,30 +122,25 @@ static void remove_allocated_tensor(struct ggml_allocr * alloc, struct ggml_tens
 }
 #endif

-// check if a tensor is allocated by this buffer
-static bool ggml_allocr_is_own(struct ggml_allocr * alloc, const struct ggml_tensor * tensor) {
-    return tensor->buffer == alloc->buffer;
-}

-static bool ggml_is_view(struct ggml_tensor * t) {
-    return t->view_src != NULL;
+static size_t ggml_allocator_get_alloc_size(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
+    return ggml_nbytes(tensor);
+
+    UNUSED(alloc);
 }

 void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
-    GGML_ASSERT(!ggml_is_view(tensor)); // views generally get data pointer from one of their sources
-    GGML_ASSERT(tensor->data == NULL); // avoid allocating tensor which already has memory allocated
-
-    size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor);
+    size_t size = ggml_allocator_get_alloc_size(alloc, tensor);
    size = aligned_offset(NULL, size, alloc->alignment);

    AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);

    size_t max_avail = 0;

-    // find the best fitting free block besides the last block
+    // find the best fitting free block
    int best_fit_block = -1;
    size_t best_fit_size = SIZE_MAX;
-    for (int i = 0; i < alloc->n_free_blocks - 1; i++) {
+    for (int i = 0; i < alloc->n_free_blocks; i++) {
        struct free_block * block = &alloc->free_blocks[i];
        max_avail = MAX(max_avail, block->size);
        if (block->size >= size && block->size <= best_fit_size) {
@@ -163,17 +152,10 @@ void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor)
    AT_PRINTF("block %d\n", best_fit_block);

    if (best_fit_block == -1) {
-        // the last block is our last resort
-        struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1];
-        max_avail = MAX(max_avail, block->size);
-        if (block->size >= size) {
-            best_fit_block = alloc->n_free_blocks - 1;
-        } else {
-            fprintf(stderr, "%s: not enough space in the buffer (needed %zu, largest block available %zu)\n",
-                    __func__, size, max_avail);
-            GGML_ASSERT(!"not enough space in the buffer");
-            return;
-        }
+        fprintf(stderr, "%s: not enough space in the buffer (needed %zu, largest block available %zu)\n",
+                __func__, size, max_avail);
+        GGML_ASSERT(!"not enough space in the buffer");
+        return;
    }
    struct free_block * block = &alloc->free_blocks[best_fit_block];
    void * addr = block->addr;
@@ -188,9 +170,6 @@ void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor)
    }

    tensor->data = addr;
-    AT_PRINTF("%s: allocated data at %p\n", __func__, tensor->data);
-    tensor->buffer = alloc->buffer;
-    ggml_backend_buffer_init_tensor(alloc->buffer, tensor);

 #ifdef GGML_ALLOCATOR_DEBUG
    add_allocated_tensor(alloc, tensor);
@@ -210,22 +189,19 @@ void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor)
 }

 // this is a very naive implementation, but for our case the number of free blocks should be very small
-static void ggml_allocr_free_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
-    if (ggml_allocr_is_own(alloc, tensor) == false) {
+static void ggml_allocator_free_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
+    void * ptr = tensor->data;
+
+    if (ptr < alloc->data || (char*)ptr >= (char*)alloc->data + alloc->max_size) {
        // the tensor was not allocated in this buffer
        // this can happen because the graph allocator will try to free weights and other tensors from different buffers
        // the easiest way to deal with this is just to ignore it
-        AT_PRINTF("ignoring %s (their buffer: %p, our buffer: %p)\n", tensor->name, (void *)tensor->buffer, (void *)alloc->buffer);
        return;
    }

-    void * ptr = tensor->data;
-
-    size_t size = ggml_backend_buffer_get_alloc_size(alloc->buffer, tensor);
+    size_t size = ggml_allocator_get_alloc_size(alloc, tensor);
    size = aligned_offset(NULL, size, alloc->alignment);
-    AT_PRINTF("%s: freeing %s at %p (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, ptr, size, alloc->n_free_blocks);
-
-    ggml_backend_buffer_free_tensor(alloc->buffer, tensor);
+    AT_PRINTF("%s: freeing %s (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, size, alloc->n_free_blocks);

 #ifdef GGML_ALLOCATOR_DEBUG
    remove_allocated_tensor(alloc, tensor);
@@ -279,39 +255,27 @@ static void ggml_allocr_free_tensor(struct ggml_allocr * alloc, struct ggml_tens
    alloc->n_free_blocks++;
 }

-void ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, const int * list, int n) {
-    for (int i = 0; i < n; i++) {
-        alloc->parse_seq[i] = list[i];
-    }
-    alloc->parse_seq_len = n;
-}
-
 void ggml_allocr_reset(struct ggml_allocr * alloc) {
    alloc->n_free_blocks = 1;
    size_t align_offset = aligned_offset(alloc->data, 0, alloc->alignment);
    alloc->free_blocks[0].addr = (char *)alloc->data + align_offset;
-    alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset;
+    alloc->free_blocks[0].size = alloc->size - align_offset;
 }

 struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment) {
-    struct ggml_backend_buffer * buffer = ggml_backend_cpu_buffer_from_ptr(NULL, data, size);
-
-    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr));
+    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr) /* + n_free_blocks * sizeof(struct free_block) */);

    *alloc = (struct ggml_allocr){
-        /*.buffer        = */ buffer,
-        /*.buffer_owned  = */ true,
-        /*.base          = */ ggml_backend_buffer_get_base(buffer),
+        /*.data          = */ data,
+        /*.size          = */ size,
        /*.alignment     = */ alignment,
        /*.n_free_blocks = */ 0,
        /*.free_blocks   = */ {{0}},
        /*.hash_table    = */ {{0}},
        /*.max_size      = */ 0,
        /*.measure       = */ false,
-        /*.parse_seq     = */ {0},
-        /*.parse_seq_len = */ 0,
 #ifdef GGML_ALLOCATOR_DEBUG
-        /*.allocated_tensors = */ {0},
+        /*.allocated_tensors = */ = {0},
 #endif
    };

@@ -320,30 +284,25 @@ struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment)
    return alloc;
 }

+// address and size of the buffer when measuring
+// it needs to be large enough to fit all the tensors, but it cannot overlap with other existing buffers
+static void * const MEASURE_BASE_ADDR = (void *) 0x1000;
+static const size_t MEASURE_MAX_SIZE  = 1ULL<<40; // 1 TB
+
 struct ggml_allocr * ggml_allocr_new_measure(size_t alignment) {
-    struct ggml_allocr * alloc = ggml_allocr_new((void *)0x1000, (size_t)-0x1001, alignment);
-    alloc->measure = true;
-
-    return alloc;
-}
-
-struct ggml_allocr * ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer) {
-    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr));
+    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr) /* + n_free_blocks * sizeof(struct free_block) */);

    *alloc = (struct ggml_allocr){
-        /*.buffer        = */ buffer,
-        /*.buffer_owned  = */ false,
-        /*.base          = */ ggml_backend_buffer_get_base(buffer),
-        /*.alignment     = */ ggml_backend_buffer_get_alignment(buffer),
+        /*.data          = */ MEASURE_BASE_ADDR,
+        /*.size          = */ MEASURE_MAX_SIZE,
+        /*.alignment     = */ alignment,
        /*.n_free_blocks = */ 0,
        /*.free_blocks   = */ {{0}},
        /*.hash_table    = */ {{0}},
        /*.max_size      = */ 0,
-        /*.measure       = */ false,
-        /*.parse_seq     = */ {0},
-        /*.parse_seq_len = */ 0,
+        /*.measure       = */ true,
 #ifdef GGML_ALLOCATOR_DEBUG
-        /*.allocated_tensors = */ {0},
+        /*.allocated_tensors = */ = {0},
 #endif
    };

@@ -353,9 +312,6 @@ struct ggml_allocr * ggml_allocr_new_from_buffer(struct ggml_backend_buffer * bu
 }

 void ggml_allocr_free(struct ggml_allocr * alloc) {
-    if (alloc->buffer_owned) {
-        ggml_backend_buffer_free(alloc->buffer);
-    }
    free(alloc);
 }

@@ -365,6 +321,11 @@ bool ggml_allocr_is_measure(struct ggml_allocr * alloc) {

 //////////// compute graph allocator

+static bool ggml_is_view(struct ggml_tensor * t) {
+    return t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW || t->op == GGML_OP_TRANSPOSE ||
+           t->op == GGML_OP_PERMUTE || t->op == GGML_OP_CPY;
+}
+
 static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
    if (a->type != b->type) {
        return false;
@@ -380,6 +341,28 @@ static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml
    return true;
 }

+static struct ggml_tensor * get_view_parent(struct ggml_tensor * t) {
+    switch (t->op) {
+        case GGML_OP_PERMUTE:
+        case GGML_OP_RESHAPE:
+        case GGML_OP_TRANSPOSE:
+        case GGML_OP_VIEW:
+            return t->src[0];
+        case GGML_OP_CPY:
+            return t->src[1];
+        default:
+            return NULL;
+    }
+}
+
+static struct ggml_tensor * get_view_source(struct ggml_tensor * t) {
+    struct ggml_tensor * parent = t;
+    do {
+        parent = get_view_parent(parent);
+    } while (ggml_is_view(parent));
+    return parent;
+}
+
 static bool ggml_op_can_inplace(enum ggml_op op) {
    switch (op) {
        case GGML_OP_SCALE:
@@ -387,6 +370,7 @@ static bool ggml_op_can_inplace(enum ggml_op op) {
        case GGML_OP_DIAG_MASK_INF:
        case GGML_OP_ADD:
        case GGML_OP_ADD1:
+        case GGML_OP_ACC:
        case GGML_OP_SUB:
        case GGML_OP_MUL:
        case GGML_OP_DIV:
@@ -396,7 +380,9 @@ static bool ggml_op_can_inplace(enum ggml_op op) {
        case GGML_OP_UNARY:
        case GGML_OP_ROPE:
        case GGML_OP_RMS_NORM:
+        case GGML_OP_SET:
        case GGML_OP_SOFT_MAX:
+        case GGML_OP_CONT:
            return true;

        default:
@@ -404,23 +390,28 @@ static bool ggml_op_can_inplace(enum ggml_op op) {
    }
 }

-static void init_view(struct ggml_allocr * alloc, struct ggml_tensor * view) {
-    assert(view->view_src != NULL && view->view_src->data != NULL);
-    view->backend = view->view_src->backend;
-    view->buffer  = view->view_src->buffer;
-    view->data    = (char *)view->view_src->data + view->view_offs;
-
-    // FIXME: the view should be initialized by the owning buffer, but currently this breaks the CUDA backend
-    // due to the ggml_tensor_extra_gpu ring buffer overwriting the KV cache extras
-    assert(ggml_allocr_is_measure(alloc) || !view->buffer || view->buffer->backend == alloc->buffer->backend);
-    ggml_backend_buffer_init_tensor(alloc->buffer, view);
-}
-
 static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node) {
    struct hash_node * ht = alloc->hash_table;
    if (node->data == NULL) {
        if (ggml_is_view(node)) {
-            init_view(alloc, node);
+            size_t offset;
+            switch(node->op) {
+                case GGML_OP_VIEW:
+                    memcpy(&offset, node->op_params, sizeof(size_t));
+                    node->data = (char *) node->src[0]->data + offset;
+                    break;
+                case GGML_OP_PERMUTE:
+                case GGML_OP_RESHAPE:
+                case GGML_OP_TRANSPOSE:
+                    node->data = node->src[0]->data;
+                    break;
+                case GGML_OP_CPY:
+                    node->data = node->src[1]->data;
+                    break;
+                default:
+                    GGML_ASSERT(!"unknown view op");
+                    break;
+            }
        } else {
            // see if we can reuse a parent's buffer (inplace)
            if (ggml_op_can_inplace(node->op)) {
@@ -431,7 +422,8 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)
                    }

                    // if the node's data is external, then we cannot re-use it
-                    if (ggml_allocr_is_own(alloc, parent) == false) {
+                    if ((char *) parent->data < (char *) alloc->data ||
+                        (char *) parent->data >= ((char *) alloc->data + alloc->size)) {
                        AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);
                        continue;
                    }
@@ -439,7 +431,7 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)
                    struct hash_node * p_hn = hash_get(ht, parent);
                    if (parent->data != NULL && p_hn->n_children == 1 && p_hn->n_views == 0 && ggml_are_same_layout(node, parent)) {
                        if (ggml_is_view(parent)) {
-                            struct ggml_tensor * view_src = parent->view_src;
+                            struct ggml_tensor * view_src = get_view_source(parent);
                            struct hash_node * view_src_hn = hash_get(ht, view_src);
                            if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {
                                // TODO: the offset of the view parent must be kept to ensure that the op doesn't overwrite
@@ -448,19 +440,15 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)
                                // adding a view_src pointer to the tensor would solve this and simplify the code dealing with views
                                // for now, we only reuse the parent's data if the offset is zero (view_src->data == parent->data)
                                AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);
-                                node->view_src = view_src;
-                                view_src_hn->n_views += 1;
-                                init_view(alloc, node);
+                                node->data = parent->data;
                                return;
                            }
                        }
                        else {
                            AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);
-                            node->view_src = parent;
-                            p_hn->n_views += 1;
-                            init_view(alloc, node);
-                            return;
+                            node->data = parent->data;
                        }
+                        return;
                    }
                }
            }
@@ -469,7 +457,7 @@ static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node)
    }
 }

-size_t ggml_allocr_alloc_graph_n(
+static size_t ggml_allocator_alloc_graph_tensors_n(
    struct ggml_allocr * alloc,
    struct ggml_cgraph ** graphs, int n_graphs,
    struct ggml_tensor *** inputs, struct ggml_tensor *** outputs) {
@@ -485,12 +473,8 @@ size_t ggml_allocr_alloc_graph_n(
            struct ggml_tensor * node = gf->nodes[i];

            if (ggml_is_view(node)) {
-                struct ggml_tensor * view_src = node->view_src;
+                struct ggml_tensor * view_src = get_view_source(node);
                hash_get(ht, view_src)->n_views += 1;
-                if (node->buffer == NULL && node->data != NULL) {
-                    // view of a pre-allocated tensor, didn't call init_view() yet
-                    init_view(alloc, node);
-                }
            }

            for (int j = 0; j < GGML_MAX_SRC; j++) {
@@ -499,9 +483,6 @@ size_t ggml_allocr_alloc_graph_n(
                    break;
                }
                hash_get(ht, parent)->n_children += 1;
-                if (ggml_is_view(parent) && parent->buffer == NULL && parent->data != NULL) {
-                    init_view(alloc, parent);
-                }
            }
        }
    }
@@ -518,92 +499,70 @@ size_t ggml_allocr_alloc_graph_n(
                allocate_node(alloc, input);
            }
        }
-        // if we have parse_seq then we allocate nodes following the list, and we only free nodes at barriers
-        int last_barrier_pos = 0;
-        int n_nodes = alloc->parse_seq_len ? alloc->parse_seq_len : gf->n_nodes;
+        for (int i = 0; i < gf->n_nodes; i++) {
+            struct ggml_tensor * node = gf->nodes[i];

-        for (int ind = 0; ind < n_nodes; ind++) {
-            // allocate a node if there is no parse_seq or this is not a barrier
-            if ((alloc->parse_seq_len==0) || alloc->parse_seq[ind] != -1) {
-                int i = alloc->parse_seq_len ? alloc->parse_seq[ind] : ind;
-                struct ggml_tensor * node = gf->nodes[i];
-
-                // allocate parents (leafs)
-                for (int j = 0; j < GGML_MAX_SRC; j++) {
-                    struct ggml_tensor * parent = node->src[j];
-                    if (parent == NULL) {
-                        break;
-                    }
-                    allocate_node(alloc, parent);
+            // allocate parents (leafs)
+            for (int j = 0; j < GGML_MAX_SRC; j++) {
+                struct ggml_tensor * parent = node->src[j];
+                if (parent == NULL) {
+                    break;
                }
-
-                // allocate node
-                allocate_node(alloc, node);
-
-                AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);
-                for (int j = 0; j < GGML_MAX_SRC; j++) {
-                    struct ggml_tensor * parent = node->src[j];
-                    if (parent == NULL) {
-                        break;
-                    }
-                    AT_PRINTF("%s", parent->name);
-                    if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {
-                        AT_PRINTF(", ");
-                    }
-                }
-                AT_PRINTF("\n");
+                allocate_node(alloc, parent);
            }

+            // allocate node
+            allocate_node(alloc, node);
+
+            AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);
+            for (int j = 0; j < GGML_MAX_SRC; j++) {
+                struct ggml_tensor * parent = node->src[j];
+                if (parent == NULL) {
+                    break;
+                }
+                AT_PRINTF("%s", parent->name);
+                if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {
+                    AT_PRINTF(", ");
+                }
+            }
+            AT_PRINTF("\n");
+
            // update parents
-            // update immediately if there is no parse_seq
-            // update only at barriers if there is parse_seq
-            if ((alloc->parse_seq_len == 0) || alloc->parse_seq[ind] == -1) {
-                int update_start = alloc->parse_seq_len ? last_barrier_pos : ind;
-                int update_end   = alloc->parse_seq_len ? ind              : ind + 1;
-                for (int i = update_start; i < update_end; i++) {
-                    int node_i = alloc->parse_seq_len ? alloc->parse_seq[i] : i;
-                    struct ggml_tensor * node = gf->nodes[node_i];
+            for (int j = 0; j < GGML_MAX_SRC; j++) {
+                struct ggml_tensor * parent = node->src[j];
+                if (parent == NULL) {
+                    break;
+                }
+                struct hash_node * p_hn = hash_get(ht, parent);
+                p_hn->n_children -= 1;

-                    for (int j = 0; j < GGML_MAX_SRC; j++) {
-                        struct ggml_tensor * parent = node->src[j];
-                        if (parent == NULL) {
-                            break;
+                //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);
+
+                if (p_hn->n_children == 0 && p_hn->n_views == 0) {
+                    if (ggml_is_view(parent)) {
+                        struct ggml_tensor * view_src = get_view_source(parent);
+                        struct hash_node * view_src_hn = hash_get(ht, view_src);
+                        view_src_hn->n_views -= 1;
+                        AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src->n_children, view_src->n_views);
+                        if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src->data != node->data) {
+                            ggml_allocator_free_tensor(alloc, view_src);
                        }
-                        struct hash_node * p_hn = hash_get(ht, parent);
-                        p_hn->n_children -= 1;
-
-                        //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);
-
-                        if (p_hn->n_children == 0 && p_hn->n_views == 0) {
-                            if (ggml_is_view(parent)) {
-                                struct ggml_tensor * view_src = parent->view_src;
-                                struct hash_node * view_src_hn = hash_get(ht, view_src);
-                                view_src_hn->n_views -= 1;
-                                AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src_hn->n_children, view_src_hn->n_views);
-                                if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src->data != node->data) {
-                                    ggml_allocr_free_tensor(alloc, view_src);
-                                }
-                            }
-                            else {
-                                if (parent->data != node->data) {
-                                    ggml_allocr_free_tensor(alloc, parent);
-                                }
-                            }
+                    }
+                    else {
+                        if (parent->data != node->data) {
+                            ggml_allocator_free_tensor(alloc, parent);
                        }
                    }
                }
-                AT_PRINTF("\n");
-                if (alloc->parse_seq_len) {
-                    last_barrier_pos = ind + 1;
-                }
            }
+            AT_PRINTF("\n");
        }
        // free graph outputs here that wouldn't be freed otherwise because they have no children
        if (outputs != NULL && outputs[g] != NULL) {
            for (int i = 0; outputs[g][i] != NULL; i++) {
                struct ggml_tensor * output = outputs[g][i];
                AT_PRINTF("output: %s\n", output->name);
-                ggml_allocr_free_tensor(alloc, output);
+                ggml_allocator_free_tensor(alloc, output);
            }
        }
    }
@@ -612,9 +571,5 @@ size_t ggml_allocr_alloc_graph_n(
 }

 size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph) {
-    return ggml_allocr_alloc_graph_n(alloc, &graph, 1, NULL, NULL);
-}
-
-size_t ggml_allocr_max_size(struct ggml_allocr * alloc) {
-    return alloc->max_size;
+    return ggml_allocator_alloc_graph_tensors_n(alloc, &graph, 1, NULL, NULL);
 }
--- a/runner/ggml-alloc.h
+++ b/runner/ggml-alloc.h
@@ -1,5 +1,5 @@
 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
@@ -32,27 +32,16 @@
 extern "C" {
 #endif

-struct ggml_backend_buffer;

 GGML_API struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment);
 GGML_API struct ggml_allocr * ggml_allocr_new_measure(size_t alignment);
-GGML_API struct ggml_allocr * ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer);

-// tell the allocator to parse nodes following the order described in the list
-// you should call this if your graph are optimized to execute out-of-order
-GGML_API void   ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, const int * list, int n);
-
-GGML_API void   ggml_allocr_free       (struct ggml_allocr * alloc);
-GGML_API bool   ggml_allocr_is_measure (struct ggml_allocr * alloc);
-GGML_API void   ggml_allocr_reset      (struct ggml_allocr * alloc);
-GGML_API void   ggml_allocr_alloc      (struct ggml_allocr * alloc, struct ggml_tensor * tensor);
+GGML_API void   ggml_allocr_free(struct ggml_allocr * alloc);
+GGML_API bool   ggml_allocr_is_measure(struct ggml_allocr * alloc);
+GGML_API void   ggml_allocr_reset(struct ggml_allocr * alloc);
+GGML_API void   ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor);
 GGML_API size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph);
-GGML_API size_t ggml_allocr_max_size   (struct ggml_allocr * alloc);

-GGML_API size_t ggml_allocr_alloc_graph_n(
-                    struct ggml_allocr * alloc,
-                    struct ggml_cgraph ** graphs, int n_graphs,
-                    struct ggml_tensor *** inputs, struct ggml_tensor *** outputs);

 #ifdef  __cplusplus
 }
--- a/runner/ggml-cuda.cu
+++ b/runner/ggml-cuda.cu
--- a/llm/ggml-cuda.h
+++ b/llm/ggml-cuda.h
@@ -0,0 +1,63 @@
+/**
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
+ *
+ * MIT License
+ *
+ * Copyright (c) 2023 Georgi Gerganov
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#pragma once
+
+#include "ggml.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define GGML_CUDA_MAX_DEVICES       16
+
+void   ggml_init_cublas(void);
+void   ggml_cuda_set_tensor_split(const float * tensor_split);
+
+void   ggml_cuda_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+bool   ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+size_t ggml_cuda_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+void   ggml_cuda_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);
+
+// TODO: export these with GGML_API
+void * ggml_cuda_host_malloc(size_t size);
+void   ggml_cuda_host_free(void * ptr);
+
+void   ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor);
+
+void   ggml_cuda_free_data(struct ggml_tensor * tensor);
+void   ggml_cuda_assign_buffers(struct ggml_tensor * tensor);
+void   ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor);
+void   ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor);
+void   ggml_cuda_set_main_device(int main_device);
+void   ggml_cuda_set_mul_mat_q(bool mul_mat_q);
+void   ggml_cuda_set_scratch_size(size_t scratch_size);
+void   ggml_cuda_free_scratch(void);
+bool   ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);
+
+#ifdef  __cplusplus
+}
+#endif
--- a/runner/ggml-metal.h
+++ b/runner/ggml-metal.h
@@ -1,7 +1,7 @@
 //go:build darwin

 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
@@ -47,15 +47,11 @@

 #pragma once

-#include "ggml.h"
-#include "ggml-backend.h"
-
 #include <stddef.h>
 #include <stdbool.h>

 // max memory buffers that can be mapped to the device
 #define GGML_METAL_MAX_BUFFERS 16
-#define GGML_METAL_MAX_COMMAND_BUFFERS 32

 struct ggml_tensor;
 struct ggml_cgraph;
@@ -64,22 +60,12 @@ struct ggml_cgraph;
 extern "C" {
 #endif

-//
-// internal API
-// temporary exposed to user-code
-//
-
 struct ggml_metal_context;

-void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);
-
 // number of command buffers to use
 struct ggml_metal_context * ggml_metal_init(int n_cb);
 void ggml_metal_free(struct ggml_metal_context * ctx);

-void * ggml_metal_host_malloc(size_t n);
-void   ggml_metal_host_free  (void * data);
-
 // set the number of command buffers to use
 void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb);

@@ -105,29 +91,15 @@ void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor *

 // try to find operations that can be run concurrently in the graph
 // you should run it again if the topology of your graph changes
-void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf, bool check_mem);
+void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);

-// if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized
-int ggml_metal_if_optimized(struct ggml_metal_context * ctx);
-
-// output the concur_list for ggml_alloc
-int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx);
+// if the graph has been optimized for concurrently dispatch
+bool ggml_metal_if_optimized(struct ggml_metal_context * ctx);

 // same as ggml_graph_compute but uses Metal
 // creates gf->n_threads command buffers in parallel
 void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);

-//
-// backend API
-// user-code should use only these functions
-//
-
-GGML_API ggml_backend_t ggml_backend_metal_init(void);
-
-GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);
-
-GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
-
 #ifdef __cplusplus
 }
 #endif
--- a/runner/ggml-metal.m
+++ b/runner/ggml-metal.m
--- a/llm/ggml-metal.metal
+++ b/llm/ggml-metal.metal
--- a/runner/ggml-mpi.c
+++ b/runner/ggml-mpi.c
@@ -1,7 +1,7 @@
 //go:build mpi

 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
--- a/runner/ggml-mpi.h
+++ b/runner/ggml-mpi.h
@@ -1,7 +1,7 @@
 //go:build mpi

 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
--- a/runner/ggml-opencl.cpp
+++ b/runner/ggml-opencl.cpp
@@ -1,7 +1,7 @@
 //go:build opencl

 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
@@ -47,7 +47,7 @@
 #pragma warning(disable: 4244 4267) // possible loss of data
 #endif

-#define CL_DMMV_LOCAL_SIZE 32
+#define CL_DMMV_BLOCK_SIZE 32

 #ifndef K_QUANTS_PER_ITERATION
 #define K_QUANTS_PER_ITERATION 1
@@ -230,14 +230,14 @@ inline void get_scale_min_k4(int j, const __global uint8_t *q, uint8_t *d, uint8

 __kernel void dequantize_block_q2_K(__global const struct block_q2_K *x, __global float *yy)
 {
-    const int i = get_group_id(0) + get_global_offset(0);
+    const int i = get_group_id(0);
    const int tid = get_local_id(0);
    const int n = tid / 32;
    const int l = tid - 32 * n;
    const int is = 8 * n + l / 16;

    const uint8_t q = x[i].qs[32 * n + l];
-    __global float *y = yy + get_group_id(0) * QK_K + 128 * n;
+    __global float *y = yy + i * QK_K + 128 * n;

    const float dall = vload_half(0, &x[i].d);
    const float dmin = vload_half(0, &x[i].dmin);
@@ -251,7 +251,7 @@ __kernel void dequantize_block_q2_K(__global const struct block_q2_K *x, __globa
 __kernel void dequantize_block_q3_K(__global const struct block_q3_K *x, __global float *yy)
 {
    int r = get_local_id(0) / 4;
-    int i = get_group_id(0) + get_global_offset(0);
+    int i = get_group_id(0);
    int tid = r / 2;
    int is0 = r % 2;
    int l0 = 16 * is0 + 4 * (get_local_id(0) % 4);
@@ -269,7 +269,7 @@ __kernel void dequantize_block_q3_K(__global const struct block_q3_K *x, __globa
    float d_all = vload_half(0, &x[i].d);
    float dl = d_all * (us - 32);

-    __global float *y = yy + get_group_id(0) * QK_K + 128 * n + 32 * j;
+    __global float *y = yy + i * QK_K + 128 * n + 32 * j;
    const __global uint8_t *q = x[i].qs + 32 * n;
    const __global uint8_t *hm = x[i].hmask;

@@ -279,14 +279,14 @@ __kernel void dequantize_block_q3_K(__global const struct block_q3_K *x, __globa

 __kernel void dequantize_block_q4_K(__global const struct block_q4_K *x, __global float *yy)
 {
-    const int i = get_group_id(0) + get_global_offset(0);
+    const int i = get_group_id(0);
    const int tid = get_local_id(0);
    const int il = tid / 8;
    const int ir = tid % 8;
    const int is = 2 * il;
    const int n = 4;

-    __global float *y = yy + get_group_id(0) * QK_K + 64 * il + n * ir;
+    __global float *y = yy + i * QK_K + 64 * il + n * ir;

    const float dall = vload_half(0, &x[i].d);
    const float dmin = vload_half(0, &x[i].dmin);
@@ -309,13 +309,13 @@ __kernel void dequantize_block_q4_K(__global const struct block_q4_K *x, __globa

 __kernel void dequantize_block_q5_K(__global const struct block_q5_K *x, __global float *yy)
 {
-    const int i = get_group_id(0) + get_global_offset(0);
+    const int i = get_group_id(0);
    const int tid = get_local_id(0);
    const int il = tid / 16;
    const int ir = tid % 16;
    const int is = 2 * il;

-    __global float *y = yy + get_group_id(0) * QK_K + 64 * il + 2 * ir;
+    __global float *y = yy + i * QK_K + 64 * il + 2 * ir;

    const float dall = vload_half(0, &x[i].d);
    const float dmin = vload_half(0, &x[i].dmin);
@@ -341,13 +341,13 @@ __kernel void dequantize_block_q5_K(__global const struct block_q5_K *x, __globa

 __kernel void dequantize_block_q6_K(__global const struct block_q6_K *x, __global float *yy)
 {
-    const int i = get_group_id(0) + get_global_offset(0);
+    const int i = get_group_id(0);
    const int tid = get_local_id(0);
    const int ip = tid / 32;
    const int il = tid - 32 * ip;
    const int is = 8 * ip + il / 16;

-    __global float *y = yy + get_group_id(0) * QK_K + 128 * ip + il;
+    __global float *y = yy + i * QK_K + 128 * ip + il;

    const float d = vload_half(0, &x[i].d);

@@ -366,7 +366,7 @@ __kernel void dequantize_mul_mat_vec_q2_K(__global const struct block_q2_K * xx,
    const int row = get_group_id(0);

    const int num_blocks_per_row = ncols / QK_K;
-    const int ib0 = row*num_blocks_per_row + get_global_offset(0);
+    const int ib0 = row*num_blocks_per_row;

    __global const struct block_q2_K * x = xx + ib0;

@@ -441,7 +441,7 @@ __kernel void dequantize_mul_mat_vec_q3_K(__global const struct block_q3_K * xx,
    const int row = get_group_id(0);

    const int num_blocks_per_row = ncols / QK_K;
-    const int ib0 = row*num_blocks_per_row + get_global_offset(0);
+    const int ib0 = row*num_blocks_per_row;

    __global const struct block_q3_K * x = xx + ib0;

@@ -517,7 +517,7 @@ __kernel void dequantize_mul_mat_vec_q4_K(__global const struct block_q4_K * xx,

    const int row = get_group_id(0);
    const int num_blocks_per_row = ncols / QK_K;
-    const int ib0 = row*num_blocks_per_row + get_global_offset(0);
+    const int ib0 = row*num_blocks_per_row;

    const int tid = get_local_id(0)/K_QUANTS_PER_ITERATION;  // 0...15
    const int ix  = get_local_id(0)%K_QUANTS_PER_ITERATION;
@@ -590,7 +590,7 @@ __kernel void dequantize_mul_mat_vec_q5_K(__global const struct block_q5_K * xx,

    const int row = get_group_id(0);
    const int num_blocks_per_row = ncols / QK_K;
-    const int ib0 = row*num_blocks_per_row + get_global_offset(0);
+    const int ib0 = row*num_blocks_per_row;

    const int tid = get_local_id(0)/2;  // 0...15
    const int ix  = get_local_id(0)%2;
@@ -669,7 +669,7 @@ __kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx,
    const int row = get_group_id(0);

    const int num_blocks_per_row = ncols / QK_K;
-    const int ib0 = row*num_blocks_per_row + get_global_offset(0);
+    const int ib0 = row*num_blocks_per_row;

    __global const struct block_q6_K * x = xx + ib0;

@@ -758,7 +758,7 @@ __kernel void KERNEL_NAME(__global X_TYPE* x, __global float* y) {
    const uint qk = QUANT_K;
    const uint qr = QUANT_R;

-    const int ib = i/qk + get_global_offset(0); // block index
+    const int ib = i/qk; // block index
    const int iqs = (i%qk)/qr; // quant index
    const int iybs = i - i%qk; // y block start index
    const int y_offset = qr == 1 ? 1 : qk/2;
@@ -773,21 +773,19 @@ __kernel void KERNEL_NAME(__global X_TYPE* x, __global float* y) {

 std::string dequant_mul_mat_vec_template = MULTILINE_QUOTE(
 __kernel void KERNEL_NAME(__global X_TYPE* x, __local float* tmp, __global float* y, __global float* dst, const int ncols) {
-    const int local_size = get_local_size(0);
+    const int block_size = get_local_size(0);
    const int row = get_group_id(0);
    const int tid = get_local_id(0);

    const uint qk = QUANT_K;
    const uint qr = QUANT_R;

-    const int col_step = local_size * 2;
    const int y_offset = qr == 1 ? 1 : qk/2;

-    x += get_global_offset(0);
-
    tmp[tid] = 0;

-    for (int col = tid*2; col < ncols; col += col_step) {
+    for (int i = 0; i < ncols/block_size; i += 2) {
+        const int col = i*block_size + 2*tid;
        const int ib = (row*ncols + col)/qk; // block index
        const int iqs = (col%qk)/qr; // quant index
        const int iybs = col - col%qk; // y block start index
@@ -803,7 +801,7 @@ __kernel void KERNEL_NAME(__global X_TYPE* x, __local float* tmp, __global float

    // sum up partial sums and write back result
    barrier(CLK_LOCAL_MEM_FENCE);
-    for (int s=local_size/2; s>0; s>>=1) {
+    for (int s=block_size/2; s>0; s>>=1) {
        if (tid < s) {
            tmp[tid] += tmp[tid + s];
        }
@@ -877,7 +875,7 @@ std::array<std::string, 2> mul_str_values = {
    "mul_f32", "float"
 };

-static std::string& replace(std::string& s, const std::string& from, const std::string& to) {
+std::string& replace(std::string& s, const std::string& from, const std::string& to) {
    size_t pos = 0;
    while ((pos = s.find(from, pos)) != std::string::npos) {
         s.replace(pos, from.length(), to);
@@ -886,7 +884,7 @@ static std::string& replace(std::string& s, const std::string& from, const std::
    return s;
 }

-static std::string generate_kernels() {
+std::string generate_kernels() {
    std::stringstream src;
    src << program_source << '\n';
    src << k_quants_source << '\n';
@@ -1364,7 +1362,7 @@ void ggml_cl_free_data(const struct ggml_tensor* tensor) {
        return;
    }

-    cl_mem mem = (cl_mem)tensor->extra;
+    cl_mem mem = (cl_mem)tensor->data;
    clReleaseMemObject(mem);
 }

@@ -1379,42 +1377,30 @@ static cl_int ggml_cl_h2d_tensor_2d(cl_command_queue queue, cl_mem dst, size_t o
    const enum ggml_type type = src->type;
    const size_t ts = ggml_type_size(type);
    const size_t bs = ggml_blck_size(type);
-    const uint64_t row_size = ts*ne0/bs;

-    const char * x = (const char *) src->data + i2*nb2 + i3*nb3;
-    if (nb0 == ts && nb1 == row_size) {
-        return clEnqueueWriteBuffer(queue, dst, CL_FALSE, offset, ne1*row_size, x, 0, NULL, ev);
+    const void * x = (const void *) ((const char *) src->data + i2*nb2 + i3*nb3);
+    if (nb0 == ts && nb1 == ts*ne0/bs) {
+        err = clEnqueueWriteBuffer(queue, dst, CL_FALSE, offset, ne1*nb1, x, 0, NULL, ev);
+        return err;
    }
    if (nb0 == ts) {
        const size_t buffer_origin[3] = { offset, 0, 0 };
        const size_t host_origin[3] = { 0, 0, 0 };
-        const size_t region[3] = { row_size, ne1, 1 };
-        return clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, row_size, 0, nb1, 0, x, 0, NULL, ev);
+        const size_t region[3] = { ts*ne0/bs, ne1, 1 };
+        err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, ts*ne0/bs, 0, nb1, 0, x, 0, NULL, ev);
+        return err;
    }
-    std::vector<cl_event> events;
-    if (ev && ne1>1) events.reserve(ne1-1);
    for (uint64_t i1 = 0; i1 < ne1; i1++) {
        // pretend the row is a matrix with cols=1
-        const size_t buffer_origin[3] = { offset + i1*row_size, 0, 0 };
+        const size_t buffer_origin[3] = { offset, i1, 0 };
        const size_t host_origin[3] = { 0, 0, 0 };
-        const size_t region[3] = { ts, ne0/bs, 1 };
-        // if an event is requested, make the last write wait for all previous writes to complete
-        if (ev && i1) {
-            events.push_back(*ev);
-        }
-        cl_uint nevents = i1 == ne1-1 ? events.size() : 0U;
-        err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, ts, 0, nb0, 0, x + i1*nb1, nevents, nevents ? events.data() : nullptr, ev);
+        const size_t region[3] = { ts/bs, ne0, 1 };
+        err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, 0, 0, nb0, 0, ((const char *)x) + i1*nb0, 0, NULL, ev);
        if (err != CL_SUCCESS) {
-            for (auto event : events) {
-                clReleaseEvent(event);
-            }
-            return err;
+            break;
        }
    }
-    for (auto event : events) {
-        CL_CHECK(clReleaseEvent(event));
-    }
-    return CL_SUCCESS;
+    return err;
 }

 static void ggml_cl_mul_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@@ -1423,46 +1409,75 @@ static void ggml_cl_mul_f32(const ggml_tensor * src0, const ggml_tensor * src1,
    const int64_t ne01 = src0->ne[1];
    const int64_t ne02 = src0->ne[2];
    const int64_t ne03 = src0->ne[3];
+    const int64_t ne0 = ne00 * ne01 * ne02 * ne03;
    const int64_t ne10 = src1->ne[0];
    const int64_t ne11 = src1->ne[1];
    const int64_t ne12 = src1->ne[2];
    const int64_t ne13 = src1->ne[3];
+    const int64_t nb10 = src1->nb[0];
    const int nb2  = dst->nb[2];
    const int nb3  = dst->nb[3];
    size_t x_size;
    size_t d_size;

-    cl_mem d_X = ggml_cl_pool_malloc(ne00 * ne01 * sizeof(float), &x_size); // src0
-    cl_mem d_Y = (cl_mem) src1->extra; // src1 is already on device, broadcasted.
-    cl_mem d_D = ggml_cl_pool_malloc(ne00 * ne01 * sizeof(float), &d_size); // dst
+    cl_mem d_X = ggml_cl_pool_malloc(ne0 * sizeof(float), &x_size); // src0
+    cl_mem d_Y = (cl_mem) src1->data; // src1 is already on device, broadcasted.
+    cl_mem d_D = ggml_cl_pool_malloc(ne0 * sizeof(float), &d_size); // dst


    for (int64_t i03 = 0; i03 < ne03; i03++) {
        for (int64_t i02 = 0; i02 < ne02; i02++) {
+            const int i0 = i03*ne02 + i02;
+
            cl_event ev;

            // copy src0 to device
-            CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, &ev));
+            CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, i0, src0, i03, i02, &ev));

-            const int64_t i13 = i03%ne13;
-            const int64_t i12 = i02%ne12;
-            const int i1 = i13*ne12*ne11 + i12*ne11;
+            if (nb10 == sizeof(float)) {
+                // Contiguous, avoid overhead from queueing many kernel runs
+                const int64_t i13 = i03%ne13;
+                const int64_t i12 = i02%ne12;
+                const int i1 = i13*ne12*ne11 + i12*ne11;

-            cl_int x_offset = 0;
-            cl_int y_offset = i1*ne10;
-            cl_int d_offset = 0;
+                cl_int x_offset = 0;
+                cl_int y_offset = i1*ne10;
+                cl_int d_offset = 0;

-            size_t global = ne00 * ne01;
-            cl_int ky = ne10 * ne11;
+                size_t global = ne00 * ne01;
+                cl_int ky = ne10;
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 0, sizeof(cl_mem), &d_X));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 1, sizeof(cl_int), &x_offset));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 2, sizeof(cl_mem), &d_Y));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 3, sizeof(cl_int), &y_offset));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 4, sizeof(cl_mem), &d_D));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 5, sizeof(cl_int), &d_offset));
+                CL_CHECK(clSetKernelArg(mul_f32_cl, 6, sizeof(cl_int), &ky));
+                CL_CHECK(clEnqueueNDRangeKernel(queue, mul_f32_cl, 1, NULL, &global, NULL, 1, &ev, NULL));
+            } else {
+                for (int64_t i01 = 0; i01 < ne01; i01++) {
+                    const int64_t i13 = i03%ne13;
+                    const int64_t i12 = i02%ne12;
+                    const int64_t i11 = i01%ne11;
+                    const int i1 = i13*ne12*ne11 + i12*ne11 + i11;

-            CL_CHECK(clSetKernelArg(mul_f32_cl, 0, sizeof(cl_mem), &d_X));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 1, sizeof(cl_int), &x_offset));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 2, sizeof(cl_mem), &d_Y));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 3, sizeof(cl_int), &y_offset));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 4, sizeof(cl_mem), &d_D));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 5, sizeof(cl_int), &d_offset));
-            CL_CHECK(clSetKernelArg(mul_f32_cl, 6, sizeof(cl_int), &ky));
-            CL_CHECK(clEnqueueNDRangeKernel(queue, mul_f32_cl, 1, NULL, &global, NULL, 1, &ev, NULL));
+                    cl_int x_offset = i01*ne00;
+                    cl_int y_offset = i1*ne10;
+                    cl_int d_offset = i01*ne00;
+
+                    // compute
+                    size_t global = ne00;
+                    cl_int ky = ne10;
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 0, sizeof(cl_mem), &d_X));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 1, sizeof(cl_int), &x_offset));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 2, sizeof(cl_mem), &d_Y));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 3, sizeof(cl_int), &y_offset));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 4, sizeof(cl_mem), &d_D));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 5, sizeof(cl_int), &d_offset));
+                    CL_CHECK(clSetKernelArg(mul_f32_cl, 6, sizeof(cl_int), &ky));
+                    CL_CHECK(clEnqueueNDRangeKernel(queue, mul_f32_cl, 1, NULL, &global, NULL, 1, &ev, NULL));
+                }
+            }

            CL_CHECK(clReleaseEvent(ev));
            CL_CHECK(clFinish(queue));
@@ -1489,15 +1504,10 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr

    const int64_t ne10 = src1->ne[0];
    const int64_t ne11 = src1->ne[1];
-    const int64_t ne12 = src1->ne[2];
-    const int64_t ne13 = src1->ne[3];

    const int nb2  = dst->nb[2];
    const int nb3  = dst->nb[3];

-    const int64_t r2 = ne12 / ne02;
-    const int64_t r3 = ne13 / ne03;
-
    const float alpha = 1.0f;
    const float beta = 0.0f;
    const int x_ne = ne01 * ne00;
@@ -1509,53 +1519,42 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr
    size_t d_size;
    cl_mem d_X;
    if (src0->backend == GGML_BACKEND_GPU) { // NOLINT
-        d_X = (cl_mem) src0->extra;
+        d_X = (cl_mem) src0->data;
    } else {
-        d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size);
+        d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size);
    }
    cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size);
    cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size);

-    size_t x_offset = 0;
-
    for (int64_t i03 = 0; i03 < ne03; i03++) {
-        // TODO: copy src0 here when r3>1
-        for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {
-            for (int64_t i02 = 0; i02 < ne02; i02++) {
-                if (src0->backend == GGML_BACKEND_GPU) {
-                    x_offset = (i03 * ne02 + i02) * x_ne;
-                } else {
-                    // copy src0 to device
-                    CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
-                }
-
-                for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {
-                    // copy src1 to device
-                    CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL));
-
-                    CL_CHECK(clFinish(queue));
-
-                    // compute
-                    cl_event ev_sgemm;
-                    clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
-                                                               clblast::Transpose::kYes, clblast::Transpose::kNo,
-                                                               ne01, ne11, ne10,
-                                                               alpha,
-                                                               d_X, x_offset, ne00,
-                                                               d_Y, 0, ne10,
-                                                               beta,
-                                                               d_D, 0, ne01,
-                                                               &queue, &ev_sgemm);
-
-                    if (status != clblast::StatusCode::kSuccess) {
-                        GGML_ASSERT(false);
-                    }
-
-                    // copy dst to host
-                    float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);
-                    CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL));
-                }
+        for (int64_t i02 = 0; i02 < ne02; i02++) {
+            // copy data to device
+            if (src0->backend != GGML_BACKEND_GPU) {
+                CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
            }
+            CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, NULL));
+
+            CL_CHECK(clFinish(queue));
+
+            // compute
+            cl_event ev_sgemm;
+            clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
+                                                       clblast::Transpose::kYes, clblast::Transpose::kNo,
+                                                       ne01, ne11, ne10,
+                                                       alpha,
+                                                       d_X, 0, ne00,
+                                                       d_Y, 0, ne10,
+                                                       beta,
+                                                       d_D, 0, ne01,
+                                                       &queue, &ev_sgemm);
+
+            if (status != clblast::StatusCode::kSuccess) {
+                GGML_ASSERT(false);
+            }
+
+            // copy dst to host
+            float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
+            CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL));
        }
    }

@@ -1566,7 +1565,7 @@ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * sr
    ggml_cl_pool_free(d_D, d_size);
 }

-static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, void * wdata, size_t wsize) {
+static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, void * wdata, size_t /* wsize */) {
    GGML_ASSERT(fp16_support);

    const int64_t ne00 = src0->ne[0];
@@ -1576,8 +1575,6 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr

    const int64_t ne10 = src1->ne[0];
    const int64_t ne11 = src1->ne[1];
-    const int64_t ne12 = src1->ne[2];
-    const int64_t ne13 = src1->ne[3];

    const int nb10 = src1->nb[0];
    const int nb11 = src1->nb[1];
@@ -1587,25 +1584,18 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr
    const int nb2  = dst->nb[2];
    const int nb3  = dst->nb[3];

-    const int64_t r2 = ne12 / ne02;
-    const int64_t r3 = ne13 / ne03;
-
    const ggml_fp16_t alpha = ggml_fp32_to_fp16(1.0f);
    const ggml_fp16_t beta = ggml_fp32_to_fp16(0.0f);
    const int x_ne = ne01 * ne00;
    const int y_ne = ne11 * ne10;
    const int d_ne = ne11 * ne01;

-    GGML_ASSERT(wsize >= sizeof(ggml_fp16_t) * y_ne);
-    GGML_ASSERT(wsize >= sizeof(ggml_fp16_t) * d_ne);
-    ggml_fp16_t * const tmp = (ggml_fp16_t *) wdata;
-
    size_t x_size;
    size_t y_size;
    size_t d_size;
    cl_mem d_X;
    if (src0->backend == GGML_BACKEND_GPU) { // NOLINT
-        d_X = (cl_mem) src0->extra;
+        d_X = (cl_mem) src0->data;
    } else {
        d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size);
    }
@@ -1615,71 +1605,63 @@ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * sr
    bool src1_cont_rows = nb10 == sizeof(float);
    bool src1_cont_cols = (size_t)nb11 == ne11*sizeof(float);

-    size_t x_offset = 0;
-
    for (int64_t i03 = 0; i03 < ne03; i03++) {
-        // TODO: copy src0 here when r3>1
-        for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {
-            for (int64_t i02 = 0; i02 < ne02; i02++) {
-                if (src0->backend == GGML_BACKEND_GPU) {
-                    x_offset = (i03 * ne02 + i02) * x_ne;
-                } else {
-                    // copy src0 to device
-                    CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
+        for (int64_t i02 = 0; i02 < ne02; i02++) {
+            // copy src0 to device
+            if (src0->backend != GGML_BACKEND_GPU) {
+                CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
+            }
+
+            // convert src1 to fp16
+            // TODO: use multiple threads
+            ggml_fp16_t * const tmp = (ggml_fp16_t *) wdata + (ne11 * ne10) * (i03 * ne02 + i02);
+            char * src1i = (char *) src1->data + i03*nb13 + i02*nb12;
+            if (src1_cont_rows) {
+                if (src1_cont_cols) {
+                    ggml_fp32_to_fp16_row((float *) src1i, tmp, ne10*ne11);
                }
-
-                for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {
-                    // convert src1 to fp16
-                    // TODO: use multiple threads
-                    char * src1i = (char *) src1->data + i13*nb13 + i12*nb12;
-                    if (src1_cont_rows) {
-                        if (src1_cont_cols) {
-                            ggml_fp32_to_fp16_row((float *) src1i, tmp, ne10*ne11);
-                        }
-                        else {
-                            for (int64_t i11 = 0; i11 < ne11; i11++) {
-                                ggml_fp32_to_fp16_row((float *) (src1i + i11*nb11), tmp + i11*ne10, ne10);
-                            }
-                        }
+                else {
+                    for (int64_t i01 = 0; i01 < ne11; i01++) {
+                        ggml_fp32_to_fp16_row((float *) (src1i + i01*nb11), tmp + i01*ne10, ne10);
                    }
-                    else {
-                        for (int64_t i11 = 0; i11 < ne11; i11++) {
-                            for (int64_t i10 = 0; i10 < ne10; i10++) {
-                                // very slow due to no inlining
-                                tmp[i11*ne10 + i10] = ggml_fp32_to_fp16(*(float *) (src1i + i11*nb11 + i10*nb10));
-                            }
-                        }
-                    }
-
-                    // copy src1 to device
-                    CL_CHECK(clEnqueueWriteBuffer(queue, d_Y, false, 0, sizeof(ggml_fp16_t) * y_ne, tmp, 0, NULL, NULL));
-
-                    CL_CHECK(clFinish(queue));
-
-                    // compute
-                    cl_event ev_sgemm;
-                    clblast::StatusCode status = clblast::Gemm<cl_half>(clblast::Layout::kColMajor,
-                                                               clblast::Transpose::kYes, clblast::Transpose::kNo,
-                                                               ne01, ne11, ne10,
-                                                               alpha,
-                                                               d_X, x_offset, ne00,
-                                                               d_Y, 0, ne10,
-                                                               beta,
-                                                               d_D, 0, ne01,
-                                                               &queue, &ev_sgemm);
-
-                    if (status != clblast::StatusCode::kSuccess) {
-                        GGML_ASSERT(false);
-                    }
-
-                    // copy dst to host, then convert to float
-                    CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL));
-
-                    float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);
-
-                    ggml_fp16_to_fp32_row(tmp, d, d_ne);
                }
            }
+            else {
+                for (int64_t i01 = 0; i01 < ne11; i01++) {
+                    for (int64_t i00 = 0; i00 < ne10; i00++) {
+                        // very slow due to no inlining
+                        tmp[i01*ne10 + i00] = ggml_fp32_to_fp16(*(float *) (src1i + i01*nb11 + i00*nb10));
+                    }
+                }
+            }
+
+            // copy src1 to device
+            CL_CHECK(clEnqueueWriteBuffer(queue, d_Y, false, 0, sizeof(ggml_fp16_t) * y_ne, tmp, 0, NULL, NULL));
+
+            CL_CHECK(clFinish(queue));
+
+            // compute
+            cl_event ev_sgemm;
+            clblast::StatusCode status = clblast::Gemm<cl_half>(clblast::Layout::kColMajor,
+                                                       clblast::Transpose::kYes, clblast::Transpose::kNo,
+                                                       ne01, ne11, ne10,
+                                                       alpha,
+                                                       d_X, 0, ne00,
+                                                       d_Y, 0, ne10,
+                                                       beta,
+                                                       d_D, 0, ne01,
+                                                       &queue, &ev_sgemm);
+
+            if (status != clblast::StatusCode::kSuccess) {
+                GGML_ASSERT(false);
+            }
+
+            // copy dst to host, then convert to float
+            CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL));
+
+            float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
+
+            ggml_fp16_to_fp32_row(tmp, d, d_ne);
        }
    }

@@ -1698,24 +1680,18 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor *

    const int64_t ne10 = src1->ne[0];
    const int64_t ne11 = src1->ne[1];
-    const int64_t ne12 = src1->ne[2];
-    const int64_t ne13 = src1->ne[3];

    const int nb2  = dst->nb[2];
    const int nb3  = dst->nb[3];
    const ggml_type type = src0->type;
-    const bool mul_mat_vec = ne11 == 1 && ne00%2 == 0;
-
-    const int64_t r2 = ne12 / ne02;
-    const int64_t r3 = ne13 / ne03;
+    const bool mul_mat_vec = ne11 == 1;

    const float alpha = 1.0f;
    const float beta = 0.0f;
    const int x_ne = ne01 * ne00;
    const int y_ne = ne11 * ne10;
    const int d_ne = ne11 * ne01;
-    const int x_bps = x_ne / ggml_blck_size(type); // blocks per 2D slice
-    const size_t q_sz = ggml_type_size(type) * x_bps;
+    const size_t q_sz = ggml_type_size(type) * x_ne / ggml_blck_size(type);

    size_t x_size;
    size_t y_size;
@@ -1737,86 +1713,78 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor *
    GGML_ASSERT(to_fp32_cl != nullptr);

    const size_t global_denom = ggml_cl_global_denom(type);
-    const size_t local = mul_mat_vec ? CL_DMMV_LOCAL_SIZE : ggml_cl_local_size(type);
+    const size_t local = ggml_cl_local_size(type);

    size_t ev_idx = 0;
    std::vector<cl_event> events;

    for (int64_t i03 = 0; i03 < ne03; i03++) {
-        // TODO: copy and dequantize src0 here when r3>1
-        for (int64_t i13 = i03 * r3, e13 = i13 + r3; i13 < e13; i13++) {
-            for (int64_t i02 = 0; i02 < ne02; i02++) {
-                // copy src0 to device if necessary
-                if (src0->backend == GGML_BACKEND_CPU) {
-                    events.emplace_back();
-                    CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Q, 0, src0, i03, i02, events.data() + ev_idx++));
-                } else if (src0->backend == GGML_BACKEND_GPU) {
-                    d_Q = (cl_mem) src0->extra;
-                } else {
+        for (int64_t i02 = 0; i02 < ne02; i02++) {
+            // copy src0 to device if necessary
+            if (src0->backend == GGML_BACKEND_CPU) {
+                events.emplace_back();
+                CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Q, 0, src0, i03, i02, events.data() + ev_idx++));
+            } else if (src0->backend == GGML_BACKEND_GPU) {
+                d_Q = (cl_mem) src0->data;
+            } else {
+                GGML_ASSERT(false);
+            }
+            if (mul_mat_vec) { // specialized dequantize_mul_mat_vec kernel
+                // copy src1 to device
+                events.emplace_back();
+                CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, events.data() + ev_idx++));
+
+                // compute
+                const size_t global = ne01 * CL_DMMV_BLOCK_SIZE;
+                const size_t local = CL_DMMV_BLOCK_SIZE;
+                const cl_int ncols = ne00;
+                events.emplace_back();
+                CL_CHECK(clSetKernelArg(*dmmv, 0, sizeof(cl_mem), &d_Q));
+                CL_CHECK(clSetKernelArg(*dmmv, 1, sizeof(float) * local, NULL));
+                CL_CHECK(clSetKernelArg(*dmmv, 2, sizeof(cl_mem), &d_Y));
+                CL_CHECK(clSetKernelArg(*dmmv, 3, sizeof(cl_mem), &d_D));
+                CL_CHECK(clSetKernelArg(*dmmv, 4, sizeof(cl_int), &ncols));
+                CL_CHECK(clEnqueueNDRangeKernel(queue, *dmmv, 1, NULL, &global, &local, events.size() - 1, events.data(), events.data() + ev_idx++));
+            } else { // general dequantization kernel + CLBlast matrix matrix multiplication
+                // convert src0 to fp32 on device
+                const size_t global = x_ne / global_denom;
+                CL_CHECK(clSetKernelArg(*to_fp32_cl, 0, sizeof(cl_mem), &d_Q));
+                CL_CHECK(clSetKernelArg(*to_fp32_cl, 1, sizeof(cl_mem), &d_X));
+                CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, NULL, &global, local > 0 ? &local : NULL, events.size(), !events.empty() ? events.data() : NULL, NULL));
+
+                // copy src1 to device
+                CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, NULL));
+
+                events.emplace_back();
+
+                // wait for conversion
+                CL_CHECK(clFinish(queue));
+
+                // compute
+                clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
+                                                           clblast::Transpose::kYes, clblast::Transpose::kNo,
+                                                           ne01, ne11, ne10,
+                                                           alpha,
+                                                           d_X, 0, ne00,
+                                                           d_Y, 0, ne10,
+                                                           beta,
+                                                           d_D, 0, ne01,
+                                                           &queue, events.data() + ev_idx++);
+
+                if (status != clblast::StatusCode::kSuccess) {
                    GGML_ASSERT(false);
                }
-
-                if (!mul_mat_vec) {
-                    // convert src0 to fp32 on device
-                    const size_t global = x_ne / global_denom;
-                    const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0;
-                    CL_CHECK(clSetKernelArg(*to_fp32_cl, 0, sizeof(cl_mem), &d_Q));
-                    CL_CHECK(clSetKernelArg(*to_fp32_cl, 1, sizeof(cl_mem), &d_X));
-                    CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, &offset, &global, local > 0 ? &local : NULL, events.size(), !events.empty() ? events.data() : NULL, NULL));
-                }
-
-                for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) {
-                    if (mul_mat_vec) { // specialized dequantize_mul_mat_vec kernel
-                        // copy src1 to device
-                        events.emplace_back();
-                        CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, events.data() + ev_idx++));
-
-                        // compute
-                        const size_t global = ne01 * local;
-                        const size_t offset = src0->backend == GGML_BACKEND_GPU ? (i03 * ne02 + i02) * x_bps : 0;
-                        const cl_int ncols = ne00;
-                        events.emplace_back();
-                        CL_CHECK(clSetKernelArg(*dmmv, 0, sizeof(cl_mem), &d_Q));
-                        CL_CHECK(clSetKernelArg(*dmmv, 1, sizeof(float) * local, NULL));
-                        CL_CHECK(clSetKernelArg(*dmmv, 2, sizeof(cl_mem), &d_Y));
-                        CL_CHECK(clSetKernelArg(*dmmv, 3, sizeof(cl_mem), &d_D));
-                        CL_CHECK(clSetKernelArg(*dmmv, 4, sizeof(cl_int), &ncols));
-                        CL_CHECK(clEnqueueNDRangeKernel(queue, *dmmv, 1, &offset, &global, &local, events.size() - 1, events.data(), events.data() + ev_idx++));
-                    } else { // CLBlast matrix matrix multiplication
-                        // copy src1 to device
-                        CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i13, i12, NULL));
-
-                        // wait for conversion
-                        CL_CHECK(clFinish(queue));
-
-                        // compute
-                        events.emplace_back();
-                        clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
-                                                                   clblast::Transpose::kYes, clblast::Transpose::kNo,
-                                                                   ne01, ne11, ne10,
-                                                                   alpha,
-                                                                   d_X, 0, ne00,
-                                                                   d_Y, 0, ne10,
-                                                                   beta,
-                                                                   d_D, 0, ne01,
-                                                                   &queue, events.data() + ev_idx++);
-
-                        if (status != clblast::StatusCode::kSuccess) {
-                            GGML_ASSERT(false);
-                        }
-                    }
-
-                    // copy dst to host
-                    float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);
-                    CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &events[events.size() - 1], NULL));
-                    for (auto *event : events) {
-                        clReleaseEvent(event);
-                    }
-
-                    ev_idx = 0;
-                    events.clear();
-                }
            }
+
+            // copy dst to host
+            float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
+            CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &events[events.size() - 1], NULL));
+            for (auto *event : events) {
+                clReleaseEvent(event);
+            }
+
+            ev_idx = 0;
+            events.clear();
        }
    }

@@ -1848,7 +1816,7 @@ bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tens
    return false;
 }

-static bool ggml_cl_mul_mat_use_f16(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * /* dst */) {
+bool ggml_cl_mul_mat_use_f16(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * /* dst */) {
    // If device doesn't support FP16
    if (!fp16_support) {
        return false;
@@ -1891,8 +1859,8 @@ void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor *
 }

 size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {
-    if (src0->type == GGML_TYPE_F16 && ggml_cl_mul_mat_use_f16(src0, src1, dst)) {
-        return sizeof(ggml_fp16_t) * std::max(src1->ne[0] * src1->ne[1], dst->ne[0] * dst->ne[1]);
+    if (ggml_cl_mul_mat_use_f16(src0, src1, dst)) {
+        return ggml_nelements(src1) * sizeof(ggml_fp16_t);
    }
    return 0;
 }
@@ -1904,24 +1872,22 @@ void ggml_cl_transform_tensor(void * data, ggml_tensor * tensor) {
    const int64_t ne3 = tensor->ne[3];

    const ggml_type type = tensor->type;
-    const size_t s_sz = ggml_type_size(type) * (size_t) (ne0 * ne1 / ggml_blck_size(type));
-    const size_t q_sz = s_sz * (size_t) (ne2 * ne3);
+    const size_t q_sz = ggml_type_size(type) * ne0 * ne1 * ne2 * ne3 / ggml_blck_size(type);

    size_t q_size;
    cl_mem dst = ggml_cl_pool_malloc(q_sz, &q_size);

    tensor->data = data;
    // copy tensor to device
-    size_t offset = 0;
    for (int64_t i3 = 0; i3 < ne3; i3++) {
        for (int64_t i2 = 0; i2 < ne2; i2++) {
-            CL_CHECK(ggml_cl_h2d_tensor_2d(queue, dst, offset, tensor, i3, i2, NULL));
-            offset += s_sz;
+            int i = i3*ne2 + i2;
+            CL_CHECK(ggml_cl_h2d_tensor_2d(queue, dst, i*ne0*ne1, tensor, i3, i2, NULL));
        }
    }

    CL_CHECK(clFinish(queue));

-    tensor->extra = dst;
+    tensor->data = dst;
    GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);
 }
--- a/runner/ggml-opencl.h
+++ b/runner/ggml-opencl.h
@@ -1,7 +1,7 @@
 //go:build opencl

 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
--- a/runner/ggml.c
+++ b/runner/ggml.c
--- a/llm/ggml.go
+++ b/llm/ggml.go
@@ -3,96 +3,69 @@ package llm
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 )

+type ModelFamily string
+
+type ModelType uint32
+
+const (
+	ModelType3B  ModelType = 26
+	ModelType7B  ModelType = 32
+	ModelType13B ModelType = 40
+	ModelType30B ModelType = 60
+	ModelType65B ModelType = 80
+)
+
+func (mt ModelType) String() string {
+	switch mt {
+	case ModelType3B:
+		return "3B"
+	case ModelType7B:
+		return "7B"
+	case ModelType13B:
+		return "13B"
+	case ModelType30B:
+		return "30B"
+	case ModelType65B:
+		return "65B"
+	default:
+		return "Unknown"
+	}
+}
+
+type FileType interface {
+	String() string
+}
+
 type GGML struct {
 	magic uint32
 	container
 	model
 }

-const (
-	fileTypeF32 uint32 = iota
-	fileTypeF16
-	fileTypeQ4_0
-	fileTypeQ4_1
-	fileTypeQ4_1_F16
-	fileTypeQ8_0 uint32 = iota + 2
-	fileTypeQ5_0
-	fileTypeQ5_1
-	fileTypeQ2_K
-	fileTypeQ3_K_S
-	fileTypeQ3_K_M
-	fileTypeQ3_K_L
-	fileTypeQ4_K_S
-	fileTypeQ4_K_M
-	fileTypeQ5_K_S
-	fileTypeQ5_K_M
-	fileTypeQ6_K
-)
-
-func fileType(fileType uint32) string {
-	switch fileType {
-	case fileTypeF32:
-		return "F32"
-	case fileTypeF16:
-		return "F16"
-	case fileTypeQ4_0:
-		return "Q4_0"
-	case fileTypeQ4_1:
-		return "Q4_1"
-	case fileTypeQ4_1_F16:
-		return "Q4_1_F16"
-	case fileTypeQ8_0:
-		return "Q8_0"
-	case fileTypeQ5_0:
-		return "Q5_0"
-	case fileTypeQ5_1:
-		return "Q5_1"
-	case fileTypeQ2_K:
-		return "Q2_K"
-	case fileTypeQ3_K_S:
-		return "Q3_K_S"
-	case fileTypeQ3_K_M:
-		return "Q3_K_M"
-	case fileTypeQ3_K_L:
-		return "Q3_K_L"
-	case fileTypeQ4_K_S:
-		return "Q4_K_S"
-	case fileTypeQ4_K_M:
-		return "Q4_K_M"
-	case fileTypeQ5_K_S:
-		return "Q5_K_S"
-	case fileTypeQ5_K_M:
-		return "Q5_K_M"
-	case fileTypeQ6_K:
-		return "Q6_K"
-	default:
-		return "unknown"
-	}
-}
-
 type model interface {
-	ModelFamily() string
-	ModelType() string
-	FileType() string
-	NumLayers() int64
+	ModelFamily() ModelFamily
+	ModelType() ModelType
+	FileType() FileType
 }

 type container interface {
 	Name() string
-	Decode(io.Reader) (model, error)
+	Decode(io.Reader) error
 }

-type containerGGML struct{}
+type containerGGML struct {
+}

 func (c *containerGGML) Name() string {
 	return "ggml"
 }

-func (c *containerGGML) Decode(r io.Reader) (model, error) {
-	return nil, nil
+func (c *containerGGML) Decode(r io.Reader) error {
+	return nil
 }

 type containerGGMF struct {
@@ -103,18 +76,18 @@ func (c *containerGGMF) Name() string {
 	return "ggmf"
 }

-func (c *containerGGMF) Decode(r io.Reader) (model, error) {
+func (c *containerGGMF) Decode(r io.Reader) error {
 	var version uint32
 	binary.Read(r, binary.LittleEndian, &version)

 	switch version {
 	case 1:
 	default:
-		return nil, errors.New("invalid version")
+		return errors.New("invalid version")
 	}

 	c.version = version
-	return nil, nil
+	return nil
 }

 type containerGGJT struct {
@@ -125,22 +98,18 @@ func (c *containerGGJT) Name() string {
 	return "ggjt"
 }

-func (c *containerGGJT) Decode(r io.Reader) (model, error) {
+func (c *containerGGJT) Decode(r io.Reader) error {
 	var version uint32
 	binary.Read(r, binary.LittleEndian, &version)

 	switch version {
 	case 1, 2, 3:
 	default:
-		return nil, errors.New("invalid version")
+		return errors.New("invalid version")
 	}

 	c.version = version
-
-	// different model types may have different layouts for hyperparameters
-	var llama llamaModel
-	binary.Read(r, binary.LittleEndian, &llama.hyperparameters)
-	return &llama, nil
+	return nil
 }

 type containerLORA struct {
@@ -151,34 +120,32 @@ func (c *containerLORA) Name() string {
 	return "ggla"
 }

-func (c *containerLORA) Decode(r io.Reader) (model, error) {
+func (c *containerLORA) Decode(r io.Reader) error {
 	var version uint32
 	binary.Read(r, binary.LittleEndian, &version)

 	switch version {
 	case 1:
 	default:
-		return nil, errors.New("invalid version")
+		return errors.New("invalid version")
 	}

 	c.version = version
-	return nil, nil
+	return nil
 }

 const (
-	// Magic constant for `ggml` files (unversioned).
+	// / Magic constant for `ggml` files (unversioned).
 	FILE_MAGIC_GGML = 0x67676d6c
-	// Magic constant for `ggml` files (versioned, ggmf).
+	// / Magic constant for `ggml` files (versioned, ggmf).
 	FILE_MAGIC_GGMF = 0x67676d66
-	// Magic constant for `ggml` files (versioned, ggjt).
+	// / Magic constant for `ggml` files (versioned, ggjt).
 	FILE_MAGIC_GGJT = 0x67676a74
-	// Magic constant for `ggla` files (LoRA adapter).
+	// / Magic constant for `ggla` files (LoRA adapter).
 	FILE_MAGIC_GGLA = 0x67676C61
-	// Magic constant for `gguf` files (versioned, gguf)
-	FILE_MAGIC_GGUF = 0x46554747
 )

-func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
+func DecodeGGML(r io.ReadSeeker, hint ModelFamily) (*GGML, error) {
 	var ggml GGML
 	binary.Read(r, binary.LittleEndian, &ggml.magic)

@@ -191,18 +158,24 @@ func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
 		ggml.container = &containerGGJT{}
 	case FILE_MAGIC_GGLA:
 		ggml.container = &containerLORA{}
-	case FILE_MAGIC_GGUF:
-		ggml.container = &containerGGUF{}
 	default:
 		return nil, errors.New("invalid file magic")
 	}

-	model, err := ggml.Decode(r)
-	if err != nil {
+	if err := ggml.Decode(r); err != nil {
 		return nil, err
 	}

-	ggml.model = model
+	// different model types may have different layouts for hyperparameters
+	switch hint {
+	case ModelFamilyLlama:
+		var llama llamaModel
+		binary.Read(r, binary.LittleEndian, &llama.hyperparameters)
+		ggml.model = &llama
+		// TODO: sanity check hyperparameters
+	default:
+		return nil, fmt.Errorf("unsupported model type: %s", hint)
+	}

 	// final model type
 	return &ggml, nil
--- a/runner/ggml.h
+++ b/runner/ggml.h
@@ -1,5 +1,5 @@
 /**
- * llama.cpp - git 465219b9143ac01db0990bbcb0a081ef72ec2008
+ * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
 *
 * MIT License
 *
@@ -156,16 +156,13 @@
 // The data of the tensor is accessed via the "data" pointer. For example:
 //
 //   {
-//       const int nx = 2;
-//       const int ny = 3;
+//       struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2, 3);
 //
-//       struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
+//       // a[2, 1] = 1.0f;
+//       *(float *) ((char *) a->data + 2*a->nb[1] + 1*a->nb[0]) = 1.0f;
 //
-//       for (int y = 0; y < ny; y++) {
-//           for (int x = 0; x < nx; x++) {
-//               *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
-//           }
-//       }
+//       // a[0, 2] = 2.0f;
+//       *(float *) ((char *) a->data + 0*a->nb[1] + 2*a->nb[0]) = 2.0f;
 //
 //       ...
 //   }
@@ -221,14 +218,6 @@
 #    define GGML_DEPRECATED(func, hint) func
 #endif

-#ifndef __GNUC__
-#    define GGML_ATTRIBUTE_FORMAT(...)
-#elif defined(__MINGW32__)
-#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
-#else
-#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
-#endif
-
 #include <stdint.h>
 #include <stddef.h>
 #include <stdbool.h>
@@ -240,29 +229,18 @@
 #define GGML_QNT_VERSION_FACTOR 1000 // do not change this

 #define GGML_MAX_DIMS          4
-#define GGML_MAX_NODES         16384
-#define GGML_MAX_PARAMS        1024
+#define GGML_MAX_NODES         4096
+#define GGML_MAX_PARAMS        256
 #define GGML_MAX_CONTEXTS      64
 #define GGML_MAX_SRC           6
-#define GGML_MAX_NAME          64
+#define GGML_MAX_NAME          48
 #define GGML_MAX_OP_PARAMS     32
 #define GGML_DEFAULT_N_THREADS 4

-#if UINTPTR_MAX == 0xFFFFFFFF
-    #define GGML_MEM_ALIGN 4
-#else
-    #define GGML_MEM_ALIGN 16
-#endif

 #define GGML_EXIT_SUCCESS 0
 #define GGML_EXIT_ABORTED 1

-#define GGUF_MAGIC "GGUF"
-
-#define GGUF_VERSION 3
-
-#define GGUF_DEFAULT_ALIGNMENT 32
-
 #define GGML_UNUSED(x) (void)(x)

 #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
@@ -275,14 +253,6 @@
        } \
    } while (0)

-#ifndef NDEBUG
-#define GGML_UNREACHABLE() GGML_ASSERT(!"statement should not be reached")
-#elif defined(__GNUC__)
-#define GGML_UNREACHABLE() __builtin_unreachable()
-#else
-#define GGML_UNREACHABLE() ((void) 0)
-#endif
-
 // used to copy the number of elements and stride in bytes of tensors into local variables.
 // main purpose is to reduce code duplication and improve readability.
 //
@@ -311,9 +281,8 @@
 extern "C" {
 #endif

-#if defined(__ARM_NEON) && defined(__CUDACC__)
-    typedef half ggml_fp16_t;
-#elif defined(__ARM_NEON)
+#ifdef __ARM_NEON
+    // we use the built-in 16-bit float type
    typedef __fp16 ggml_fp16_t;
 #else
    typedef uint16_t ggml_fp16_t;
@@ -353,7 +322,7 @@ extern "C" {
        GGML_TYPE_COUNT,
    };

-    enum ggml_backend_type {
+    enum ggml_backend {
        GGML_BACKEND_CPU = 0,
        GGML_BACKEND_GPU = 10,
        GGML_BACKEND_GPU_SPLIT = 20,
@@ -397,12 +366,10 @@ extern "C" {
        GGML_OP_ARGMAX,
        GGML_OP_REPEAT,
        GGML_OP_REPEAT_BACK,
-        GGML_OP_CONCAT,
        GGML_OP_SILU_BACK,
        GGML_OP_NORM, // normalize
        GGML_OP_RMS_NORM,
        GGML_OP_RMS_NORM_BACK,
-        GGML_OP_GROUP_NORM,

        GGML_OP_MUL_MAT,
        GGML_OP_OUT_PROD,
@@ -428,23 +395,14 @@ extern "C" {
        GGML_OP_CLAMP,
        GGML_OP_CONV_1D,
        GGML_OP_CONV_2D,
-        GGML_OP_CONV_TRANSPOSE_1D,
-        GGML_OP_CONV_TRANSPOSE_2D,
        GGML_OP_POOL_1D,
        GGML_OP_POOL_2D,

-        GGML_OP_CONV_1D_STAGE_0,  // internal
-        GGML_OP_CONV_1D_STAGE_1,  // internal
-
-        GGML_OP_UPSCALE, // nearest interpolate
-
        GGML_OP_FLASH_ATTN,
        GGML_OP_FLASH_FF,
        GGML_OP_FLASH_ATTN_BACK,
        GGML_OP_WIN_PART,
        GGML_OP_WIN_UNPART,
-        GGML_OP_GET_REL_POS,
-        GGML_OP_ADD_REL_POS,

        GGML_OP_UNARY,

@@ -484,12 +442,6 @@ extern "C" {
        GGML_OBJECT_WORK_BUFFER
    };

-    enum ggml_log_level {
-        GGML_LOG_LEVEL_ERROR = 2,
-        GGML_LOG_LEVEL_WARN = 3,
-        GGML_LOG_LEVEL_INFO = 4
-    };
-
    // ggml object
    struct ggml_object {
        size_t offs;
@@ -506,16 +458,14 @@ extern "C" {

    // n-dimensional tensor
    struct ggml_tensor {
-        enum ggml_type         type;
-        enum ggml_backend_type backend;
-
-        struct ggml_backend_buffer * buffer;
+        enum ggml_type    type;
+        enum ggml_backend backend;

        int     n_dims;
        int64_t ne[GGML_MAX_DIMS]; // number of elements
        size_t  nb[GGML_MAX_DIMS]; // stride in bytes:
-                                   // nb[0] = ggml_type_size(type)
-                                   // nb[1] = nb[0]   * (ne[0] / ggml_blck_size(type)) + padding
+                                   // nb[0] = sizeof(type)
+                                   // nb[1] = nb[0]   * ne[0] + padding
                                   // nb[i] = nb[i-1] * ne[i-1]

        // compute data
@@ -534,16 +484,13 @@ extern "C" {
        int64_t perf_cycles;
        int64_t perf_time_us;

-        struct ggml_tensor * view_src;
-        size_t               view_offs;
-
        void * data;

        char name[GGML_MAX_NAME];

        void * extra; // extra things e.g. for ggml-cuda.cu

-        char padding[12];
+        char padding[4];
    };

    static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
@@ -567,15 +514,7 @@ extern "C" {
    // next prime after GGML_MAX_NODES
    // #define GGML_GRAPH_HASHTABLE_SIZE 4099
    // next prime after GGML_MAX_NODES * 2 (nodes + leafs)
-    // #define GGML_GRAPH_HASHTABLE_SIZE 8273
-    // #define GGML_GRAPH_HASHTABLE_SIZE 16411
-    #define GGML_GRAPH_HASHTABLE_SIZE 32771
-
-    enum ggml_cgraph_eval_order {
-        GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
-        GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
-        GGML_CGRAPH_EVAL_ORDER_COUNT
-    };
+    #define GGML_GRAPH_HASHTABLE_SIZE 8273

    // computation graph
    struct ggml_cgraph {
@@ -588,8 +527,6 @@ extern "C" {

        void * visited_hash_table[GGML_GRAPH_HASHTABLE_SIZE];

-        enum ggml_cgraph_eval_order order;
-
        // performance
        int     perf_runs;
        int64_t perf_cycles;
@@ -651,7 +588,6 @@ extern "C" {
    GGML_API int64_t ggml_nelements   (const struct ggml_tensor * tensor);
    GGML_API int64_t ggml_nrows       (const struct ggml_tensor * tensor);
    GGML_API size_t  ggml_nbytes      (const struct ggml_tensor * tensor);
-    GGML_API size_t  ggml_nbytes_pad  (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
    GGML_API size_t  ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split);

    GGML_API int     ggml_blck_size (enum ggml_type type);
@@ -729,32 +665,20 @@ extern "C" {
    GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);

    GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
-    GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
+    GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, const struct ggml_tensor * src);

-    // Context tensor enumeration and lookup
-    GGML_API struct ggml_tensor * ggml_get_first_tensor(struct ggml_context * ctx);
-    GGML_API struct ggml_tensor * ggml_get_next_tensor (struct ggml_context * ctx, struct ggml_tensor * tensor);
    GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);

    GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
    GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);
    GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);

-    // Converts a flat index into coordinates
-    GGML_API void    ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
-
    GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);
    GGML_API void    ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);

-    GGML_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
-    GGML_API void    ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
-
    GGML_API float   ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);
    GGML_API void    ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);

-    GGML_API float   ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
-    GGML_API void    ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
-
    GGML_API void *  ggml_get_data    (const struct ggml_tensor * tensor);
    GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);

@@ -762,7 +686,6 @@ extern "C" {

    GGML_API const char *         ggml_get_name   (const struct ggml_tensor * tensor);
    GGML_API struct ggml_tensor * ggml_set_name   (      struct ggml_tensor * tensor, const char * name);
-    GGML_ATTRIBUTE_FORMAT(2, 3)
    GGML_API struct ggml_tensor * ggml_format_name(      struct ggml_tensor * tensor, const char * fmt, ...);

    //
@@ -788,12 +711,6 @@ extern "C" {
            struct ggml_tensor  * a,
            struct ggml_tensor  * b);

-    GGML_API struct ggml_tensor * ggml_add_cast(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            enum   ggml_type      type);
-
    GGML_API struct ggml_tensor * ggml_add1(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
@@ -903,19 +820,11 @@ extern "C" {
            struct ggml_tensor  * a,
            struct ggml_tensor  * b);

-    // sums repetitions in a into shape of b
    GGML_API struct ggml_tensor * ggml_repeat_back(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
            struct ggml_tensor  * b);

-    // concat a and b on dim 2
-    // used in stable-diffusion
-    GGML_API struct ggml_tensor * ggml_concat(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b);
-
    GGML_API struct ggml_tensor * ggml_abs(
            struct ggml_context * ctx,
            struct ggml_tensor  * a);
@@ -1005,15 +914,14 @@ extern "C" {
            struct ggml_tensor  * b);

    // normalize along rows
+    // TODO: eps is hardcoded to 1e-5 for now
    GGML_API struct ggml_tensor * ggml_norm(
            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            float                 eps);
+            struct ggml_tensor  * a);

    GGML_API struct ggml_tensor * ggml_norm_inplace(
            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            float                 eps);
+            struct ggml_tensor  * a);

    GGML_API struct ggml_tensor * ggml_rms_norm(
            struct ggml_context * ctx,
@@ -1025,26 +933,13 @@ extern "C" {
            struct ggml_tensor  * a,
            float                 eps);

-    // group normalize along ne0*ne1*n_groups
-    // used in stable-diffusion
-    // TODO: eps is hardcoded to 1e-6 for now
-    GGML_API struct ggml_tensor * ggml_group_norm(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int                   n_groups);
-
-    GGML_API struct ggml_tensor * ggml_group_norm_inplace(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int                   n_groups);
-
    // a - x
    // b - dy
+    // TODO: update with configurable eps
    GGML_API struct ggml_tensor * ggml_rms_norm_back(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            float                 eps);
+            struct ggml_tensor  * b);

    // A: n columns, m rows
    // B: n columns, p rows  (i.e. we transpose it internally)
@@ -1125,6 +1020,7 @@ extern "C" {
            size_t                nb1,
            size_t                offset);

+
    // a -> b, return view(b)
    GGML_API struct ggml_tensor * ggml_cpy(
            struct ggml_context * ctx,
@@ -1147,33 +1043,6 @@ extern "C" {
            struct ggml_context * ctx,
            struct ggml_tensor  * a);

-    // make contiguous, with new shape
-    GGML_API struct ggml_tensor * ggml_cont_1d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int64_t               ne0);
-
-    GGML_API struct ggml_tensor * ggml_cont_2d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int64_t               ne0,
-            int64_t               ne1);
-
-    GGML_API struct ggml_tensor * ggml_cont_3d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int64_t               ne0,
-            int64_t               ne1,
-            int64_t               ne2);
-
-    GGML_API struct ggml_tensor * ggml_cont_4d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int64_t               ne0,
-            int64_t               ne1,
-            int64_t               ne2,
-            int64_t               ne3);
-
    // return view(a), b specifies the new shape
    // TODO: when we start computing gradient, make a copy instead of view
    GGML_API struct ggml_tensor * ggml_reshape(
@@ -1321,15 +1190,14 @@ extern "C" {
            struct ggml_tensor  * b);

    // rotary position embedding
-    // if mode & 1 == 1, skip n_past elements (DEPRECATED)
+    // if mode & 1 == 1, skip n_past elements
    // if mode & 2 == 1, GPT-NeoX style
    // if mode & 4 == 1, ChatGLM style
-    //
-    // b is an int32 vector with size a->ne[2], it contains the positions
+    // TODO: avoid creating a new tensor every time
    GGML_API struct ggml_tensor * ggml_rope(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
+            int                   n_past,
            int                   n_dims,
            int                   mode,
            int                   n_ctx);
@@ -1338,7 +1206,7 @@ extern "C" {
    GGML_API struct ggml_tensor * ggml_rope_inplace(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
+            int                   n_past,
            int                   n_dims,
            int                   mode,
            int                   n_ctx);
@@ -1347,7 +1215,7 @@ extern "C" {
    GGML_API struct ggml_tensor * ggml_rope_custom(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
+            int                   n_past,
            int                   n_dims,
            int                   mode,
            int                   n_ctx,
@@ -1358,39 +1226,26 @@ extern "C" {
    GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
+            int                   n_past,
            int                   n_dims,
            int                   mode,
            int                   n_ctx,
            float                 freq_base,
            float                 freq_scale);

-    // xPos RoPE, in-place, returns view(a)
-    GGML_API struct ggml_tensor * ggml_rope_xpos_inplace(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            int                   n_dims,
-            float                 base,
-            bool                  down);
-
    // rotary position embedding backward, i.e compute dx from dy
    // a - dy
    GGML_API struct ggml_tensor * ggml_rope_back(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
+            int                   n_past,
            int                   n_dims,
            int                   mode,
-            int                   n_ctx,
-            float                 freq_base,
-            float                 freq_scale,
-            float                 xpos_base,
-            bool                  xpos_down);
+            int                   n_ctx);

    // alibi position embedding
    // in-place, returns view(a)
-    GGML_API struct ggml_tensor * ggml_alibi(
+    struct ggml_tensor * ggml_alibi(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
            int                   n_past,
@@ -1399,7 +1254,7 @@ extern "C" {

    // clamp
    // in-place, returns view(a)
-    GGML_API struct ggml_tensor * ggml_clamp(
+    struct ggml_tensor * ggml_clamp(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
            float                 min,
@@ -1413,23 +1268,6 @@ extern "C" {
            int                   p0,  // padding
            int                   d0); // dilation

-    // conv_1d with padding = half
-    // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
-    GGML_API struct ggml_tensor* ggml_conv_1d_ph(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            int                   s,
-            int                   d);
-
-    GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            int                   s0,
-            int                   p0,
-            int                   d0);
-
    GGML_API struct ggml_tensor * ggml_conv_2d(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
@@ -1441,38 +1279,14 @@ extern "C" {
            int                   d0,
            int                   d1);

-
-    // kernel size is a->ne[0] x a->ne[1]
-    // stride is equal to kernel size
-    // padding is zero
-    // example:
-    // a:     16   16    3  768
-    // b:   1024 1024    3    1
-    // res:   64   64  768    1
-    // used in sam
-    GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b);
-
-    // kernel size is a->ne[0] x a->ne[1]
-    // stride is 1
-    // padding is half
-    // example:
-    // a:      3    3    256  256
-    // b:     64   64    256    1
-    // res:   64   64    256    1
-    // used in sam
-    GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b);
-
-    GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
+    // conv_1d with padding = half
+    // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
+    GGML_API struct ggml_tensor * ggml_conv_1d_ph(
            struct ggml_context * ctx,
            struct ggml_tensor  * a,
            struct ggml_tensor  * b,
-            int                   stride);
+            int                   s,
+            int                   d);

    enum ggml_op_pool {
        GGML_OP_POOL_MAX,
@@ -1499,13 +1313,6 @@ extern "C" {
            int                   p0,
            int                   p1);

-    // nearest interpolate
-    // used in stable-diffusion
-    GGML_API struct ggml_tensor * ggml_upscale(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int                   scale_factor);
-
    GGML_API struct ggml_tensor * ggml_flash_attn(
            struct ggml_context * ctx,
            struct ggml_tensor  * q,
@@ -1559,27 +1366,6 @@ extern "C" {
        struct ggml_tensor  * a,
        enum ggml_unary_op op);

-    // used in sam
-    GGML_API struct ggml_tensor * ggml_get_rel_pos(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int                   qh,
-            int                   kh);
-
-    // used in sam
-
-    GGML_API struct ggml_tensor * ggml_add_rel_pos(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * pw,
-            struct ggml_tensor  * ph);
-
-    GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * pw,
-            struct ggml_tensor  * ph);
-
    // custom operators

    typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *);
@@ -1735,8 +1521,7 @@ extern "C" {
            struct ggml_tensor  * tensor);


-    GGML_API void ggml_build_forward_expand (struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
-    GGML_API void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep);
+    GGML_API void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);

    GGML_API struct ggml_cgraph ggml_build_forward (struct ggml_tensor * tensor);
    GGML_API struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep);
@@ -1767,16 +1552,6 @@ extern "C" {
    // dump the graph into a file using the dot format
    GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);

-    // build gradient checkpointing backward graph gb for gf using provided checkpoints
-    // gb_tmp will contain original backward graph with rewritten backward process nodes,
-    // but without the second forward pass nodes.
-    GGML_API void ggml_build_backward_gradient_checkpointing(
-            struct ggml_context   * ctx,
-            struct ggml_cgraph    * gf,
-            struct ggml_cgraph    * gb,
-            struct ggml_cgraph    * gb_tmp,
-            struct ggml_tensor  * * checkpoints,
-            int                     n_checkpoints);
    //
    // optimization
    //
@@ -1803,7 +1578,6 @@ extern "C" {
        GGML_OPT_NO_CONTEXT,
        GGML_OPT_INVALID_WOLFE,
        GGML_OPT_FAIL,
-        GGML_OPT_CANCEL,

        GGML_LINESEARCH_FAIL = -128,
        GGML_LINESEARCH_MINIMUM_STEP,
@@ -1812,9 +1586,6 @@ extern "C" {
        GGML_LINESEARCH_INVALID_PARAMETERS,
    };

-    typedef void (*ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
-    typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
-
    // optimization parameters
    //
    //   see ggml.c (ggml_opt_default_params) for default values
@@ -1844,22 +1615,18 @@ extern "C" {
        bool print_forward_graph;
        bool print_backward_graph;

-        int n_gradient_accumulation;
-
        // ADAM parameters
        struct {
            int n_iter;

            float sched; // schedule multiplier (fixed, decay or warmup)
            float decay; // weight decay for AdamW, use 0.0f to disable
-            int   decay_min_ndim; // minimum number of tensor dimension to apply weight decay
            float alpha; // learning rate
            float beta1;
            float beta2;
            float eps;   // epsilon for numerical stability
            float eps_f; // epsilon for convergence test
            float eps_g; // epsilon for convergence test
-            float gclip; // gradient clipping
        } adam;

        // LBFGS parameters
@@ -1887,13 +1654,14 @@ extern "C" {

        bool just_initialized;

-        float loss_before;
-        float loss_after;
-
        struct {
-            struct ggml_tensor * g;  // current gradient
+            struct ggml_tensor * x;  // view of the parameters
+            struct ggml_tensor * g1; // gradient
+            struct ggml_tensor * g2; // gradient squared
            struct ggml_tensor * m;  // first moment
            struct ggml_tensor * v;  // second moment
+            struct ggml_tensor * mh; // first moment hat
+            struct ggml_tensor * vh; // second moment hat
            struct ggml_tensor * pf; // past function values
            float fx_best;
            float fx_prev;
@@ -1930,10 +1698,10 @@ extern "C" {

    // initialize optimizer context
    GGML_API void ggml_opt_init(
-            struct ggml_context     * ctx,
+            struct ggml_context * ctx,
            struct ggml_opt_context * opt,
-            struct ggml_opt_params    params,
-            int64_t                   nx);
+            struct ggml_opt_params params,
+            int64_t nx);

    // continue optimizing the function defined by the tensor f
    GGML_API enum ggml_opt_result ggml_opt_resume(
@@ -1947,9 +1715,7 @@ extern "C" {
            struct ggml_opt_context * opt,
            struct ggml_tensor * f,
            struct ggml_cgraph * gf,
-            struct ggml_cgraph * gb,
-            ggml_opt_callback callback,
-            void * callback_data);
+            struct ggml_cgraph * gb);

    //
    // quantization
@@ -1963,127 +1729,6 @@ extern "C" {

    GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist);

-    //
-    // gguf
-    //
-
-    enum gguf_type {
-        GGUF_TYPE_UINT8   = 0,
-        GGUF_TYPE_INT8    = 1,
-        GGUF_TYPE_UINT16  = 2,
-        GGUF_TYPE_INT16   = 3,
-        GGUF_TYPE_UINT32  = 4,
-        GGUF_TYPE_INT32   = 5,
-        GGUF_TYPE_FLOAT32 = 6,
-        GGUF_TYPE_BOOL    = 7,
-        GGUF_TYPE_STRING  = 8,
-        GGUF_TYPE_ARRAY   = 9,
-        GGUF_TYPE_UINT64  = 10,
-        GGUF_TYPE_INT64   = 11,
-        GGUF_TYPE_FLOAT64 = 12,
-        GGUF_TYPE_COUNT,       // marks the end of the enum
-    };
-
-    struct gguf_context;
-
-    struct gguf_init_params {
-        bool no_alloc;
-
-        // if not NULL, create a ggml_context and allocate the tensor data in it
-        struct ggml_context ** ctx;
-    };
-
-    GGML_API struct gguf_context * gguf_init_empty(void);
-    GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
-    //GGML_API struct gguf_context * gguf_init_from_buffer(..);
-
-    GGML_API void gguf_free(struct gguf_context * ctx);
-
-    GGML_API const char * gguf_type_name(enum gguf_type type);
-
-    GGML_API int    gguf_get_version    (const struct gguf_context * ctx);
-    GGML_API size_t gguf_get_alignment  (const struct gguf_context * ctx);
-    GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);
-    GGML_API void * gguf_get_data       (const struct gguf_context * ctx);
-
-    GGML_API int          gguf_get_n_kv(const struct gguf_context * ctx);
-    GGML_API int          gguf_find_key(const struct gguf_context * ctx, const char * key);
-    GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int key_id);
-
-    GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int key_id);
-    GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id);
-
-    // will abort if the wrong type is used for the key
-    GGML_API uint8_t      gguf_get_val_u8  (const struct gguf_context * ctx, int key_id);
-    GGML_API int8_t       gguf_get_val_i8  (const struct gguf_context * ctx, int key_id);
-    GGML_API uint16_t     gguf_get_val_u16 (const struct gguf_context * ctx, int key_id);
-    GGML_API int16_t      gguf_get_val_i16 (const struct gguf_context * ctx, int key_id);
-    GGML_API uint32_t     gguf_get_val_u32 (const struct gguf_context * ctx, int key_id);
-    GGML_API int32_t      gguf_get_val_i32 (const struct gguf_context * ctx, int key_id);
-    GGML_API float        gguf_get_val_f32 (const struct gguf_context * ctx, int key_id);
-    GGML_API uint64_t     gguf_get_val_u64 (const struct gguf_context * ctx, int key_id);
-    GGML_API int64_t      gguf_get_val_i64 (const struct gguf_context * ctx, int key_id);
-    GGML_API double       gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);
-    GGML_API bool         gguf_get_val_bool(const struct gguf_context * ctx, int key_id);
-    GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);
-    GGML_API int          gguf_get_arr_n   (const struct gguf_context * ctx, int key_id);
-    GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
-    GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
-
-    GGML_API int    gguf_get_n_tensors    (const struct gguf_context * ctx);
-    GGML_API int    gguf_find_tensor      (const struct gguf_context * ctx, const char * name);
-    GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
-    GGML_API char * gguf_get_tensor_name  (const struct gguf_context * ctx, int i);
-
-    // overrides existing values or adds a new one
-    GGML_API void gguf_set_val_u8  (struct gguf_context * ctx, const char * key, uint8_t  val);
-    GGML_API void gguf_set_val_i8  (struct gguf_context * ctx, const char * key, int8_t   val);
-    GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
-    GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t  val);
-    GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
-    GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t  val);
-    GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float    val);
-    GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
-    GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t  val);
-    GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double   val);
-    GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool     val);
-    GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
-    GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);
-    GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);
-
-    // set or add KV pairs from another context
-    GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);
-
-    // manage tensor info
-    GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
-    GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
-    GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);
-
-    // writing gguf files can be done in 2 ways:
-    //
-    // - write the entire gguf_context to a binary file in a single pass:
-    //
-    //   gguf_write_to_file(ctx, fname);
-    //
-    // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
-    //
-    //   FILE * f = fopen(fname, "wb");
-    //   fseek(f, gguf_get_meta_size(ctx), SEEK_SET);
-    //   fwrite(f, ...);
-    //   void * data = gguf_meta_get_meta_data(ctx);
-    //   fseek(f, 0, SEEK_SET);
-    //   fwrite(f, data, gguf_get_meta_size(ctx));
-    //   free(data);
-    //   fclose(f);
-    //
-
-    // write the entire context to a binary file
-    GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
-
-    // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
-    GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
-    GGML_API void   gguf_get_meta_data(const struct gguf_context * ctx, void * data);
-
    //
    // system info
    //
@@ -2096,7 +1741,6 @@ extern "C" {
    GGML_API int ggml_cpu_has_fma        (void);
    GGML_API int ggml_cpu_has_neon       (void);
    GGML_API int ggml_cpu_has_arm_fma    (void);
-    GGML_API int ggml_cpu_has_metal      (void);
    GGML_API int ggml_cpu_has_f16c       (void);
    GGML_API int ggml_cpu_has_fp16_va    (void);
    GGML_API int ggml_cpu_has_wasm_simd  (void);
@@ -2105,7 +1749,6 @@ extern "C" {
    GGML_API int ggml_cpu_has_clblast    (void);
    GGML_API int ggml_cpu_has_gpublas    (void);
    GGML_API int ggml_cpu_has_sse3       (void);
-    GGML_API int ggml_cpu_has_ssse3      (void);
    GGML_API int ggml_cpu_has_vsx        (void);

    //
@@ -2123,10 +1766,6 @@ extern "C" {
    typedef void (*ggml_vec_dot_t)   (const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT x, const void * GGML_RESTRICT y);

    typedef struct {
-        const char      * type_name;
-        int               blck_size;
-        size_t            type_size;
-        bool              is_quantized;
        ggml_to_float_t   to_float;
        ggml_from_float_t from_float;
        ggml_from_float_t from_float_reference;
@@ -2134,7 +1773,7 @@ extern "C" {
        enum ggml_type    vec_dot_type;
    } ggml_type_traits_t;

-    GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);
+    ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type i);

 #ifdef  __cplusplus
 }
--- a/llm/gguf.go
+++ b/llm/gguf.go
@@ -1,383 +0,0 @@
-package llm
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"io"
-)
-
-type containerGGUF struct {
-	Version uint32
-
-	V1 struct {
-		NumTensor uint32
-		NumKV     uint32
-	}
-
-	V2 struct {
-		NumTensor uint64
-		NumKV     uint64
-	}
-}
-
-func (c *containerGGUF) Name() string {
-	return "gguf"
-}
-
-func (c *containerGGUF) Decode(r io.Reader) (model, error) {
-	binary.Read(r, binary.LittleEndian, &c.Version)
-
-	switch c.Version {
-	case 1:
-		binary.Read(r, binary.LittleEndian, &c.V1)
-	case 2:
-		binary.Read(r, binary.LittleEndian, &c.V2)
-	default:
-		return nil, errors.New("invalid version")
-	}
-
-	model := newGGUFModel(c)
-	if err := model.Decode(r); err != nil {
-		return nil, err
-	}
-
-	return model, nil
-}
-
-const (
-	ggufTypeUint8 uint32 = iota
-	ggufTypeInt8
-	ggufTypeUint16
-	ggufTypeInt16
-	ggufTypeUint32
-	ggufTypeInt32
-	ggufTypeFloat32
-	ggufTypeBool
-	ggufTypeString
-	ggufTypeArray
-	ggufTypeUint64
-	ggufTypeInt64
-	ggufTypeFloat64
-)
-
-type kv map[string]any
-
-type ggufModel struct {
-	*containerGGUF
-	kv
-}
-
-func newGGUFModel(container *containerGGUF) *ggufModel {
-	return &ggufModel{
-		containerGGUF: container,
-		kv:            make(kv),
-	}
-}
-
-func (llm *ggufModel) NumKV() uint64 {
-	if llm.Version == 1 {
-		return uint64(llm.V1.NumKV)
-	}
-
-	return llm.V2.NumKV
-}
-
-func (llm *ggufModel) ModelFamily() string {
-	t, ok := llm.kv["general.architecture"].(string)
-	if ok {
-		return t
-	}
-
-	return "unknown"
-}
-
-func (llm *ggufModel) ModelType() string {
-	switch llm.ModelFamily() {
-	case "llama":
-		if blocks, ok := llm.kv["llama.block_count"].(uint32); ok {
-			heads, headsOK := llm.kv["llama.head_count"].(uint32)
-			headKVs, headsKVsOK := llm.kv["llama.head_count_kv"].(uint32)
-			if headsOK && headsKVsOK && heads/headKVs == 8 {
-				return "70B"
-			}
-
-			return llamaModelType(blocks)
-		}
-	case "falcon":
-		if blocks, ok := llm.kv["falcon.block_count"].(uint32); ok {
-			return falconModelType(blocks)
-		}
-	case "starcoder":
-		if blocks, ok := llm.kv["starcoder.block_count"].(uint32); ok {
-			return starCoderModelType(blocks)
-		}
-	}
-
-	return "unknown"
-}
-
-func (llm *ggufModel) FileType() string {
-	t, ok := llm.kv["general.file_type"].(uint32)
-	if ok {
-		return fileType(t)
-	}
-
-	return "unknown"
-}
-
-func (llm *ggufModel) Decode(r io.Reader) error {
-	read := llm.readString
-	if llm.Version == 1 {
-		read = llm.readStringV1
-	}
-
-	for i := 0; uint64(i) < llm.NumKV(); i++ {
-		k, err := read(r)
-		if err != nil {
-			return err
-		}
-
-		vtype := llm.readU32(r)
-
-		var v any
-		switch vtype {
-		case ggufTypeUint8:
-			v = llm.readU8(r)
-		case ggufTypeInt8:
-			v = llm.readI8(r)
-		case ggufTypeUint16:
-			v = llm.readU16(r)
-		case ggufTypeInt16:
-			v = llm.readI16(r)
-		case ggufTypeUint32:
-			v = llm.readU32(r)
-		case ggufTypeInt32:
-			v = llm.readI32(r)
-		case ggufTypeUint64:
-			v = llm.readU64(r)
-		case ggufTypeInt64:
-			v = llm.readI64(r)
-		case ggufTypeFloat32:
-			v = llm.readF32(r)
-		case ggufTypeFloat64:
-			v = llm.readF64(r)
-		case ggufTypeBool:
-			v = llm.readBool(r)
-		case ggufTypeString:
-			fn := llm.readString
-			if llm.Version == 1 {
-				fn = llm.readStringV1
-			}
-
-			s, err := fn(r)
-			if err != nil {
-				return err
-			}
-
-			v = s
-		case ggufTypeArray:
-			fn := llm.readArray
-			if llm.Version == 1 {
-				fn = llm.readArrayV1
-			}
-
-			a, err := fn(r)
-			if err != nil {
-				return err
-			}
-
-			v = a
-		default:
-			return fmt.Errorf("invalid type: %d", vtype)
-		}
-
-		llm.kv[k] = v
-	}
-
-	return nil
-}
-
-func (llm *ggufModel) NumLayers() int64 {
-	value, exists := llm.kv[fmt.Sprintf("%s.block_count", llm.ModelFamily())]
-	if !exists {
-		return 0
-	}
-
-	v := value.(uint32)
-	return int64(v)
-}
-
-func (ggufModel) readU8(r io.Reader) uint8 {
-	var u8 uint8
-	binary.Read(r, binary.LittleEndian, &u8)
-	return u8
-}
-
-func (ggufModel) readI8(r io.Reader) int8 {
-	var i8 int8
-	binary.Read(r, binary.LittleEndian, &i8)
-	return i8
-}
-
-func (ggufModel) readU16(r io.Reader) uint16 {
-	var u16 uint16
-	binary.Read(r, binary.LittleEndian, &u16)
-	return u16
-}
-
-func (ggufModel) readI16(r io.Reader) int16 {
-	var i16 int16
-	binary.Read(r, binary.LittleEndian, &i16)
-	return i16
-}
-
-func (ggufModel) readU32(r io.Reader) uint32 {
-	var u32 uint32
-	binary.Read(r, binary.LittleEndian, &u32)
-	return u32
-}
-
-func (ggufModel) readI32(r io.Reader) int32 {
-	var i32 int32
-	binary.Read(r, binary.LittleEndian, &i32)
-	return i32
-}
-
-func (ggufModel) readU64(r io.Reader) uint64 {
-	var u64 uint64
-	binary.Read(r, binary.LittleEndian, &u64)
-	return u64
-}
-
-func (ggufModel) readI64(r io.Reader) int64 {
-	var i64 int64
-	binary.Read(r, binary.LittleEndian, &i64)
-	return i64
-}
-
-func (ggufModel) readF32(r io.Reader) float32 {
-	var f32 float32
-	binary.Read(r, binary.LittleEndian, &f32)
-	return f32
-}
-
-func (ggufModel) readF64(r io.Reader) float64 {
-	var f64 float64
-	binary.Read(r, binary.LittleEndian, &f64)
-	return f64
-}
-
-func (ggufModel) readBool(r io.Reader) bool {
-	var b bool
-	binary.Read(r, binary.LittleEndian, &b)
-	return b
-}
-
-func (ggufModel) readStringV1(r io.Reader) (string, error) {
-	var nameLength uint32
-	binary.Read(r, binary.LittleEndian, &nameLength)
-
-	var b bytes.Buffer
-	if _, err := io.CopyN(&b, r, int64(nameLength)); err != nil {
-		return "", err
-	}
-
-	// gguf v1 strings are null-terminated
-	b.Truncate(b.Len() - 1)
-
-	return b.String(), nil
-}
-
-func (llm ggufModel) readString(r io.Reader) (string, error) {
-	var nameLength uint64
-	binary.Read(r, binary.LittleEndian, &nameLength)
-
-	var b bytes.Buffer
-	if _, err := io.CopyN(&b, r, int64(nameLength)); err != nil {
-		return "", err
-	}
-
-	return b.String(), nil
-}
-
-func (llm *ggufModel) readArrayV1(r io.Reader) (arr []any, err error) {
-	atype := llm.readU32(r)
-	n := llm.readU32(r)
-
-	for i := 0; uint32(i) < n; i++ {
-		switch atype {
-		case ggufTypeUint8:
-			arr = append(arr, llm.readU8(r))
-		case ggufTypeInt8:
-			arr = append(arr, llm.readU8(r))
-		case ggufTypeUint16:
-			arr = append(arr, llm.readU16(r))
-		case ggufTypeInt16:
-			arr = append(arr, llm.readI16(r))
-		case ggufTypeUint32:
-			arr = append(arr, llm.readU32(r))
-		case ggufTypeInt32:
-			arr = append(arr, llm.readI32(r))
-		case ggufTypeFloat32:
-			arr = append(arr, llm.readF32(r))
-		case ggufTypeBool:
-			arr = append(arr, llm.readBool(r))
-		case ggufTypeString:
-			s, err := llm.readStringV1(r)
-			if err != nil {
-				return nil, err
-			}
-
-			arr = append(arr, s)
-		default:
-			return nil, fmt.Errorf("invalid array type: %d", atype)
-		}
-	}
-
-	return
-}
-
-func (llm *ggufModel) readArray(r io.Reader) (arr []any, err error) {
-	atype := llm.readU32(r)
-	n := llm.readU64(r)
-
-	for i := 0; uint64(i) < n; i++ {
-		switch atype {
-		case ggufTypeUint8:
-			arr = append(arr, llm.readU8(r))
-		case ggufTypeInt8:
-			arr = append(arr, llm.readU8(r))
-		case ggufTypeUint16:
-			arr = append(arr, llm.readU16(r))
-		case ggufTypeInt16:
-			arr = append(arr, llm.readI16(r))
-		case ggufTypeUint32:
-			arr = append(arr, llm.readU32(r))
-		case ggufTypeInt32:
-			arr = append(arr, llm.readI32(r))
-		case ggufTypeUint64:
-			arr = append(arr, llm.readU64(r))
-		case ggufTypeInt64:
-			arr = append(arr, llm.readI64(r))
-		case ggufTypeFloat32:
-			arr = append(arr, llm.readF32(r))
-		case ggufTypeFloat64:
-			arr = append(arr, llm.readF64(r))
-		case ggufTypeBool:
-			arr = append(arr, llm.readBool(r))
-		case ggufTypeString:
-			s, err := llm.readString(r)
-			if err != nil {
-				return nil, err
-			}
-
-			arr = append(arr, s)
-		default:
-			return nil, fmt.Errorf("invalid array type: %d", atype)
-		}
-	}
-
-	return
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Jeffrey Morgan	1178fd2cbb	build with cmake	2023-08-21 18:36:31 -07:00
Jeffrey Morgan	97c15b601a	wip shell	2023-08-21 09:24:22 -07:00