diff --git a/core/cli/p2p_worker_hint_test.go b/core/cli/p2p_worker_hint_test.go new file mode 100644 index 0000000000..39c7a269db --- /dev/null +++ b/core/cli/p2p_worker_hint_test.go @@ -0,0 +1,36 @@ +package cli + +import ( + "strings" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" +) + +var _ = Describe("p2p worker hint", func() { + It("names a command the CLI accepts", func() { + // Parse a copy so the package-level CLI is left untouched. The vars + // are the ones cmd/local-ai/main.go supplies; the tags cannot be + // resolved without them. + app := CLI + k, err := kong.New(&app, kong.Vars{ + "basepath": kong.ExpandPath("."), + "generatedcontentpath": DefaultGeneratedContentPath(), + "uploadpath": DefaultUploadPath(), + "galleries": config.DefaultGalleriesJSON, + "backends": config.DefaultBackendGalleriesJSON, + "version": "test", + }) + Expect(err).ToNot(HaveOccurred()) + + args := strings.Fields(workerJoinCommand) + Expect(args[0]).To(Equal("local-ai")) + + ctx, err := k.Parse(args[1:]) + Expect(err).ToNot(HaveOccurred()) + Expect(ctx.Command()).To(Equal("p2p-worker p2p-llama-cpp-rpc")) + }) +}) diff --git a/core/cli/run.go b/core/cli/run.go index 865aad5af6..4ce1c8c4a4 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -32,6 +32,13 @@ import ( // When renaming flags, add the old name as an alias for backward compatibility // and document the deprecation in the help text. +// workerJoinCommand is the command a user runs on another node to join the +// P2P network with a llama.cpp worker. It is printed after `run --p2p` +// generates a token, and a test parses it against the CLI so it cannot drift +// from the command tree again (it named the pre-v4 `worker` command for a +// release cycle). +const workerJoinCommand = "local-ai p2p-worker p2p-llama-cpp-rpc" + type RunCMD struct { AddonHostPolicy string `name:"addon-host-policy" env:"LOCALAI_ADDON_HOST_POLICY" help:"Absolute path to administrator host operation policy JSON, read once at startup. Empty disables environment policy" group:"addons"` AddonHostOps string `name:"addon-host-ops" env:"LOCALAI_ADDON_HOST_OPS" help:"Add-ons that may call host operations, as a comma list of addon-id=operation-prefix. An add-on that is not listed gets no host operation channel. Empty gives none" group:"addons"` @@ -633,7 +640,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) (result error) { fmt.Println(token) xlog.Info("To use the token, you can run the following command in another node or terminal:") - fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token) + fmt.Printf("export TOKEN=\"%s\"\n%s\n", token, workerJoinCommand) } opts = append(opts, config.WithP2PToken(token)) } diff --git a/core/http/react-ui/e2e/add-node.spec.js b/core/http/react-ui/e2e/add-node.spec.js index 23192574b6..3ece7f5388 100644 --- a/core/http/react-ui/e2e/add-node.spec.js +++ b/core/http/react-ui/e2e/add-node.spec.js @@ -111,9 +111,9 @@ test.describe('Add a node', () => { test('a memory shard offers llama.cpp RPC and MLX', async ({ page }) => { await mockSwarm(page, { p2p }) await page.goto('/app/nodes/add?join=shard') - await expect(command(page)).toContainText('worker p2p-llama-cpp-rpc', { timeout: 15_000 }) + await expect(command(page)).toContainText('p2p-worker p2p-llama-cpp-rpc', { timeout: 15_000 }) await page.getByRole('radio', { name: 'MLX (Apple Silicon)' }).click() - await expect(command(page)).toContainText('worker p2p-mlx') + await expect(command(page)).toContainText('p2p-worker p2p-mlx') await expect(command(page)).toContainText('latest-metal-darwin-arm64') // MLX runs on a Mac: GPU build targets do not apply. await expect(page.getByRole('radio', { name: 'CUDA 12', exact: true })).toHaveCount(0) diff --git a/core/http/react-ui/src/utils/swarm.js b/core/http/react-ui/src/utils/swarm.js index d5209ccddf..51ba250ae5 100644 --- a/core/http/react-ui/src/utils/swarm.js +++ b/core/http/react-ui/src/utils/swarm.js @@ -249,8 +249,8 @@ export function workerCommand({ flavor, nodeType, option, dev, frontend, natsUrl export function p2pCommand({ method, option, dev, token, flavor }) { const suffix = method === 'peer' ? 'run --federated --p2p' : method === 'server' ? 'federated' - : method === 'mlx' ? 'worker p2p-mlx' - : 'worker p2p-llama-cpp-rpc' + : method === 'mlx' ? 'p2p-worker p2p-mlx' + : 'p2p-worker p2p-llama-cpp-rpc' const name = method === 'peer' ? 'local-ai' : method === 'server' ? 'local-ai-federated' : method === 'mlx' ? 'local-ai-mlx-worker' : 'local-ai-worker' const tokenValue = token || 'your-token-here' if (flavor === 'cli') return `TOKEN="${tokenValue}" local-ai ${suffix}` diff --git a/core/http/react-ui/src/utils/swarm.test.js b/core/http/react-ui/src/utils/swarm.test.js index 0e266e7b13..b8f85290e8 100644 --- a/core/http/react-ui/src/utils/swarm.test.js +++ b/core/http/react-ui/src/utils/swarm.test.js @@ -140,5 +140,6 @@ test('builds join commands that carry no secret', () => { assert.match(p2pCommand({ method: 'peer', option, dev: false, token: 'abc', flavor: 'docker' }), /run --federated --p2p$/) assert.match(p2pCommand({ method: 'mlx', option, dev: true, token: '', flavor: 'docker' }), /your-token-here/) assert.match(p2pCommand({ method: 'server', option, dev: false, token: 't', flavor: 'docker' }), /--name local-ai-federated[\s\S]*federated$/) - assert.match(p2pCommand({ method: 'shard', option, dev: false, token: 't', flavor: 'cli' }), /^TOKEN="t" local-ai worker p2p-llama-cpp-rpc$/) + assert.match(p2pCommand({ method: 'shard', option, dev: false, token: 't', flavor: 'cli' }), /^TOKEN="t" local-ai p2p-worker p2p-llama-cpp-rpc$/) + assert.match(p2pCommand({ method: 'mlx', option, dev: false, token: 't', flavor: 'cli' }), /^TOKEN="t" local-ai p2p-worker p2p-mlx$/) }) diff --git a/docs/content/features/distributed_inferencing.md b/docs/content/features/distributed_inferencing.md index c5ca7980af..54fde1c2c4 100644 --- a/docs/content/features/distributed_inferencing.md +++ b/docs/content/features/distributed_inferencing.md @@ -119,7 +119,7 @@ To reuse the same token later, restart the server with `--p2ptoken` or `P2P_TOKE 2. Start the workers. Copy the `local-ai` binary to other hosts and run as many workers as needed using the token: ```bash -TOKEN=XXX ./local-ai worker p2p-llama-cpp-rpc --llama-cpp-args="-m " +TOKEN=XXX ./local-ai p2p-worker p2p-llama-cpp-rpc --llama-cpp-args="-m " ``` (Note: You can also supply the token via command-line arguments) diff --git a/docs/content/features/mlx-distributed.md b/docs/content/features/mlx-distributed.md index 4b79b677a6..42553787c3 100644 --- a/docs/content/features/mlx-distributed.md +++ b/docs/content/features/mlx-distributed.md @@ -43,7 +43,7 @@ On each additional Mac: docker run -ti --net host \ -e TOKEN="" \ --name local-ai-mlx-worker \ - localai/localai:latest-metal-darwin-arm64 worker p2p-mlx + localai/localai:latest-metal-darwin-arm64 p2p-worker p2p-mlx ``` Workers auto-register on the P2P network. The LocalAI server discovers them and generates a hostfile for MLX distributed. @@ -192,7 +192,7 @@ Starts a worker or manual rank 0 process. | `--addr` | `MLX_DISTRIBUTED_ADDR` | `localhost:50051` | gRPC API listen address (rank 0 only, for LocalAI or external access) | | `--coordinator` | `MLX_JACCL_COORDINATOR` | | JACCL coordinator `ip:port` - rank 0's address for RDMA setup (all ranks must use the same value) | -### `worker p2p-mlx` +### `p2p-worker p2p-mlx` P2P mode - auto-discovers peers and generates hostfile.