diff --git a/tests/e2e/distributed/cluster/admin.go b/tests/e2e/distributed/cluster/admin.go index b1371f532..617bcef99 100644 --- a/tests/e2e/distributed/cluster/admin.go +++ b/tests/e2e/distributed/cluster/admin.go @@ -14,11 +14,12 @@ import ( ) const ( - // adminPassword must satisfy core/http/auth's policy (>= 12 chars and a - // zxcvbn score of 3 against hints that include "admin" and "localai"). - // This one scores 3 today; acknowledgeWeakPassword is sent alongside it so - // a future tightening of the policy cannot silently break every failover - // spec at setup time. + // adminPassword is sent with "acknowledge_weak_password": true, which sets + // PasswordPolicy{AllowWeak: true} and skips the length floor and the zxcvbn + // score entirely (core/http/auth/password.go). Only the technical + // invariants still apply: non-empty, at most 72 bytes, no NUL. The + // acknowledgement is deliberate rather than incidental, so a future + // tightening of the policy cannot break every failover spec at setup time. adminPassword = "e2e-admin-password" // sessionCookieName mirrors the unexported constant in core/http/auth. // The register handler returns 201 both for "user created, here is your @@ -43,11 +44,18 @@ func ForTestingEmpty() *Cluster { // which core/http/auth exempts from the approval gate and assigns the admin // role, so registration alone yields an active admin session. // -// Call this ONCE per cluster and share the client. Two reasons: the auth -// endpoints are rate limited to 5 requests per minute per client IP, and every -// e2e request arrives from 127.0.0.1; and the returned client is already good -// for every frontend, because sessions live in the shared Postgres auth DB and -// Go's cookie jar keys cookies by host without the port. +// Call this ONCE per cluster and share the client. Two reasons: +// +// One, a single rate limiter of 5 requests per minute per client IP guards +// POST /api/auth/token-login, POST /api/auth/register, POST /api/auth/login AND +// PUT /api/auth/password (core/http/routes/auth.go:190). They share one budget, +// and every e2e request arrives from 127.0.0.1, so a spec that changes a +// password spends from the same five. +// +// Two, the returned client is already good for every frontend: sessions live in +// the shared Postgres auth DB, the harness pins one HMAC secret across replicas +// so the session row resolves at any of them, and Go's cookie jar keys cookies +// by host without the port. func (c *Cluster) AdminSession(i int) (*http.Client, error) { base, err := c.frontendBaseURL(i) if err != nil { diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0445fe386..224018833 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -71,8 +71,11 @@ type Cluster struct { const ( defaultRegistrationToken = "e2e-token" defaultAdminEmail = "admin@e2e.local" - readinessTimeout = 90 * time.Second - readinessPoll = 200 * time.Millisecond + // testHMACSecret is shared by every frontend so a session minted at one + // replica validates at all of them. See the note in startFrontend. + testHMACSecret = "e2e-cluster-hmac-secret" + readinessTimeout = 90 * time.Second + readinessPoll = 200 * time.Millisecond // processExitTimeout bounds the post-SIGKILL wait in terminate. An unbounded // wait turns one stuck child (D state, or a Wait that never returns) into a // suite-wide Ginkgo timeout that names nothing. @@ -157,6 +160,14 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { if err := os.MkdirAll(filepath.Join(dir, "backends"), 0o755); err != nil { return nil, fmt.Errorf("creating %s dirs: %w", name, err) } + // Without an explicit LOCALAI_DATA_PATH every child resolves DataPath to + // ${cwd}/data (core/cli/run.go:48), which under `go test` is inside the + // source tree and shared by every replica: one collectiondb, one task and + // job store for processes that are meant to be independent. + dataPath := filepath.Join(dir, "data") + if err := os.MkdirAll(dataPath, 0o750); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } cmd := exec.Command(c.opts.Binary, "run", "--address", fmt.Sprintf("127.0.0.1:%d", port), @@ -171,6 +182,16 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { "LOCALAI_AUTH=true", "LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN, "LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail, + "LOCALAI_DATA_PATH="+dataPath, + // Session rows are keyed by HMAC-SHA256(token, APIKeyHMACSecret), and + // the secret is generated per instance into {DataPath}/.hmac_secret + // unless pinned (core/application/startup.go:141-148). Now that each + // replica owns its data directory, an unpinned secret would differ per + // replica, so the cookie minted at frontend 0 would hash to a session + // row that does not exist at frontend 1 and every post-failover + // /api/nodes call would 401 with nothing in the logs to explain it. + // Pinning makes the cross-replica session a property of the harness. + "LOCALAI_AUTH_HMAC_SECRET="+testHMACSecret, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, "LOCALAI_AUTO_APPROVE_NODES=true", "DEBUG=true",