test(distributed): give each frontend its own data dir and one pinned secret

Session rows are keyed by an HMAC of the token under a secret generated
per instance into {DataPath}/.hmac_secret. The replicas shared that
secret only because they shared a working directory, and that directory
was the source tree. Give each frontend LOCALAI_DATA_PATH under its own
baseDir and pin LOCALAI_AUTH_HMAC_SECRET, so a session minted at one
replica resolves at every other one by construction.

Assisted-by: Claude Opus 5 [claude-code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

tests/e2e/distributed/cluster/admin.go

@@ -14,11 +14,12 @@ import (
 )
 
 const (
-	// adminPassword must satisfy core/http/auth's policy (>= 12 chars and a
-	// zxcvbn score of 3 against hints that include "admin" and "localai").
-	// This one scores 3 today; acknowledgeWeakPassword is sent alongside it so
-	// a future tightening of the policy cannot silently break every failover
-	// spec at setup time.
+	// adminPassword is sent with "acknowledge_weak_password": true, which sets
+	// PasswordPolicy{AllowWeak: true} and skips the length floor and the zxcvbn
+	// score entirely (core/http/auth/password.go). Only the technical
+	// invariants still apply: non-empty, at most 72 bytes, no NUL. The
+	// acknowledgement is deliberate rather than incidental, so a future
+	// tightening of the policy cannot break every failover spec at setup time.
 	adminPassword = "e2e-admin-password"
 	// sessionCookieName mirrors the unexported constant in core/http/auth.
 	// The register handler returns 201 both for "user created, here is your
@@ -43,11 +44,18 @@ func ForTestingEmpty() *Cluster {
 // which core/http/auth exempts from the approval gate and assigns the admin
 // role, so registration alone yields an active admin session.
 //
-// Call this ONCE per cluster and share the client. Two reasons: the auth
-// endpoints are rate limited to 5 requests per minute per client IP, and every
-// e2e request arrives from 127.0.0.1; and the returned client is already good
-// for every frontend, because sessions live in the shared Postgres auth DB and
-// Go's cookie jar keys cookies by host without the port.
+// Call this ONCE per cluster and share the client. Two reasons:
+//
+// One, a single rate limiter of 5 requests per minute per client IP guards
+// POST /api/auth/token-login, POST /api/auth/register, POST /api/auth/login AND
+// PUT /api/auth/password (core/http/routes/auth.go:190). They share one budget,
+// and every e2e request arrives from 127.0.0.1, so a spec that changes a
+// password spends from the same five.
+//
+// Two, the returned client is already good for every frontend: sessions live in
+// the shared Postgres auth DB, the harness pins one HMAC secret across replicas
+// so the session row resolves at any of them, and Go's cookie jar keys cookies
+// by host without the port.
 func (c *Cluster) AdminSession(i int) (*http.Client, error) {
 	base, err := c.frontendBaseURL(i)
 	if err != nil {

tests/e2e/distributed/cluster/cluster.go

@@ -71,8 +71,11 @@ type Cluster struct {
 const (
 	defaultRegistrationToken = "e2e-token"
 	defaultAdminEmail        = "admin@e2e.local"
-	readinessTimeout         = 90 * time.Second
-	readinessPoll            = 200 * time.Millisecond
+	// testHMACSecret is shared by every frontend so a session minted at one
+	// replica validates at all of them. See the note in startFrontend.
+	testHMACSecret   = "e2e-cluster-hmac-secret"
+	readinessTimeout = 90 * time.Second
+	readinessPoll    = 200 * time.Millisecond
 	// processExitTimeout bounds the post-SIGKILL wait in terminate. An unbounded
 	// wait turns one stuck child (D state, or a Wait that never returns) into a
 	// suite-wide Ginkgo timeout that names nothing.
@@ -157,6 +160,14 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) {
 	if err := os.MkdirAll(filepath.Join(dir, "backends"), 0o755); err != nil {
 		return nil, fmt.Errorf("creating %s dirs: %w", name, err)
 	}
+	// Without an explicit LOCALAI_DATA_PATH every child resolves DataPath to
+	// ${cwd}/data (core/cli/run.go:48), which under `go test` is inside the
+	// source tree and shared by every replica: one collectiondb, one task and
+	// job store for processes that are meant to be independent.
+	dataPath := filepath.Join(dir, "data")
+	if err := os.MkdirAll(dataPath, 0o750); err != nil {
+		return nil, fmt.Errorf("creating %s dirs: %w", name, err)
+	}
 
 	cmd := exec.Command(c.opts.Binary, "run",
 		"--address", fmt.Sprintf("127.0.0.1:%d", port),
@@ -171,6 +182,16 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) {
 		"LOCALAI_AUTH=true",
 		"LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN,
 		"LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail,
+		"LOCALAI_DATA_PATH="+dataPath,
+		// Session rows are keyed by HMAC-SHA256(token, APIKeyHMACSecret), and
+		// the secret is generated per instance into {DataPath}/.hmac_secret
+		// unless pinned (core/application/startup.go:141-148). Now that each
+		// replica owns its data directory, an unpinned secret would differ per
+		// replica, so the cookie minted at frontend 0 would hash to a session
+		// row that does not exist at frontend 1 and every post-failover
+		// /api/nodes call would 401 with nothing in the logs to explain it.
+		// Pinning makes the cross-replica session a property of the harness.
+		"LOCALAI_AUTH_HMAC_SECRET="+testHMACSecret,
 		"LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken,
 		"LOCALAI_AUTO_APPROVE_NODES=true",
 		"DEBUG=true",