diff --git a/pkg/model/process.go b/pkg/model/process.go index 4fd6041e4..fb1ed004a 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -231,6 +231,16 @@ func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string return ml.startProcess(grpcProcess, id, serverAddress, args...) } +// newProcessStateDir creates the directory a backend process uses for its pid, +// state and log files, and reports why when it cannot. +func newProcessStateDir() (string, error) { + dir, err := os.MkdirTemp(os.TempDir(), "go-processmanager") + if err != nil { + return "", fmt.Errorf("creating backend process state directory under %s: %w", os.TempDir(), err) + } + return dir, nil +} + func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) { // Make sure the process is executable // Check first if it has executable permissions @@ -261,8 +271,19 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string // and the GPU would silently fall back to CPU). No-op for other backends. env = append(env, vulkanICDEnv(workDir)...) + // Resolve the state directory here rather than through + // process.WithTemporaryStateDir(). process.New applies its options but + // discards the error they return, so a temp directory that cannot be + // created leaves StateDir empty and every later option unapplied. Run() + // then reported "mkdir : no such file or directory" with no path, hiding + // the real cause (a full volume, or a TMPDIR that no longer resolves). + stateDir, err := newProcessStateDir() + if err != nil { + return nil, err + } + grpcControlProcess := process.New( - process.WithTemporaryStateDir(), + process.WithStateDir(stateDir), process.WithName(filepath.Base(grpcProcess)), process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), process.WithEnvironment(env...), diff --git a/pkg/model/process_statedir_test.go b/pkg/model/process_statedir_test.go new file mode 100644 index 000000000..207c2991d --- /dev/null +++ b/pkg/model/process_statedir_test.go @@ -0,0 +1,38 @@ +package model + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backend process state directory", func() { + It("reports why the state directory could not be created", func() { + // A worker whose volume is full, or whose TMPDIR no longer resolves, + // cannot get a state directory. go-processmanager's New() drops the + // option error, leaving StateDir empty, and Run() then failed with + // "mkdir : no such file or directory" naming no path at all. Resolving + // the directory here keeps the real cause attached. + GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "does-not-exist")) + + dir, err := newProcessStateDir() + Expect(err).To(HaveOccurred()) + Expect(dir).To(BeEmpty()) + Expect(err.Error()).To(ContainSubstring("backend process state directory")) + Expect(err.Error()).To(ContainSubstring("does-not-exist"), + "the error must name the directory it could not create") + }) + + It("returns a usable directory when the temp location works", func() { + GinkgoT().Setenv("TMPDIR", GinkgoT().TempDir()) + + dir, err := newProcessStateDir() + Expect(err).ToNot(HaveOccurred()) + Expect(dir).ToNot(BeEmpty()) + info, statErr := os.Stat(dir) + Expect(statErr).ToNot(HaveOccurred()) + Expect(info.IsDir()).To(BeTrue()) + }) +})