diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 05cc9c2f6..b40ddf465 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -22,6 +22,7 @@ The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./ - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`. - Verbose Ginkgo output is written to `coverage/logs/.log`, with the prior run retained as `.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs. + - Suites run in parallel by default and each recursive root invocation has a five-minute budget. Override auto-detected parallelism with `COVERAGE_PROCS`; tune diagnostics with `COVERAGE_SUITE_TIMEOUT` and `COVERAGE_PROGRESS_AFTER`. A timeout is a performance failure to investigate, not a reason to raise the committed default. - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure. - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build. - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise. diff --git a/Makefile b/Makefile index b39926f8c..1974358d1 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,9 @@ COVERAGE_COVERPKG?=github.com/mudler/LocalAI/core/...,github.com/mudler/LocalAI/ ## the coverage CI job doesn't do. COVERAGE_E2E_ROOTS?=./tests/e2e COVERAGE_E2E_LABELS?=!real-models +COVERAGE_PROCS?=0 +COVERAGE_SUITE_TIMEOUT?=5m +COVERAGE_PROGRESS_AFTER?=30s ## Drop generated protobuf from the denominator (it has no tests by design). COVERAGE_EXCLUDE_RE?=grpc/proto/.*[.]pb[.]go @@ -232,6 +235,9 @@ test-coverage: prepare-test COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \ COVERAGE_E2E_ROOTS="$(COVERAGE_E2E_ROOTS)" \ COVERAGE_E2E_LABELS="$(COVERAGE_E2E_LABELS)" \ + COVERAGE_PROCS="$(COVERAGE_PROCS)" \ + COVERAGE_SUITE_TIMEOUT="$(COVERAGE_SUITE_TIMEOUT)" \ + COVERAGE_PROGRESS_AFTER="$(COVERAGE_PROGRESS_AFTER)" \ COVERAGE_EXCLUDE_RE='$(COVERAGE_EXCLUDE_RE)' \ OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \ scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS) diff --git a/core/http/app_test.go b/core/http/app_test.go index 36072b96e..d9adf61d0 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -297,9 +297,7 @@ func getRequest(url string, header http.Header) (error, int, []byte) { return nil, resp.StatusCode, body } -const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml` - -var _ = Describe("API test", func() { +var _ = Describe("API test", Serial, func() { var app *echo.Echo var client *openai.Client @@ -308,6 +306,7 @@ var _ = Describe("API test", func() { var cancel context.CancelFunc var tmpdir string var modelDir string + var bertEmbeddingsURL string // localAIApp captures the Application so AfterEach can synchronously // stop the spawned gRPC backend processes. application.New cancels // them asynchronously on context cancel, which races with test-binary @@ -332,6 +331,17 @@ var _ = Describe("API test", func() { modelDir = filepath.Join(tmpdir, "models") err = os.Mkdir(modelDir, 0750) Expect(err).ToNot(HaveOccurred()) + fixtureDir := filepath.Join(modelDir, ".fixtures") + err = os.Mkdir(fixtureDir, 0750) + Expect(err).ToNot(HaveOccurred()) + galleryFixturePath := filepath.Join(fixtureDir, "bert-embeddings-gallery.yaml") + err = os.WriteFile(galleryFixturePath, []byte("name: bert\nconfig_file: |\n name: bert\n backend: embeddings\n usage: You can test this model with curl like this\n parameters:\n model: bert\n"), 0600) + Expect(err).ToNot(HaveOccurred()) + bertEmbeddingsURL = "file://" + galleryFixturePath + // Additional files are cache inputs, not behavior under test here. Seed the + // destination so model application never reaches the public network. + err = os.WriteFile(filepath.Join(modelDir, "foo.yaml"), []byte("fixture: true\n"), 0600) + Expect(err).ToNot(HaveOccurred()) c, cancel = context.WithCancel(context.Background()) @@ -511,7 +521,7 @@ var _ = Describe("API test", func() { fmt.Println(response) resp = response return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) Expect(resp["message"]).ToNot(ContainSubstring("error")) dat, err := os.ReadFile(filepath.Join(modelDir, "bert2.yaml")) @@ -556,7 +566,7 @@ var _ = Describe("API test", func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) @@ -580,7 +590,7 @@ var _ = Describe("API test", func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) diff --git a/core/http/openresponses_test.go b/core/http/openresponses_test.go index f30674362..ab5dd716a 100644 --- a/core/http/openresponses_test.go +++ b/core/http/openresponses_test.go @@ -28,7 +28,7 @@ import ( // the registered model is "Qwen3-VL-2B-Instruct-Q4_K_M", not the repo name. const testModel = "Qwen3-VL-2B-Instruct-Q4_K_M" -var _ = Describe("Open Responses API", func() { +var _ = Describe("Open Responses API", Serial, func() { var app *echo.Echo var localApp *application.Application var localModelDir string diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index 4f3bb1683..bd3739737 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -75,13 +75,18 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal } var status *galleryop.OpStatus - // wait for op to finish + poll := time.NewTicker(50 * time.Millisecond) + defer poll.Stop() for { status = galleryService.GetStatus(uuid.String()) if status != nil && status.Processed { break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return ctx.Err() + case <-poll.C: + } } if status.Error != nil { diff --git a/core/startup/model_preload_test.go b/core/startup/model_preload_test.go index 525f183cf..ad662a5fa 100644 --- a/core/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -3,6 +3,8 @@ package startup_test import ( "context" "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" @@ -39,7 +41,11 @@ var _ = Describe("Preload test", func() { Context("Preloading from strings", func() { It("loads from embedded full-urls", func() { - url := "https://raw.githubusercontent.com/mudler/LocalAI-examples/main/configurations/phi-2.yaml" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("name: phi-2\nbackend: llama-cpp\nparameters:\n model: phi-2.gguf\n")) + })) + defer server.Close() + url := server.URL + "/phi-2.yaml" fileName := fmt.Sprintf("%s.yaml", "phi-2") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ @@ -59,7 +65,11 @@ var _ = Describe("Preload test", func() { Expect(string(content)).To(ContainSubstring("name: phi-2")) }) It("downloads from urls", func() { - url := "huggingface://TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("tiny local GGUF fixture")) + })) + defer server.Close() + url := server.URL + "/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" fileName := fmt.Sprintf("%s.gguf", "tinyllama-1.1b-chat-v0.3.Q2_K") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ diff --git a/pkg/downloader/cancel_test.go b/pkg/downloader/cancel_test.go index 76f8a2df5..57f9bba95 100644 --- a/pkg/downloader/cancel_test.go +++ b/pkg/downloader/cancel_test.go @@ -59,9 +59,7 @@ var _ = Describe("Download cancellation", func() { } BeforeEach(func() { - dir, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath = dir + "/cancel_model" + filePath = GinkgoT().TempDir() + "/cancel_model" }) AfterEach(func() { @@ -112,7 +110,7 @@ var _ = Describe("Download cancellation", func() { Expect(err).To(HaveOccurred()) Expect(errors.Is(err, context.Canceled)).To(BeTrue()) - Expect(filePath + ".partial").ToNot(BeAnExistingFile(), + Expect(filePath+".partial").ToNot(BeAnExistingFile(), "a deliberate user cancel must not leave a dangling .partial behind") }) diff --git a/pkg/downloader/stall_test.go b/pkg/downloader/stall_test.go index 34ae2d348..7c9f2e5f6 100644 --- a/pkg/downloader/stall_test.go +++ b/pkg/downloader/stall_test.go @@ -18,9 +18,7 @@ var _ = Describe("Download stall timeout", func() { var savedTimeout time.Duration BeforeEach(func() { - dir, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath = dir + "/stall_model" + filePath = GinkgoT().TempDir() + "/stall_model" savedTimeout = DownloadStallTimeout }) diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index 9cb667b57..a9e601cc9 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -263,9 +263,7 @@ var _ = Describe("Download Test", func() { _, err = _mockDataSha.Write(mockData) Expect(err).ToNot(HaveOccurred()) mockDataSha = fmt.Sprintf("%x", _mockDataSha.Sum(nil)) - dir, err := os.Getwd() - filePath = dir + "/my_supercool_model" - Expect(err).NotTo(HaveOccurred()) + filePath = GinkgoT().TempDir() + "/my_supercool_model" }) Context("URI DownloadFile", func() { diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go index 1a8829431..da5b2f037 100644 --- a/pkg/model/loader_test.go +++ b/pkg/model/loader_test.go @@ -65,8 +65,7 @@ var _ = Describe("ModelLoader", func() { BeforeEach(func() { // Setup the model loader with a test directory - modelPath = "/tmp/test_model_path" - os.Mkdir(modelPath, 0755) + modelPath = GinkgoT().TempDir() systemState, err := system.GetSystemState( system.WithModelPath(modelPath), @@ -75,11 +74,6 @@ var _ = Describe("ModelLoader", func() { modelLoader = model.NewModelLoader(systemState) }) - AfterEach(func() { - // Cleanup test directory - os.RemoveAll(modelPath) - }) - Context("NewModelLoader", func() { It("should create a new ModelLoader with an empty model map", func() { Expect(modelLoader).ToNot(BeNil()) diff --git a/pkg/oci/blob.go b/pkg/oci/blob.go index e034c4162..63aa44d5f 100644 --- a/pkg/oci/blob.go +++ b/pkg/oci/blob.go @@ -16,6 +16,10 @@ import ( ) func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer) error { + return fetchImageBlob(ctx, r, reference, dst, statusReader, false) +} + +func fetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer, plainHTTP bool) error { // 0. Create a file store for the output fs, err := os.Create(dst) if err != nil { @@ -29,6 +33,7 @@ func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader return fmt.Errorf("failed to create repository: %v", err) } repo.SkipReferrersGC = true + repo.PlainHTTP = plainHTTP // Identify LocalAI to the registry. This mirrors oras' auth.DefaultClient // (same retry policy) but advertises a LocalAI User-Agent instead of the diff --git a/pkg/oci/blob_test.go b/pkg/oci/blob_test.go index cef29a972..76b7d49b6 100644 --- a/pkg/oci/blob_test.go +++ b/pkg/oci/blob_test.go @@ -1,10 +1,14 @@ -package oci_test +package oci import ( "context" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" "os" + "strings" - . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,11 +16,22 @@ import ( var _ = Describe("OCI", func() { Context("pulling images", func() { It("should fetch blobs correctly", func() { + payload := []byte("local OCI blob fixture") + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + w.Header().Set("Content-Length", fmt.Sprint(len(payload))) + if r.Method != http.MethodHead { + _, _ = w.Write(payload) + } + })) + defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = FetchImageBlob(context.TODO(), "registry.ollama.ai/library/gemma", "sha256:c1864a5eb19305c40519da12cc543519e48a0697ecd30e15d5ac228644957d12", f.Name(), nil) + err = fetchImageBlob(context.Background(), strings.TrimPrefix(server.URL, "http://")+"/library/gemma", digest, f.Name(), nil, true) Expect(err).NotTo(HaveOccurred()) + Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/pkg/oci/ollama.go b/pkg/oci/ollama.go index f0a874013..42bf64bfc 100644 --- a/pkg/oci/ollama.go +++ b/pkg/oci/ollama.go @@ -35,6 +35,10 @@ type LayerDetail struct { } func OllamaModelManifest(image string) (*Manifest, error) { + return ollamaModelManifest("https", "registry.ollama.ai", image) +} + +func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) { // parse the repository and tag from `image`. `image` should be for e.g. gemma:2b, or foobar/gemma:2b // if there is a : in the image, then split it @@ -42,7 +46,7 @@ func OllamaModelManifest(image string) (*Manifest, error) { tag, repository, image := ParseImageParts(image) // get e.g. https://registry.ollama.ai/v2/library/llama3/manifests/latest - req, err := http.NewRequest("GET", "https://registry.ollama.ai/v2/"+repository+"/"+image+"/manifests/"+tag, nil) + req, err := http.NewRequest("GET", scheme+"://"+registry+"/v2/"+repository+"/"+image+"/manifests/"+tag, nil) if err != nil { return nil, err } @@ -65,7 +69,11 @@ func OllamaModelManifest(image string) (*Manifest, error) { } func OllamaModelBlob(image string) (string, error) { - manifest, err := OllamaModelManifest(image) + return ollamaModelBlob("https", "registry.ollama.ai", image) +} + +func ollamaModelBlob(scheme, registry, image string) (string, error) { + manifest, err := ollamaModelManifest(scheme, registry, image) if err != nil { return "", err } @@ -81,12 +89,16 @@ func OllamaModelBlob(image string) (string, error) { } func OllamaFetchModel(ctx context.Context, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { + return ollamaFetchModel(ctx, "https", "registry.ollama.ai", image, output, statusWriter) +} + +func ollamaFetchModel(ctx context.Context, scheme, registry, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { _, repository, imageNoTag := ParseImageParts(image) - blobID, err := OllamaModelBlob(image) + blobID, err := ollamaModelBlob(scheme, registry, image) if err != nil { return err } - return FetchImageBlob(ctx, fmt.Sprintf("registry.ollama.ai/%s/%s", repository, imageNoTag), blobID, output, statusWriter) + return fetchImageBlob(ctx, fmt.Sprintf("%s/%s/%s", registry, repository, imageNoTag), blobID, output, statusWriter, scheme == "http") } diff --git a/pkg/oci/ollama_test.go b/pkg/oci/ollama_test.go index fbda69e6b..bed92a19c 100644 --- a/pkg/oci/ollama_test.go +++ b/pkg/oci/ollama_test.go @@ -1,10 +1,15 @@ -package oci_test +package oci import ( "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "os" + "strings" - . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,11 +17,26 @@ import ( var _ = Describe("OCI", func() { Context("ollama", func() { It("pulls model files", func() { + payload := []byte("local Ollama model fixture") + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + if strings.Contains(r.URL.Path, "/manifests/") { + _ = json.NewEncoder(w).Encode(Manifest{SchemaVersion: 2, Layers: []LayerDetail{{Digest: digest, MediaType: "application/vnd.ollama.image.model", Size: len(payload)}}}) + return + } + w.Header().Set("Content-Length", fmt.Sprint(len(payload))) + if r.Method != http.MethodHead { + _, _ = w.Write(payload) + } + })) + defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = OllamaFetchModel(context.TODO(), "gemma:2b", f.Name(), nil) + err = ollamaFetchModel(context.Background(), "http", strings.TrimPrefix(server.URL, "http://"), "gemma:2b", f.Name(), nil) Expect(err).NotTo(HaveOccurred()) + Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/scripts/run-coverage.sh b/scripts/run-coverage.sh index 88120bb58..debbade12 100755 --- a/scripts/run-coverage.sh +++ b/scripts/run-coverage.sh @@ -21,6 +21,9 @@ # "!real-models" (those specs need a downloaded model). # COVERAGE_EXCLUDE_RE egrep pattern of profile lines to drop before merging, # e.g. generated protobuf (grpc/proto/.*\.pb\.go). +# COVERAGE_PROCS parallel Ginkgo processes; 0 lets Ginkgo detect CPUs. +# COVERAGE_SUITE_TIMEOUT maximum duration of each recursive root (default 5m). +# COVERAGE_PROGRESS_AFTER emit diagnostics when a spec is slow (default 30s). # # Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log # for each root is kept with a .previous suffix, so a noisy failure remains @@ -66,6 +69,14 @@ rm -f "$out_dir"/cover-*.out rm -f "$merged" fail=0 +procs="${COVERAGE_PROCS:-0}" +suite_timeout="${COVERAGE_SUITE_TIMEOUT:-5m}" +progress_after="${COVERAGE_PROGRESS_AFTER:-30s}" +parallel_flags="-p --keep-going --timeout=$suite_timeout --poll-progress-after=$progress_after --poll-progress-interval=10s" +if [ "$procs" -gt 0 ] 2>/dev/null; then + parallel_flags="$parallel_flags --procs=$procs --compilers=$procs" +fi + # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a # --tags value that contains a space. The unit roots were captured above, so # overwriting the positional parameters here is safe. @@ -112,7 +123,9 @@ for root in $unit_roots; do log="$log_dir/$(log_name "$root")" rotate_log "$log" echo "run-coverage: testing $root (full output: $log)" - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \ + # parallel_flags is intentionally word-split: it contains CLI arguments only. + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v -r "$@" \ --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ && echo "run-coverage: PASS — $root" \ || { fail=1; report_failure "$root" "$log"; } @@ -125,13 +138,15 @@ for root in ${COVERAGE_E2E_ROOTS:-}; do rotate_log "$log" echo "run-coverage: testing $root (full output: $log)" if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v "$@" \ --label-filter="$COVERAGE_E2E_LABELS" \ --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ && echo "run-coverage: PASS — $root" \ || { fail=1; report_failure "$root" "$log"; } else - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v "$@" \ --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ && echo "run-coverage: PASS — $root" \ || { fail=1; report_failure "$root" "$log"; }