From 753e98bb9f2d2b0a1ff004a005ea130ff84263f3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 5 Oct 2026 16:23:52 +0000 Subject: [PATCH] :bug: Raise default binfile import text total to 500MiB Giant legitimate exports hold hundreds of MB of JSON across hundreds of thousands of KB-sized entries without tripping any per-entry cap, so the 200MiB cumulative default rejected them out of the box. Raise it to 500MiB; the 20MiB per-entry cap still bounds single-entry inflation. Pin the floor with a regression test. AI-assisted-by: muse-spark-1.3-contributor --- backend/src/app/binfile/common.clj | 10 +++++++--- backend/test/backend_tests/binfile_test.clj | 7 +++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 2909e35815..951b225818 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -68,11 +68,15 @@ (* 1024 1024 20)) ;; Maximum total decompressed size allowed for all JSON/text zip entries -;; combined within a single import job: 200MiB. Bounds the case where many +;; combined within a single import job: 500MiB. Bounds the case where many ;; entries, each individually under default-max-text-entry-size, still sum -;; to an unreasonable total. +;; to an unreasonable total. Set well above the 200MiB it replaces because +;; legitimate giant exports exist: hundreds of thousands of KB-sized +;; entries (pages, shapes) that never trip the per-entry cap but sum past +;; 400MiB. The per-entry cap still bounds single-entry DEFLATE +;; amplification, so this only moves the cumulative ceiling. (def ^:const default-max-text-total-size - (* 1024 1024 200)) + (* 1024 1024 500)) ;; Maximum number of entries allowed in the import zip: 500,000. (def ^:const default-max-zip-entries diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 715def2ad2..0ffb1d2262 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -2153,6 +2153,13 @@ (t/is (= :validation (:type out))) (t/is (= :max-file-size-reached (:code out)))))) +(t/deftest default-text-total-budget-fits-giant-legit-files + ;; A real-world giant export held ~437 MiB of JSON across ~178k + ;; KB-sized entries without tripping any per-entry cap. The default + ;; cumulative budget must clear that scale, or such files cannot be + ;; imported out of the box. + (t/is (>= bfc/default-max-text-total-size (* 1024 1024 450)))) + (defn- text-entries-sizes "Returns the decompressed sizes of every `.json` entry in the zip at `zip-path`. Used to pick a cumulative budget that sits between the