🐛 Add migration fixing files with shape-ref cycles (#5663)

* 🐛 Add migration fixing files with shape-ref cycles

* :wip: Add optimized version of migration 62

---------

Co-authored-by: Andrey Antukh <niwi@niwi.nz>

CHANGES.md

@@ -1,5 +1,14 @@
 # CHANGELOG
 
+## 2.4.3 (Unreleased)
+
+### :bug: Bugs fixed
+
+- Fix errors from editable select on measures menu [Taiga #9888](https://tree.taiga.io/project/penpot/issue/9888)
+- Fix exception on importing some templates from templates slider
+- Consolidate adding share button to workspace
+
+
 ## 2.4.2
 
 ### :bug: Bugs fixed
@@ -28,12 +37,12 @@
   (penpot). Because of that, the default NGINX listen port is now 8080 instead of 80, so
   you will have to modify your infrastructure to apply this change.
 
-- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because, 
-  starting with the next versions, Redis is no longer distributed under an open-source license. 
-  On-premise users are obviously free to upgrade to the version they are using or a more modern one. 
-  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume 
-  associated with the Redis container because the 7.2 storage format may not be compatible with what 
-  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate 
+- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because,
+  starting with the next versions, Redis is no longer distributed under an open-source license.
+  On-premise users are obviously free to upgrade to the version they are using or a more modern one.
+  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume
+  associated with the Redis container because the 7.2 storage format may not be compatible with what
+  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate
   whether to move to an open-source version of Redis (such as https://valkey.io/).
 
 ### :heart: Community contributions (Thank you!)

backend/resources/app/templates/debug.tmpl

@@ -114,37 +114,13 @@ Debug Main Page
     </fieldset>
     <fieldset>
       <legend>Import binfile:</legend>
-      <desc>Import penpot file in binary
-      format. If <strong>overwrite</strong> is checked, all files will
-      be overwritten using the same ids found in the file instead of
-      generating a new ones.</desc>
+      <desc>Import penpot file in binary format.</desc>
 
       <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
         <div class="row">
           <input type="file" name="file" value="" />
         </div>
 
-        <div class="row">
-          <label>Overwrite?</label>
-          <input type="checkbox" name="overwrite" />
-          <br />
-          <small>
-            Instead of creating a new file with all relations remapped,
-            reuses all ids and updates/overwrites the objects that are
-            already exists on the database.
-            <strong>Warning, this operation should be used with caution.</strong>
-          </small>
-        </div>
-
-        <div class="row">
-          <label>Migrate?</label>
-          <input type="checkbox" name="migrate" />
-          <br />
-          <small>
-            Applies the file migrations on the importation process.
-          </small>
-        </div>
-
         <div class="row">
           <input type="submit" name="upload" value="Upload" />
         </div>

backend/src/app/binfile/common.clj

@@ -30,7 +30,9 @@
    [app.worker :as-alias wrk]
    [clojure.set :as set]
    [clojure.walk :as walk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [datoteka.fs :as fs]
+   [datoteka.io :as io]))
 
 (set! *warn-on-reflection* true)
 
@@ -52,6 +54,20 @@
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn parse-file-format
+  [template]
+  (assert (fs/path? template) "expected InputStream for `template`")
+
+  (with-open [^java.lang.AutoCloseable input (io/input-stream template)]
+    (let [buffer (byte-array 4)]
+      (io/read-to-buffer input buffer)
+      (if (and (= (aget buffer 0) 80)
+               (= (aget buffer 1) 75)
+               (= (aget buffer 2) 3)
+               (= (aget buffer 3) 4))
+        :binfile-v3
+        :binfile-v1))))
+
 (def xf-map-id
   (map :id))
 

backend/src/app/binfile/v1.clj

@@ -298,7 +298,7 @@
 (defmulti write-section ::section)
 
 (defn write-export!
-  [{:keys [::include-libraries ::embed-assets] :as cfg}]
+  [{:keys [::bfc/include-libraries ::bfc/embed-assets] :as cfg}]
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
@@ -323,7 +323,7 @@
             [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
 
 (defmethod write-section :v1/metadata
-  [{:keys [::output ::ids ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/ids ::bfc/include-libraries] :as cfg}]
   (if-let [fids (get-files cfg ids)]
     (let [lids (when include-libraries
                  (bfc/get-libraries cfg ids))
@@ -335,7 +335,7 @@
               :hint "unable to retrieve files for export")))
 
 (defmethod write-section :v1/files
-  [{:keys [::output ::embed-assets ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/embed-assets ::bfc/include-libraries] :as cfg}]
 
   ;; Initialize SIDS with empty vector
   (vswap! bfc/*state* assoc :sids [])
@@ -382,7 +382,7 @@
       (vswap! bfc/*state* update :sids into bfc/xf-map-media-id thumbnails))))
 
 (defmethod write-section :v1/rels
-  [{:keys [::output ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/include-libraries] :as cfg}]
   (let [ids  (-> bfc/*state* deref :files set)
         rels (when include-libraries
                (bfc/get-files-rels cfg ids))]
@@ -421,15 +421,15 @@
 (defmulti read-import ::version)
 (defmulti read-section ::section)
 
-(s/def ::profile-id ::us/uuid)
-(s/def ::project-id ::us/uuid)
-(s/def ::input io/input-stream?)
+(s/def ::bfc/profile-id ::us/uuid)
+(s/def ::bfc/project-id ::us/uuid)
+(s/def ::bfc/input io/input-stream?)
 (s/def ::overwrite? (s/nilable ::us/boolean))
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
 
 ;; FIXME: replace with schema
 (s/def ::read-import-options
-  (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
+  (s/keys :req [::db/pool ::sto/storage ::bfc/project-id ::bfc/profile-id ::bfc/input]
           :opt [::overwrite? ::ignore-index-errors?]))
 
 (defn read-import!
@@ -439,7 +439,7 @@
 
   `::bfc/overwrite`: if true, instead of creating new files and remapping id references,
   it reuses all ids and updates existing objects; defaults to `false`."
-  [{:keys [::input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
+  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
 
   (dm/assert!
    "expected input stream"
@@ -453,7 +453,7 @@
     (read-import (assoc options ::version version ::bfc/timestamp timestamp))))
 
 (defn- read-import-v1
-  [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}]
+  [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}]
 
   (bfc/disable-database-timeouts! cfg)
 
@@ -473,7 +473,7 @@
                 (let [options (-> cfg
                                   (assoc ::bfc/features features)
                                   (assoc ::section section)
-                                  (assoc ::input input))]
+                                  (assoc ::bfc/input input))]
                   (binding [bfc/*options* options]
                     (events/tap :progress {:op :import :section section})
                     (read-section options))))
@@ -491,7 +491,7 @@
   (db/tx-run! options read-import-v1))
 
 (defmethod read-section :v1/metadata
-  [{:keys [::input]}]
+  [{:keys [::bfc/input]}]
   (let [{:keys [version files]} (read-obj! input)]
     (l/dbg :hint "metadata readed"
            :version (:full version)
@@ -509,7 +509,7 @@
         thumbnails))
 
 (defmethod read-section :v1/files
-  [{:keys [::db/conn ::input ::project-id ::bfc/overwrite ::name] :as system}]
+  [{:keys [::db/conn ::bfc/input ::bfc/project-id ::bfc/overwrite ::bfc/name] :as system}]
 
   (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))]
     (let [file       (read-obj! input)
@@ -576,7 +576,7 @@
         file-id'))))
 
 (defmethod read-section :v1/rels
-  [{:keys [::db/conn ::input ::bfc/timestamp]}]
+  [{:keys [::db/conn ::bfc/input ::bfc/timestamp]}]
   (let [rels (read-obj! input)
         ids  (into #{} (-> bfc/*state* deref :files))]
     ;; Insert all file relations
@@ -600,7 +600,7 @@
                   ::l/sync? true))))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::db/conn ::input ::bfc/overwrite ::bfc/timestamp] :as cfg}]
+  [{:keys [::db/conn ::bfc/input ::bfc/overwrite ::bfc/timestamp] :as cfg}]
   (let [storage (sto/resolve cfg)
         ids     (read-obj! input)
         thumb?  (into #{} (map :media-id) (:thumbnails @bfc/*state*))]
@@ -674,17 +674,17 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
   (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
+   "expected a set of uuid's for `::bfc/ids` parameter"
    (and (set? ids)
         (every? uuid? ids)))
 
@@ -719,12 +719,12 @@
                 :cause @cs)))))
 
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg))))
 
   (dm/assert!
    "expected instance of jio/IOFactory for `input`"
@@ -738,7 +738,7 @@
     (try
       (binding [*position* (atom 0)]
         (pu/with-open [input (io/input-stream input)]
-          (read-import! (assoc cfg ::input input))))
+          (read-import! (assoc cfg ::bfc/input input))))
 
       (catch ZstdIOException cause
         (ex/raise :type :validation

backend/src/app/binfile/v3.clj

@@ -192,7 +192,7 @@
   (.closeEntry output))
 
 (defn- get-file
-  [{:keys [::embed-assets ::include-libraries] :as cfg} file-id]
+  [{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id]
 
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
@@ -330,7 +330,7 @@
         (write-entry! output path color)))))
 
 (defn- export-files
-  [{:keys [::ids ::include-libraries ::output] :as cfg}]
+  [{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}]
   (let [ids  (into ids (when include-libraries (bfc/get-libraries cfg ids)))
         rels (if include-libraries
                (->> (bfc/get-files-rels cfg ids)
@@ -509,7 +509,7 @@
     (json/read reader :key-fn json/read-kebab-key)))
 
 (defn- read-file
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
@@ -517,7 +517,7 @@
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -526,7 +526,7 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -540,7 +540,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -553,7 +553,7 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-component-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -566,7 +566,7 @@
        (not-empty)))
 
 (defn- read-file-typographies
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -579,7 +579,7 @@
        (not-empty)))
 
 (defn- read-file-shapes
-  [{:keys [::input ::file-id ::page-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -592,7 +592,7 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
@@ -608,7 +608,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -638,7 +638,7 @@
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::db/conn ::project-id ::file-id ::file-name] :as cfg}]
+  [{:keys [::db/conn ::bfc/project-id ::file-id ::file-name] :as cfg}]
   (let [file-id'   (bfc/lookup-index file-id)
         file       (read-file cfg)
         media      (read-file-media cfg)
@@ -714,7 +714,7 @@
                      :library-file-id libr-id})))))
 
 (defn- import-storage-objects
-  [{:keys [::input ::entries ::bfc/timestamp] :as cfg}]
+  [{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}]
   (events/tap :progress {:section :storage-objects})
 
   (let [storage (sto/resolve cfg)
@@ -810,7 +810,7 @@
                   {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::input ::name] :or {timestamp (dt/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
 
   (dm/assert!
    "expected zip file"
@@ -878,17 +878,17 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
   (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
+   "expected a set of uuid's for `::bfc/ids` parameter"
    (and (set? ids)
         (every? uuid? ids)))
 
@@ -930,14 +930,13 @@
                 :aborted @ab
                 :cause @cs)))))
 
-
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg))))
 
   (dm/assert!
    "expected instance of jio/IOFactory for `input`"
@@ -950,7 +949,7 @@
     (l/info :hint "import: started" :id (str id))
     (try
       (with-open [input (ZipFile. (fs/file input))]
-        (import-files (assoc cfg ::input input)))
+        (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
         (vreset! cs cause)

backend/src/app/http/debug.clj

@@ -7,7 +7,9 @@
 (ns app.http.debug
   (:refer-clojure :exclude [error-handler])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -280,23 +282,23 @@
       (ex/raise :type :validation
                 :code :missing-arguments))
 
-    (let [path (tmp/tempfile :prefix "penpot.export.")]
+    (let [path (tmp/tempfile :prefix "penpot.export." :min-age "30m")]
       (with-open [output (io/output-stream path)]
         (-> cfg
-            (assoc ::bf.v1/ids file-ids)
-            (assoc ::bf.v1/embed-assets embed?)
-            (assoc ::bf.v1/include-libraries libs?)
-            (bf.v1/export-files! output)))
+            (assoc ::bfc/ids file-ids)
+            (assoc ::bfc/embed-assets embed?)
+            (assoc ::bfc/include-libraries libs?)
+            (bf.v3/export-files! output)))
 
       (if clone?
         (let [profile    (profile/get-profile pool profile-id)
               project-id (:default-project-id profile)
               cfg        (assoc cfg
-                                ::bf.v1/overwrite false
-                                ::bf.v1/profile-id profile-id
-                                ::bf.v1/project-id project-id
-                                ::bf.v1/input path)]
-          (bf.v1/import-files! cfg)
+                                ::bfc/overwrite false
+                                ::bfc/profile-id profile-id
+                                ::bfc/project-id project-id
+                                ::bfc/input path)]
+          (bf.v3/import-files! cfg)
           {::yres/status  200
            ::yres/headers {"content-type" "text/plain"}
            ::yres/body    "OK CLONED"})
@@ -315,23 +317,24 @@
               :hint "missing upload file"))
 
   (let [profile    (profile/get-profile pool profile-id)
-        project-id (:default-project-id profile)
-        overwrite? (contains? params :overwrite)
-        migrate?   (contains? params :migrate)]
+        project-id (:default-project-id profile)]
 
     (when-not project-id
       (ex/raise :type :validation
                 :code :missing-project
                 :hint "project not found"))
 
-    (let [path (-> params :file :path)
-          cfg  (assoc cfg
-                      ::bf.v1/overwrite overwrite?
-                      ::bf.v1/migrate migrate?
-                      ::bf.v1/profile-id profile-id
-                      ::bf.v1/project-id project-id
-                      ::bf.v1/input path)]
-      (bf.v1/import-files! cfg)
+    (let [path   (-> params :file :path)
+          format (bfc/parse-file-format path)
+          cfg    (assoc cfg
+                        ::bfc/profile-id profile-id
+                        ::bfc/project-id project-id
+                        ::bfc/input path)]
+
+      (if (= format :binfile-v3)
+        (bf.v3/import-files! cfg)
+        (bf.v1/import-files! cfg))
+
       {::yres/status  200
        ::yres/headers {"content-type" "text/plain"}
        ::yres/body    "OK"})))

backend/src/app/http/sse.clj

@@ -64,7 +64,8 @@
                           (catch Throwable cause
                             (events/tap :error (errors/handle' cause request))
                             (when-not (ex/instance? java.io.EOFException cause)
-                              (l/err :hint "unexpected error on processing sse response" :cause cause)))
+                              (binding [l/*context* (errors/request->context request)]
+                                (l/err :hint "unexpected error on processing sse response" :cause cause))))
                           (finally
                             (sp/close! events/*channel*)
                             (px/await! listener)))))))}))

backend/src/app/rpc/commands/binfile.clj

@@ -7,6 +7,7 @@
 (ns app.rpc.commands.binfile
   (:refer-clojure :exclude [assert])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
    [app.common.logging :as l]
@@ -46,9 +47,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v1/ids #{file-id})
-           (assoc ::bf.v1/embed-assets embed-assets)
-           (assoc ::bf.v1/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v1/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -61,9 +62,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v3/ids #{file-id})
-           (assoc ::bf.v3/embed-assets embed-assets)
-           (assoc ::bf.v3/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v3/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -93,10 +94,10 @@
 (defn- import-binfile-v1
   [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
   (let [cfg (-> cfg
-                (assoc ::bf.v1/project-id project-id)
-                (assoc ::bf.v1/profile-id profile-id)
-                (assoc ::bf.v1/name name)
-                (assoc ::bf.v1/input (:path file)))]
+                (assoc ::bfc/project-id project-id)
+                (assoc ::bfc/profile-id profile-id)
+                (assoc ::bfc/name name)
+                (assoc ::bfc/input (:path file)))]
 
     ;; NOTE: the importation process performs some operations that are
     ;; not very friendly with virtual threads, and for avoid
@@ -107,10 +108,10 @@
 (defn- import-binfile-v3
   [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
   (let [cfg (-> cfg
-                (assoc ::bf.v3/project-id project-id)
-                (assoc ::bf.v3/profile-id profile-id)
-                (assoc ::bf.v3/name name)
-                (assoc ::bf.v3/input (:path file)))]
+                (assoc ::bfc/project-id project-id)
+                (assoc ::bfc/profile-id profile-id)
+                (assoc ::bfc/name name)
+                (assoc ::bfc/input (:path file)))]
     ;; NOTE: the importation process performs some operations that are
     ;; not very friendly with virtual threads, and for avoid
     ;; unexpected blocking of other concurrent operations we dispatch

backend/src/app/rpc/commands/management.clj

@@ -9,6 +9,7 @@
   (:require
    [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
@@ -25,6 +26,7 @@
    [app.rpc.doc :as-alias doc]
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
+   [app.storage.tmp :as tmp]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
@@ -400,11 +402,20 @@
                     ;; that are not very friendly with virtual threads, and for
                     ;; avoid unexpected blocking of other concurrent operations
                     ;; we dispatch that operation to a dedicated executor.
-                    (let [cfg    (-> cfg
-                                     (assoc ::bf.v1/project-id project-id)
-                                     (assoc ::bf.v1/profile-id profile-id)
-                                     (assoc ::bf.v1/input template))
-                          result (px/invoke! executor (partial bf.v1/import-files! cfg))]
+                    (let [template (tmp/tempfile-from template
+                                                      :prefix "penpot.template."
+                                                      :suffix ""
+                                                      :min-age "30m")
+                          format   (bfc/parse-file-format template)
+
+                          cfg      (-> cfg
+                                       (assoc ::bfc/project-id project-id)
+                                       (assoc ::bfc/profile-id profile-id)
+                                       (assoc ::bfc/input template))
+
+                          result   (if (= format :binfile-v3)
+                                     (px/invoke! executor (partial bf.v3/import-files! cfg))
+                                     (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
                       (db/update! conn :project
                                   {:modified-at (dt/now)}

backend/src/app/setup/templates.clj

@@ -54,6 +54,7 @@
                               (::setup/templates cfg))]
     (let [dest (fs/join fs/*cwd* "builtin-templates")
           path (or (:path template) (fs/join dest template-id))]
+
       (if (fs/exists? path)
         (io/input-stream path)
         (let [resp (http/req! cfg

backend/src/app/storage/tmp.clj

@@ -16,10 +16,13 @@
    [app.util.time :as dt]
    [app.worker :as wrk]
    [datoteka.fs :as fs]
+   [datoteka.io :as io]
    [integrant.core :as ig]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp])
   (:import
+   java.io.InputStream
+   java.io.OutputStream
    java.nio.file.Files))
 
 (def default-tmp-dir "/tmp/penpot")
@@ -86,3 +89,12 @@
     (fs/delete-on-exit! path)
     (sp/offer! queue [path (some-> min-age dt/duration)])
     path))
+
+(defn tempfile-from
+  "Create a new tempfile from from consuming the stream"
+  [input & {:as options}]
+  (let [path (tempfile options)]
+    (with-open [^InputStream input (io/input-stream input)]
+      (with-open [^OutputStream output (io/output-stream path)]
+        (io/copy input output)))
+    path))

backend/test/backend_tests/binfile_test.clj

@@ -7,6 +7,7 @@
 (ns backend-tests.binfile-test
   "Internal binfile test, no RPC involved"
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v3 :as v3]
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
@@ -93,15 +94,15 @@
 
     (v3/export-files!
      (-> th/*system*
-         (assoc ::v3/ids #{(:id file)})
-         (assoc ::v3/embed-assets false)
-         (assoc ::v3/include-libraries false))
+         (assoc ::bfc/ids #{(:id file)})
+         (assoc ::bfc/embed-assets false)
+         (assoc ::bfc/include-libraries false))
      (io/output-stream output))
 
     (let [result (-> th/*system*
-                     (assoc ::v3/project-id (:default-project-id profile))
-                     (assoc ::v3/profile-id (:id profile))
-                     (assoc ::v3/input output)
+                     (assoc ::bfc/project-id (:default-project-id profile))
+                     (assoc ::bfc/profile-id (:id profile))
+                     (assoc ::bfc/input output)
                      (v3/import-files!))]
       (t/is (= (count result) 1))
       (t/is (every? uuid? result)))))

common/src/app/common/files/defaults.cljc

@@ -6,4 +6,4 @@
 
 (ns app.common.files.defaults)
 
-(def version 59)
+(def version 62)

common/src/app/common/files/migrations.cljc

@@ -25,6 +25,7 @@
    [app.common.text :as txt]
    [app.common.types.color :as ctc]
    [app.common.types.component :as ctk]
+   [app.common.types.container :as ctn]
    [app.common.types.file :as ctf]
    [app.common.types.shape :as cts]
    [app.common.types.shape.shadow :as ctss]
@@ -1145,6 +1146,39 @@
         (update :pages-index update-vals update-container)
         (update :components update-vals update-container))))
 
+(defn migrate-up-62
+  [data]
+  (let [xform-cycles-ids
+        (comp (filter #(= (:id %) (:shape-ref %)))
+              (map :id))
+
+        remove-cycles
+        (fn [objects]
+          (let [cycles-ids (into #{} xform-cycles-ids (vals objects))
+                to-detach  (->> cycles-ids
+                                (map #(get objects %))
+                                (map #(ctn/get-head-shape objects %))
+                                (map :id)
+                                distinct
+                                (mapcat #(ctn/get-children-in-instance objects %))
+                                (map :id)
+                                set)]
+
+            (reduce-kv (fn [objects id shape]
+                         (if (contains? to-detach id)
+                           (assoc objects id (ctk/detach-shape shape))
+                           objects))
+                       objects
+                       objects)))
+
+        update-component
+        (fn [component]
+          ;; we only have encounter this on deleted components,
+          ;; so the relevant objects are inside the component
+          (d/update-when component :objects remove-cycles))]
+
+    (update data :components update-vals update-component)))
+
 (def migrations
   "A vector of all applicable migrations"
   [{:id 2 :migrate-up migrate-up-2}
@@ -1194,5 +1228,5 @@
    {:id 55 :migrate-up migrate-up-55}
    {:id 56 :migrate-up migrate-up-56}
    {:id 57 :migrate-up migrate-up-57}
-   {:id 59 :migrate-up migrate-up-59}])
-
+   {:id 59 :migrate-up migrate-up-59}
+   {:id 62 :migrate-up migrate-up-62}])

docs/plugins/create-a-plugin.md

@@ -7,6 +7,14 @@ title: 2. Create a Plugin
 
 This guide covers the creation of a Penpot plugin. Penpot offers two ways to kickstart your development:
 
+<p class="advice">
+Have you got an idea for a new plugin? Great! But first take a look at <a
+href="https://penpot.app/penpothub/plugins">the plugin overview</a> to see if already
+exists, and consider joining efforts with other developers. This does not imply that we
+won't accept plugins that do similar things, since anything can be improved and done in
+different ways.
+</p>
+
 1. Using a Template:
 
    - **Typescript template**: Using the <a target="_blank" href="https://github.com/penpot/penpot-plugin-starter-template">Penpot Plugin Starter Template</a>: A basic template with the required files for quickstarting your plugin. This template uses Typescript and Vite.

docs/technical-guide/configuration.md

@@ -405,6 +405,14 @@ where users will access the application:
 PENPOT_PUBLIC_URI: http://localhost:9001
 ```
 
+<p class="advice">
+    If you plan to serve Penpot under different domain than `localhost` without HTTPS,
+    you need to disable the `secure` flag on cookies, with the `disable-secure-session-cookies` flag.
+    This is a configuration NOT recommended for production environments.
+</p>
+
+Check all the [flags](#other-flags) to fully customize your instance.
+
 ## Frontend ##
 
 In comparison with backend, frontend only has a small number of runtime configuration
@@ -424,8 +432,8 @@ To connect the frontend to the exporter and backend, you need to fill out these
 
 ```bash
 # Frontend
-PENPOT_BACKEND_URI: http://your-penpot-backend
-PENPOT_EXPORTER_URI: http://your-penpot-exporter
+PENPOT_BACKEND_URI: http://your-penpot-backend:6060
+PENPOT_EXPORTER_URI: http://your-penpot-exporter:6061
 ```
 
 These variables are used for generate correct nginx.conf file on container startup.
@@ -480,3 +488,4 @@ __Since version 2.0.0__
 [2]: /technical-guide/getting-started#configure-penpot-with-docker
 [3]: /technical-guide/developer/common#dev-environment
 [4]: https://github.com/penpot/penpot/blob/main/docker/images/files/nginx.conf
+

docs/technical-guide/getting-started.md

@@ -227,6 +227,9 @@ docker compose -f docker-compose.yaml pull
 
 This will fetch the latest images. When you do <code class="language-bash">docker compose up</code> again, the containers will be recreated with the latest version.
 
+<p class="advice">
+    It is strongly recommended to update the Penpot version in small increments, rather than updating between two distant versions.
+</p>
 
 **Important: Upgrade from version 1.x to 2.0**
 

frontend/src/app/main/ui/workspace/right_header.cljs

@@ -7,7 +7,6 @@
 (ns app.main.ui.workspace.right-header
   (:require-macros [app.main.style :as stl])
   (:require
-   [app.config :as cf]
    [app.main.data.events :as ev]
    [app.main.data.modal :as modal]
    [app.main.data.shortcuts :as scd]
@@ -250,9 +249,7 @@
           :on-click toggle-history}
          i/history]])
 
-     (when (and
-            (not (:is-default team))
-            (cf/external-feature-flag "share-01" "test"))
+     (when  (not (:is-default team))
        [:a {:class (stl/css :viewer-btn)
             :title (tr "workspace.header.share")
             :on-click open-share-dialog}

frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.cljs

@@ -13,9 +13,7 @@
    [app.common.types.shape.impl :as shape.impl]
    [app.common.types.shape.layout :as ctl]
    [app.common.types.shape.radius :as ctsr]
-   [app.common.types.tokens-lib :as ctob]
    [app.main.constants :refer [size-presets]]
-   [app.main.data.tokens :as dt]
    [app.main.data.workspace :as udw]
    [app.main.data.workspace.interactions :as dwi]
    [app.main.data.workspace.shapes :as dwsh]
@@ -25,13 +23,8 @@
    [app.main.ui.components.dropdown :refer [dropdown]]
    [app.main.ui.components.numeric-input :refer [numeric-input*]]
    [app.main.ui.components.radio-buttons :refer [radio-button radio-buttons]]
-   [app.main.ui.context :as muc]
    [app.main.ui.hooks :as hooks]
    [app.main.ui.icons :as i]
-   [app.main.ui.workspace.tokens.core :as wtc]
-   [app.main.ui.workspace.tokens.editable-select :refer [editable-select]]
-   [app.main.ui.workspace.tokens.style-dictionary :as sd]
-   [app.main.ui.workspace.tokens.token-types :as wtty]
    [app.util.dom :as dom]
    [app.util.i18n :as i18n :refer [tr]]
    [clojure.set :refer [rename-keys union]]
@@ -92,8 +85,6 @@
                   (reduce #(union %1 %2) (map #(get type->options %) all-types))
                   (get type->options type))
 
-        design-tokens? (mf/use-ctx muc/design-tokens)
-
         ids-with-children (or ids-with-children ids)
 
         old-shapes (if (= type :multiple)
@@ -106,34 +97,6 @@
         selection-parents-ref (mf/use-memo (mf/deps ids) #(refs/parents-by-ids ids))
         selection-parents     (mf/deref selection-parents-ref)
 
-        tokens (sd/use-active-theme-sets-tokens)
-        tokens-by-type (mf/use-memo
-                        (mf/deps tokens)
-                        #(ctob/group-by-type tokens))
-
-        border-radius-tokens (:border-radius tokens-by-type)
-        border-radius-options (mf/use-memo
-                               (mf/deps shape border-radius-tokens)
-                               #(wtc/tokens->select-options
-                                 {:shape shape
-                                  :tokens border-radius-tokens
-                                  :attributes (wtty/token-attributes :border-radius)}))
-        sizing-tokens (:sizing tokens-by-type)
-        width-options (mf/use-memo
-                       (mf/deps shape sizing-tokens)
-                       #(wtc/tokens->select-options
-                         {:shape shape
-                          :tokens sizing-tokens
-                          :attributes (wtty/token-attributes :sizing)
-                          :selected-attributes #{:width}}))
-        height-options (mf/use-memo
-                        (mf/deps shape sizing-tokens)
-                        #(wtc/tokens->select-options
-                          {:shape shape
-                           :tokens sizing-tokens
-                           :attributes (wtty/token-attributes :sizing)
-                           :selected-attributes #{:height}}))
-
         flex-child?       (->> selection-parents (some ctl/flex-layout?))
         absolute?         (ctl/item-absolute? shape)
         flex-container?   (ctl/flex-layout? shape)
@@ -247,22 +210,9 @@
         (mf/use-fn
          (mf/deps ids)
          (fn [value attr]
-           (let [token-value (wtc/maybe-resolve-token-value value)
-                 undo-id (js/Symbol)]
-             (binding [shape.impl/*wasm-sync* true]
-               (if-not design-tokens?
-                 (st/emit! (udw/trigger-bounding-box-cloaking ids)
-                           (udw/update-dimensions ids attr (or token-value value)))
-                 (st/emit! (udw/trigger-bounding-box-cloaking ids)
-                           (dwu/start-undo-transaction undo-id)
-                           (dwsh/update-shapes ids
-                                               (if token-value
-                                                 #(assoc-in % [:applied-tokens attr] (:id value))
-                                                 #(d/dissoc-in % [:applied-tokens attr]))
-                                               {:reg-objects? true
-                                                :attrs [:applied-tokens]})
-                           (udw/update-dimensions ids attr (or token-value value))
-                           (dwu/commit-undo-transaction undo-id)))))))
+           (binding [shape.impl/*wasm-sync* true]
+             (st/emit! (udw/trigger-bounding-box-cloaking ids)
+                       (udw/update-dimensions ids attr value)))))
 
         on-proportion-lock-change
         (mf/use-fn
@@ -335,27 +285,13 @@
              (on-switch-to-radius-4)
              (on-switch-to-radius-1))))
 
-        on-border-radius-token-unapply
-        (mf/use-fn
-         (mf/deps ids change-radius)
-         (fn [token]
-           (let [token-value (wtc/maybe-resolve-token-value token)]
-             (st/emit!
-              (change-radius (fn [shape]
-                               (-> (dt/unapply-token-id shape (wtty/token-attributes :border-radius))
-                                   (ctsr/set-radius-1 token-value))))))))
-
         on-radius-1-change
         (mf/use-fn
          (mf/deps ids change-radius)
          (fn [value]
-           (let [token-value (wtc/maybe-resolve-token-value value)]
-             (st/emit!
-              (change-radius (fn [shape]
-                               (-> (dt/maybe-apply-token-to-shape {:token (when token-value value)
-                                                                   :shape shape
-                                                                   :attributes (wtty/token-attributes :border-radius)})
-                                   (ctsr/set-radius-1 (or token-value value)))))))))
+           (st/emit!
+            (change-radius (fn [shape]
+                             (ctsr/set-radius-1 shape value))))))
 
         on-radius-multi-change
         (mf/use-fn
@@ -464,50 +400,24 @@
                                     :disabled disabled-width-sizing?)
                :title (tr "workspace.options.width")}
          [:span {:class (stl/css :icon-text)} "W"]
-         (if-not design-tokens?
-           [:> numeric-input* {:min 0.01
-                               :no-validate true
-                               :placeholder (if (= :multiple (:width values)) (tr "settings.multiple") "--")
-                               :on-change on-width-change
-                               :disabled disabled-width-sizing?
-                               :class (stl/css :numeric-input)
-                               :value (:width values)}]
-           [:& editable-select
-            {:placeholder (if (= :multiple (:rx values)) (tr "settings.multiple") "--")
-             :class (stl/css :token-select)
-             :disabled disabled-width-sizing?
-             :on-change on-width-change
-             :on-token-remove #(on-width-change (wtc/maybe-resolve-token-value %))
-             :options width-options
-             :position :left
-             :value (:width values)
-             :input-props {:type "number"
-                           :no-validate true
-                           :min 0.01}}])]
+         [:> numeric-input* {:min 0.01
+                             :no-validate true
+                             :placeholder (if (= :multiple (:width values)) (tr "settings.multiple") "--")
+                             :on-change on-width-change
+                             :disabled disabled-width-sizing?
+                             :class (stl/css :numeric-input)
+                             :value (:width values)}]]
         [:div {:class (stl/css-case :height true
                                     :disabled disabled-height-sizing?)
                :title (tr "workspace.options.height")}
          [:span {:class (stl/css :icon-text)} "H"]
-         (if-not design-tokens?
-           [:> numeric-input* {:min 0.01
-                               :no-validate true
-                               :placeholder (if (= :multiple (:height values)) (tr "settings.multiple") "--")
-                               :on-change on-height-change
-                               :disabled disabled-height-sizing?
-                               :class (stl/css :numeric-input)
-                               :value (:height values)}]
-           [:& editable-select
-            {:placeholder (if (= :multiple (:rx values)) (tr "settings.multiple") "--")
-             :class (stl/css :token-select)
-             :disabled disabled-height-sizing?
-             :on-change on-height-change
-             :on-token-remove #(on-height-change (wtc/maybe-resolve-token-value %))
-             :options height-options
-             :position :right
-             :value (:height values)
-             :input-props {:type "number"
-                           :no-validate true
-                           :min 0.01}}])]
+         [:> numeric-input* {:min 0.01
+                             :no-validate true
+                             :placeholder (if (= :multiple (:height values)) (tr "settings.multiple") "--")
+                             :on-change on-height-change
+                             :disabled disabled-height-sizing?
+                             :class (stl/css :numeric-input)
+                             :value (:height values)}]]
         [:button {:class (stl/css-case
                           :lock-size-btn true
                           :selected (true? proportion-lock)
@@ -564,24 +474,13 @@
               [:div {:class (stl/css :radius-1)
                      :title (tr "workspace.options.radius")}
                [:span {:class (stl/css :icon)}  i/corner-radius]
-               (if-not design-tokens?
-                 [:> numeric-input*
-                  {:placeholder (if (= :multiple (:rx values)) (tr "settings.multiple") "--")
-                   :ref radius-input-ref
-                   :min 0
-                   :on-change on-radius-1-change
-                   :class (stl/css :numeric-input)
-                   :value (:rx values)}]
-                 [:& editable-select
-                  {:placeholder (if (= :multiple (:rx values)) (tr "settings.multiple") "--")
-                   :class (stl/css :token-select)
-                   :on-change on-radius-1-change
-                   :on-token-remove on-border-radius-token-unapply
-                   :options border-radius-options
-                   :position :right
-                   :value (:rx values)
-                   :input-props {:type "number"
-                                 :min 0}}])]
+               [:> numeric-input*
+                {:placeholder (if (= :multiple (:rx values)) (tr "settings.multiple") "--")
+                 :ref radius-input-ref
+                 :min 0
+                 :on-change on-radius-1-change
+                 :class (stl/css :numeric-input)
+                 :value (:rx values)}]]
 
               @radius-multi?
               [:div {:class (stl/css :radius-1)