wip

🐛 Bugfixing

.circleci/config.yml

@@ -11,11 +11,11 @@ jobs:
       - image: cimg/redis:7.0.5
 
     working_directory: ~/repo
-    resource_class: large
+    resource_class: medium+
 
     environment:
-      # Customize the JVM maximum heap limit
-      JVM_OPTS: -Xmx4g
+      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
+      NODE_OPTIONS: --max-old-space-size=4096
 
     steps:
       - checkout
@@ -28,50 +28,59 @@ jobs:
            - v1-dependencies-
 
       - run: cd .clj-kondo && cat config.edn
+      - run: cat .cljfmt.edn
+      - run: clj-kondo --version
 
-      - run:
-          name: frontend styles prettier
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn run lint-scss
+      # - run:
+      #     name: "fmt check [clj]"
+      #     command: |
+      #       yarn run fmt:clj:check
 
       - run:
           name: common lint
           working_directory: "./common"
           command: |
-            clj-kondo --version
-            clj-kondo --parallel --lint src/
+            yarn install
+            yarn run lint:clj
 
       - run:
           name: frontend lint
           working_directory: "./frontend"
           command: |
-            clj-kondo --version
-            clj-kondo --parallel --lint src/
+            yarn install
+            yarn run lint:scss
+            yarn run lint:clj
 
       - run:
           name: backend lint
           working_directory: "./backend"
           command: |
-            clj-kondo --version
-            clj-kondo --parallel --lint src/
+            yarn install
+            yarn run lint:clj
 
       - run:
-          working_directory: "./common"
-          name: common tests
+          name: exporter lint
+          working_directory: "./exporter"
           command: |
             yarn install
+            yarn run lint:clj
+
+      - run:
+          name: "common tests"
+          working_directory: "./common"
+          command: |
             yarn test
             clojure -X:dev:test :patterns '["common-tests.*-test"]'
 
-          environment:
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
-            JVM_OPTS: -Xmx4g
-            NODE_OPTIONS: --max-old-space-size=4096
+      - run:
+          name: "frontend tests"
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn test
 
       - run:
-          name: backend test
+          name: "backend tests"
           working_directory: "./backend"
           command: |
             clojure -X:dev:test :patterns '["backend-tests.*-test"]'
@@ -81,18 +90,6 @@ jobs:
             PENPOT_TEST_DATABASE_USERNAME: penpot_test
             PENPOT_TEST_DATABASE_PASSWORD: penpot_test
             PENPOT_TEST_REDIS_URI: "redis://localhost/1"
-            JVM_OPTS: -Xmx4g
-
-      - run:
-          name: frontend tests
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn test
-
-          environment:
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
-            NODE_OPTIONS: --max-old-space-size=4096
 
       - save_cache:
          paths:

.clj-kondo/config.edn

@@ -5,6 +5,7 @@
   promesa.exec.csp/go-loop clojure.core/loop
   rumext.v2/defc clojure.core/defn
   rumext.v2/fnc clojure.core/fn
+  promesa.util/with-open clojure.core/with-open
   app.common.data/export clojure.core/def
   app.common.data.macros/get-in clojure.core/get-in
   app.common.data.macros/with-open clojure.core/with-open
@@ -16,15 +17,24 @@
   {app.common.data.macros/export hooks.export/export
    potok.core/reify hooks.export/potok-reify
    app.util.services/defmethod hooks.export/service-defmethod
+   app.common.record/defrecord hooks.export/penpot-defrecord
    app.db/with-atomic hooks.export/penpot-with-atomic
+   rumext.v2/fnc hooks.export/rumext-fnc
    }}
 
  :output
  {:exclude-files
   ["data_readers.clj"
-   "app/util/perf.cljs"
-   "app/common/logging.cljc"
-   "app/common/exceptions.cljc"]}
+   "src/app/util/perf.cljs"
+   "src/app/common/logging.cljc"
+   "src/app/common/exceptions.cljc"
+   "^(?:backend|frontend|exporter|common)/build.clj"
+   "^(?:backend|frontend|exporter|common)/deps.edn"
+   "^(?:backend|frontend|exporter|common)/scripts/"
+   "^(?:backend|frontend|exporter|common)/dev/"
+   "^(?:backend|frontend|exporter|common)/test/"]
+
+  :linter-name true}
 
  :linters
  {:unsorted-required-namespaces
@@ -60,4 +70,3 @@
    :exclude-destructured-keys-in-fn-args false
    }
   }}
-

.clj-kondo/hooks/export.clj

@@ -41,18 +41,35 @@
 
 (defn penpot-with-atomic
   [{:keys [node]}]
-  (let [[_ params & other] (:children node)
+  (let [[params & body] (rest (:children node))]
+    (if (api/vector-node? params)
+      (let [[sym val opts] (:children params)]
+        (when-not (and sym val)
+          (throw (ex-info "No sym and val provided" {})))
+        {:node (api/list-node
+                (list*
+                 (api/token-node 'let)
+                 (api/vector-node [sym val])
+                 opts
+                 body))})
 
-        result (if (api/vector-node? params)
-                 (api/list-node
-                  (into [(api/token-node (symbol "clojure.core" "with-open")) params] other))
-                 (api/list-node
-                  (into [(api/token-node (symbol "clojure.core" "with-open"))
-                         (api/vector-node [params params])]
-                        other)))
+      {:node (api/list-node
+              (into [(api/token-node 'let)
+                     (api/vector-node [params params])]
+                    body))})))
+
+(defn rumext-fnc
+  [{:keys [node]}]
+  (let [[cname mdata params & body] (rest (:children node))
+        [params body] (if (api/vector-node? mdata)
+                        [mdata (cons params body)]
+                        [params body])]
+    (let [result (api/list-node
+                  (into [(api/token-node 'fn)
+                         params]
+                        (cons mdata body)))]
+      {:node result})))
 
-        ]
-    {:node result}))
 
 (defn penpot-defrecord
   [{:keys [:node]}]

.cljfmt.edn

@@ -0,0 +1,8 @@
+{:sort-ns-references? true
+ :remove-multiple-non-indenting-spaces? false
+ :remove-surrounding-whitespace? true
+ :remove-consecutive-blank-lines? false
+ :extra-indents {rumext.v2/fnc [[:inner 0]]
+                 promesa.exec/thread [[:inner 0]]
+                 specify! [[:inner 0] [:inner 1]]}
+ }

.gitignore

@@ -1,7 +1,16 @@
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
 *-init.clj
+*.css.json
 *.jar
 *.orig
 *.penpot
+*.css.json
 .calva
 .clj-kondo
 .cpcache
@@ -57,3 +66,4 @@
 /web
 clj-profiler/
 node_modules
+frontend/.storybook/preview-body.html

.vscode/settings.json

@@ -1,9 +1,9 @@
 {
-    "files.exclude": {
-        "**/.clj-kondo": true,
-        "**/.cpcache": true,
-        "**/.lsp": true,
-        "**/.shadow-cljs": true,
-        "**/node_modules": true
-    }
+  "files.exclude": {
+    "**/.clj-kondo": true,
+    "**/.cpcache": true,
+    "**/.lsp": true,
+    "**/.shadow-cljs": true,
+    "**/node_modules": true
+  }
 }

.yarnrc.yml

@@ -0,0 +1,9 @@
+enableGlobalCache: true
+
+enableImmutableCache: false
+
+enableImmutableInstalls: false
+
+enableTelemetry: false
+
+nodeLinker: node-modules

CHANGES.md

@@ -1,5 +1,53 @@
 # CHANGELOG
 
+## 1.20.0
+
+### :boom: Breaking changes & Deprecations
+
+### :sparkles: New features
+
+- Select through stroke only rectangle [Taiga #5484](https://tree.taiga.io/project/penpot/issue/5484)
+- Override browser Ctrl+ and Ctrl- zoom with Penpot Zoom [Taiga #3200](https://tree.taiga.io/project/penpot/us/3200)
+
+### :bug: Bugs fixed
+
+- Fix pixelated thumbnails [Github
+  #3681](https://github.com/penpot/penpot/issues/3681) [Github #3661](https://github.com/penpot/penpot/issues/3661)
+
+### :arrow_up: Deps updates
+
+### :heart: Community contributions by (Thank you!)
+
+## 1.19.5
+
+### :bug: New features
+
+- Fix problem with alignment performance
+
+## 1.19.4
+
+### :sparkles: New features
+
+- Improve selected colors [Taiga #5805]( https://tree.taiga.io/project/penpot/us/5805)
+
+### :bug: Bugs fixed
+
+- Fix problem with z-index field in non-absolute items
+
+## 1.19.3
+
+### :sparkles: New features
+
+- Remember last color mode in colorpicker [Taiga #5508](https://tree.taiga.io/project/penpot/issue/5508)
+- Improve layers multiselection behaviour [Github #5741](https://github.com/penpot/penpot/issues/5741)
+- Remember last active team across logouts / sessions [Github #3325](https://github.com/penpot/penpot/issues/3325)
+
+### :bug: Bugs fixed
+
+- List view is discarded on tab change on Workspace Assets Sidebar tab [Github #3547](https://github.com/penpot/penpot/issues/3547)
+- Fix message popup remains open when exiting workspace with browser back button [Taiga #5747](https://tree.taiga.io/project/penpot/issue/5747)
+- When editing text if font is changed, the proportions of the rendered shape are wrong [Taiga #5786](https://tree.taiga.io/project/penpot/issue/5786)
+
 ## 1.19.2
 
 ### :sparkles: New features
@@ -41,6 +89,8 @@
 - Add support for local caching of google fonts (this avoids exposing the final user IP to
   goolge and reduces the amount of request sent to google)
 - Set smooth/instant autoscroll depending on distance [GitHub #3377](https://github.com/penpot/penpot/issues/3377)
+- New component icon [Taiga #5290](https://tree.taiga.io/project/penpot/us/5290)
+- Show a confirmation dialog when an user tries to publish an empty library [Taiga #5294](https://tree.taiga.io/project/penpot/us/5294)
 
 ### :bug: Bugs fixed
 
@@ -102,6 +152,7 @@
 - Fix create typography with section closed [Taiga #5574](https://tree.taiga.io/project/penpot/issue/5574)
 - Fix exports menu on viewer mode [Taiga #5568](https://tree.taiga.io/project/penpot/issue/5568)
 - Fix create empty comments [Taiga #5536](https://tree.taiga.io/project/penpot/issue/5536)
+- Fix text changes not propagated to copy [Taiga #5364](https://tree.taiga.io/project/penpot/issue/5364)
 - Fix position of text cursor is a bit too high in Invitations section [Taiga #5511](https://tree.taiga.io/project/penpot/issue/5511)
 - Fix undo when updating several texts [Taiga #5197](https://tree.taiga.io/project/penpot/issue/5197)
 - Fix assets right click button for multiple selection [Taiga #5545](https://tree.taiga.io/project/penpot/issue/5545)

README.md

@@ -26,7 +26,7 @@
 
 ![feature-readme](https://user-images.githubusercontent.com/1045247/189871786-0b44f7cf-3a0a-4445-a87b-9919ec398bf7.gif)
 
-**:tada: [Important Notice!] :tada:** Our very first **Penpot Fest** is happening on June 28-30, Barcelona (Spain). **Secure yourself a ticket** to know everything about the present and future of Penpot and be part of the conversation! See details on the amazing venue and speakers lineup at [penpotfest.org](https://penpotfest.org)! :zap:
+🎇 **Penpot Fest exceeded all expectations - it was a complete success!** 🎇 Penpot Fest is our first Design event that brought designers and developers from the Open Source communities and beyond. Watch the replay of the talks on our [Youtube channel](https://www.youtube.com/playlist?list=PLgcCPfOv5v56-fghJo2dHNBqL9zlDTslh) or [Peertube channel](https://peertube.kaleidos.net/w/p/1tWgyJTt8sKbWwCEcBimZW)
 
 Penpot is the first **Open Source** design and prototyping platform meant for cross-domain teams. Non dependent on operating systems, Penpot is web based and works with open standards (SVG). Penpot invites designers all over the world to fall in love with open source while getting developers excited about the design process in return.
 
@@ -109,6 +109,7 @@ Every sort of contribution will be very helpful to enhance Penpot. How you’ll
 - Create and [share Libraries & templates](https://penpot.app/libraries-templates.html) that will be helpful for the community
 - Become a [translator](https://help.penpot.app/contributing-guide/translations)
 - Give feedback: [Mail us](mailto:support@penpot.app)
+- **Contribute to Penpot's code:** [Watch this video](https://www.youtube.com/watch?v=TpN0osiY-8k) by Alejandro Alonso, CIO and developer at Penpot, where he gives us a hands-on demo of how to use Penpot’s repository and make changes in both front and back end
 
 To find (almost) everything you need to know on how to contribute to Penpot, refer to the [contributing-guide](https://help.penpot.app/contributing-guide/).
 

backend/.gitignore

@@ -0,0 +1,7 @@
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.11.1"}
-  org.clojure/core.async {:mvn/version "1.6.673"}
+  org.clojure/clojure {:mvn/version "1.12.0-alpha5"}
+  org.clojure/tools.namespace {:mvn/version "1.4.4"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.5-4"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.5-10"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,30 +17,32 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.2.4.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.2.6.RELEASE"}
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
   funcool/yetti
-  {:git/tag "v9.16"
-   :git/sha "7df3e08"
+  {:git/tag "v10.0"
+   :git/sha "520613f"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
-  com.github.seancorfield/next.jdbc {:mvn/version "1.3.883"}
+  com.github.seancorfield/next.jdbc {:mvn/version "1.3.894"}
   metosin/reitit-core {:mvn/version "0.6.0"}
+  nrepl/nrepl {:mvn/version "1.1.0"}
+  cider/cider-nrepl {:mvn/version "0.43.1"}
 
   org.postgresql/postgresql {:mvn/version "42.6.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "5.0.1"}
+  com.zaxxer/HikariCP {:mvn/version "5.1.0"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.5.351"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.6"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.8"}
 
-  org.jsoup/jsoup {:mvn/version "1.16.1"}
+  org.jsoup/jsoup {:mvn/version "1.16.2"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -49,14 +51,13 @@
   org.lz4/lz4-java {:mvn/version "1.8.0"}
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
-  integrant/integrant {:mvn/version "0.8.1"}
 
   dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.11.4"}
+  markdown-clj/markdown-clj {:mvn/version "1.11.7"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.20.96"}
+  software.amazon.awssdk/s3 {:mvn/version "2.20.138"}
   }
 
  :paths ["src" "resources" "target/classes"]
@@ -64,7 +65,6 @@
  {:dev
   {:extra-deps
    {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
-    org.clojure/tools.namespace {:mvn/version "RELEASE"}
     clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
     org.clojure/data.csv {:mvn/version "RELEASE"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
@@ -73,7 +73,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.9.3" :git/sha "e537cd1"}}
+   {io.github.clojure/tools.build {:git/tag "v0.9.5" :git/sha "24f2894"}}
    :ns-default build}
 
   :test

backend/dev/user.clj

@@ -19,10 +19,12 @@
    [app.common.schema.generators :as sg]
    [app.common.spec :as us]
    [app.common.transit :as t]
+   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.config :as cfg]
+   [app.config :as cf]
+   [app.db :as db]
    [app.main :as main]
-   [app.srepl.helpers]
+   [app.srepl.helpers :as srepl.helpers]
    [app.srepl.main :as srepl]
    [app.util.blob :as blob]
    [app.util.json :as json]
@@ -48,13 +50,12 @@
    [malli.generator :as mg]
    [malli.registry :as mr]
    [malli.transform :as mt]
-   [malli.util :as mu]))
+   [malli.util :as mu]
+   [promesa.exec :as px]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
 (set! *warn-on-reflection* true)
 
-(defonce system nil)
-
 ;; --- Benchmarking Tools
 
 (defmacro run-quick-bench
@@ -92,20 +93,14 @@
 (defn- start
   []
   (try
-    (alter-var-root #'system (fn [sys]
-                               (when sys (ig/halt! sys))
-                               (-> (merge main/system-config main/worker-config)
-                                   (ig/prep)
-                                   (ig/init))))
+    (main/start)
     :started
     (catch Throwable cause
       (ex/print-throwable cause))))
 
 (defn- stop
   []
-  (alter-var-root #'system (fn [sys]
-                             (when sys (ig/halt! sys))
-                             nil))
+  (main/stop)
   :stopped)
 
 (defn restart
@@ -118,62 +113,26 @@
   (stop)
   (repl/refresh-all :after 'user/start))
 
-(defn compression-bench
-  [data]
-  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
-        v1 (time (humanize (alength (blob/encode data {:version 1}))))
-        v3 (time (humanize (alength (blob/encode data {:version 3}))))
-        v4 (time (humanize (alength (blob/encode data {:version 4}))))
-        v5 (time (humanize (alength (blob/encode data {:version 5}))))
-        v6 (time (humanize (alength (blob/encode data {:version 6}))))
-        ]
-    (print-table
-     [{
-       :v1 v1
-       :v3 v3
-       :v4 v4
-       :v5 v5
-       :v6 v6
-       }])))
+;; (defn compression-bench
+;;   [data]
+;;   (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
+;;         v1 (time (humanize (alength (blob/encode data {:version 1}))))
+;;         v3 (time (humanize (alength (blob/encode data {:version 3}))))
+;;         v4 (time (humanize (alength (blob/encode data {:version 4}))))
+;;         v5 (time (humanize (alength (blob/encode data {:version 5}))))
+;;         v6 (time (humanize (alength (blob/encode data {:version 6}))))
+;;         ]
+;;     (print-table
+;;      [{
+;;        :v1 v1
+;;        :v3 v3
+;;        :v4 v4
+;;        :v5 v5
+;;        :v6 v6
+;;        }])))
 
 (defonce debug-tap
   (do
     (add-tap #(locking debug-tap
                 (prn "tap debug:" %)))
     1))
-
-
-(sm/def! ::test
-  [:map {:title "Foo"}
-   [:x :int]
-   [:y {:min 0} :double]
-   [:bar
-    [:map {:title "Bar"}
-     [:z :string]
-     [:v ::sm/uuid]]]
-   [:items
-    [:vector ::dt/instant]]])
-
-(sm/def! ::test2
-  [:multi {:title "Foo" :dispatch :type}
-   [:x
-    [:map {:title "FooX"}
-     [:type [:= :x]]
-     [:x :int]]]
-   [:y
-    [:map
-     [:type [:= :x]]
-     [:y [::sm/one-of #{:a :b :c}]]]]
-   [:z
-    [:map {:title "FooZ"}
-     [:z
-      [:multi {:title "Bar" :dispatch :type}
-       [:a
-        [:map
-         [:type [:= :a]]
-         [:a :int]]]
-       [:b
-        [:map
-         [:type [:= :b]]
-         [:b :int]]]]]]]])
-

backend/package.json

@@ -1,18 +1,26 @@
 {
-  "name": "uxbox-back",
-  "version": "0.1.0",
-  "description": "The Open-Source prototyping tool",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "build-emails": "./scripts/build-email-templates.sh"
-  },
+  "name": "backend",
+  "version": "1.0.0",
+  "license": "MPL-2.0",
+  "author": "Kaleidos INC",
+  "private": true,
+  "packageManager": "yarn@4.0.2",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/uxbox/uxbox.git"
+    "url": "https://github.com/penpot/penpot"
+  },
+  "dependencies": {
+    "luxon": "^3.4.2",
+    "sax": "^1.2.4"
   },
-  "author": "Uxbox",
-  "license": "SEE LICENSE IN <LICENSE>",
   "devDependencies": {
-    "mjml": "^4.6.3"
+    "nodemon": "^3.0.1",
+    "source-map-support": "^0.5.21",
+    "ws": "^8.13.0"
+  },
+  "scripts": {
+    "fmt:clj:check": "cljfmt check --parallel=false src/ test/",
+    "fmt:clj": "cljfmt fix --parallel=true src/ test/",
+    "lint:clj": "clj-kondo --parallel --lint src/"
   }
 }

backend/resources/app/templates/api-doc-entry.tmpl

@@ -25,6 +25,12 @@
           <span>SCHEMA</span>
         </span>
       {% endif %}
+
+      {% if item.sse %}
+        <span class="tag">
+          <span>SSE</span>
+        </span>
+      {% endif %}
     </div>
   </div>
   <div class="rpc-row-detail hidden">

backend/resources/app/templates/api-doc.css

@@ -156,7 +156,7 @@ h4 {
 }
 
 .rpc-row-info > .module {
-  width: 120px;
+  width: 150px;
   font-weight: bold;
   border-right: 1px dotted #777;
   text-align: right;

backend/resources/app/templates/api-doc.tmpl

@@ -56,6 +56,17 @@
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
 
+        <h3>SSE (Server-Sent Events)</h3>
+        <p>The methods marked with <b>SSE</b> returns
+        a <a href="https://html.spec.whatwg.org/multipage/server-sent-events.html"> SSE
+        formatted</a> stream on the response body, always with status 200. The events are
+        always encoded using `application/transit+json` encoding (for now no content
+        negotiation is possible on methods that return SSE streams). </p>
+        <p>On the javascript side you can use
+        the <a href="https://github.com/rexxars/eventsource-parser">eventsoure-parser</a>
+        library for propertly parsing the response body using the
+        standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
+        API</a></p>
 
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share

backend/resources/app/templates/debug.tmpl

@@ -6,11 +6,17 @@ Debug Main Page
 
 {% block content %}
 <nav>
-  <h1>Debug INDEX:</h1>
-  <div>[<a href="/dbg/error">ERRORS</a>]</div>
+  <div class="title">
+    <h1>ADMIN DEBUG INTERFACE</h1>
+  </div>
 </nav>
-<main class="index">
+<main class="dashboard">
   <section class="widget">
+    <fieldset>
+      <legend>Error reports</legend>
+      <desc><a href="/dbg/error">CLICK HERE TO SEE THE ERROR REPORTS</a> </desc>
+    </fieldset>
+
     <fieldset>
       <legend>Download file data:</legend>
       <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
@@ -37,9 +43,42 @@ Debug Main Page
           <input type="checkbox" name="reuseid" />
         </div>
 
-        <input type="submit" value="Upload" />
+        <div class="row">
+          <input type="submit" value="Upload" />
+        </div>
       </form>
     </fieldset>
+
+    <fieldset>
+      <legend>Profile Management</legend>
+      <form method="post" action="/dbg/actions/resend-email-verification">
+        <div class="row">
+          <input type="email" name="email" placeholder="example@example.com" value="" />
+        </div>
+
+        <div class="row">
+          <label for="force-verify">Are you sure?</label>
+          <input id="force-verify" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" name="resend" value="Resend Verification" />
+          <input type="submit" name="verify" value="Verify" />
+        </div>
+
+        <div class="row">
+          <input type="submit" class="danger" name="block" value="Block" />
+          <input type="submit" class="danger" name="unblock" value="Unblock" />
+        </div>
+      </form>
+    </fieldset>
+
+
+
   </section>
 
   <section class="widget">
@@ -123,5 +162,35 @@ Debug Main Page
       </form>
     </fieldset>
   </section>
+
+  <section class="widget">
+    <fieldset>
+      <legend>Reset file version</legend>
+      <desc>Allows reset file data version to a specific number/</desc>
+
+      <form method="post" action="/dbg/actions/reset-file-data-version">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="number" style="width:100px" name="version" placeholder="version" value="32" />
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
 </main>
 {% endblock %}

backend/resources/app/templates/styles.css

@@ -116,29 +116,50 @@ nav > div:not(:last-child) {
   width: unset;
 }
 
-.index {
+.dashboard {
   margin-top: 40px;
   display: flex;
 }
 
-.index > section {
-  padding: 10px;
-  background-color: #e3e3e3;
+.widget {
   max-width: 400px;
   margin: 5px;
   height: fit-content;
 }
 
-.index fieldset:not(:first-child) {
+.widget input[type=submit] {
+  outline: none;
+  border: 1px solid gray;
+  border-radius: 2px;
+  padding: 3px 5px;
+}
+
+.widget input[type=submit].danger {
+  outline: none;
+  border: 1px solid red;
+  border-radius: 2px;
+  padding: 3px 5px;
+}
+
+
+
+.widget > fieldset {
+  padding: 10px;
+  background-color: #f9f9f9;
+}
+
+.widget > fieldset:not(:last-child) {
+  margin-bottom: 10px;
+}
+
+
+
+.dashboard fieldset:not(:first-child) {
   margin-top: 15px;
 }
 
-/* .index > section:not(:last-child) { */
-/*   margin-bottom: 10px; */
-/* } */
 
-
-.index > section > h2 {
+.widget > h2 {
   margin-top: 0px;
 }
 

backend/resources/climit.edn

@@ -3,12 +3,17 @@
 ;; Optional: queue, ommited means Integer/MAX_VALUE
 ;; Optional: timeout, ommited means no timeout
 ;; Note: queue and timeout are excluding
-{:update-file-by-id {:permits 1 :queue 3}
- :update-file {:permits 20}
+{:update-file/by-profile
+ {:permits 1 :queue 5}
 
- :derive-password {:permits 8}
- :process-font  {:permits 4 :queue 32}
- :process-image {:permits 8 :queue 32}
+ :update-file/global {:permits 20}
 
- :submit-audit-events-by-profile
+ :derive-password/global {:permits 8}
+ :process-font/global  {:permits 4}
+ :process-image/global {:permits 8}
+
+ :file-thumbnail-ops/by-profile
+ {:permits 2}
+
+ :submit-audit-events/by-profile
  {:permits 1 :queue 3}}

backend/resources/log4j2-devenv.xml

@@ -22,8 +22,8 @@
     <Logger name="org.postgresql" level="error" />
 
     <Logger name="app.rpc.commands.binfile" level="debug" />
-    <Logger name="app.storage.tmp" level="debug" />
-    <Logger name="app.worker" level="info" />
+    <Logger name="app.storage.tmp" level="info" />
+    <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
     <Logger name="app.http.websocket" level="info" />
     <Logger name="app.util.websocket" level="info" />
@@ -31,6 +31,7 @@
     <Logger name="app.rpc.rlimit" level="info" />
     <Logger name="app.rpc.climit" level="info" />
     <Logger name="app.rpc.mutations.files" level="info" />
+    <Logger name="app.common.files.migrations" level="info" />
 
     <Logger name="app.loggers" level="debug" additivity="false">
       <AppenderRef ref="main" level="debug" />

backend/resources/log4j2.xml

@@ -13,10 +13,14 @@
     <Logger name="org.postgresql" level="error" />
 
     <Logger name="app.util" level="info" />
+
+    <Logger name="app.loggers" level="debug" />
+
     <Logger name="app" level="info" additivity="false">
       <AppenderRef ref="console" />
     </Logger>
 
+
     <Root level="info">
       <AppenderRef ref="console" />
     </Root>

backend/scripts/manage.py

@@ -44,11 +44,16 @@ def send_eval(expr):
         s.send(b":repl/quit\n\n")
 
         with s.makefile() as f:
-            result = json.load(f)
-            tag = result.get("tag", None)
-            if tag != "ret":
-                raise RuntimeError("unexpected response from PREPL")
-            return result.get("val", None), result.get("exception", None)
+            while True:
+                line = f.readline()
+                result = json.loads(line)
+                tag = result.get("tag", None)
+                if tag == "ret":
+                    return result.get("val", None), result.get("exception", None)
+                elif tag == "out":
+                    print(result.get("val"), end="")
+                else:
+                    raise RuntimeError("unexpected response from PREPL")
 
 def encode(val):
     return json.dumps(json.dumps(val))
@@ -60,7 +65,7 @@ def print_error(res):
 
 def run_cmd(params):
     try:
-        expr = "(app.srepl.ext/run-json-cmd {})".format(encode(params))
+        expr = "(app.srepl.cli/exec {})".format(encode(params))
         res, failed = send_eval(expr)
         if failed:
             print_error(res)
@@ -140,12 +145,22 @@ def derive_password(password):
     res = run_cmd(params)
     print(f"Derived password: \"{res}\"")
 
+
+def migrate_components_v2():
+    params = {
+        "cmd": "migrate-v2",
+        "params": {}
+    }
+
+    run_cmd(params)
+
 available_commands = (
     "create-profile",
     "update-profile",
     "delete-profile",
     "search-profile",
     "derive-password",
+    "migrate-components-v2",
 )
 
 parser = argparse.ArgumentParser(
@@ -217,3 +232,8 @@ elif args.action == "search-profile":
         email = input("Email: ")
 
     search_profile(email)
+
+elif args.action == "migrate-components-v2":
+    migrate_components_v2()
+
+

backend/scripts/nrepl

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+clojure -J-Xms50m -J-Xmx256m -J-XX:+UseSerialGC -Sdeps '{:deps {reply/reply {:mvn/version "0.5.0"}}}' -M -m reply.main --attach localhost:6064 -e "(in-ns 'app.main)"

backend/scripts/repl

@@ -10,9 +10,10 @@ export PENPOT_FLAGS="\
        enable-login-with-google \
        enable-login-with-github \
        enable-login-with-gitlab \
+       enable-backend-worker \
        enable-backend-asserts \
-       enable-fdata-storage-pointer-map \
-       enable-fdata-storage-objets-map \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
@@ -24,7 +25,12 @@ export PENPOT_FLAGS="\
        enable-rpc-rlimit \
        enable-soft-rpc-rlimit \
        enable-webhooks \
-       enable-access-tokens";
+       enable-access-tokens \
+       disable-feature-components-v2 \
+       enable-file-validation \
+       enable-file-schema-validation \
+       disable-soft-file-schema-validation \
+       disable-soft-file-validation";
 
 # export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
 # export PENPOT_DATABASE_USERNAME="penpot"
@@ -39,10 +45,13 @@ export PENPOT_FLAGS="\
 # export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
 
 # Initialize MINIO config
-mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
-mc admin user add penpot-s3 penpot-devenv penpot-devenv
-mc admin policy set penpot-s3 readwrite user=penpot-devenv
-mc mb penpot-s3/penpot -p
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
 
 export AWS_ACCESS_KEY_ID=penpot-devenv
 export AWS_SECRET_ACCESS_KEY=penpot-devenv
@@ -50,21 +59,23 @@ export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
 export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
 export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
 
-#-J-Djdk.virtualThreadScheduler.parallelism=16
-
 export OPTIONS="
        -A:jmx-remote -A:dev \
        -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
        -J-Djdk.attach.allowAttachSelf \
+       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
        -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -J-XX:+EnableDynamicAgentLoading \
        -J-XX:-OmitStackTraceInFastThrow \
        -J-XX:+UnlockDiagnosticVMOptions \
        -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full \
-       -J--enable-preview";
+       -J-Djdk.tracePinnedThreads=full"
+
+# Enable preview
+export OPTIONS="$OPTIONS -J--enable-preview"
 
 # Setup HEAP
-export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
+# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
 # export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
 
 # Increase virtual thread pool size
@@ -77,7 +88,7 @@ export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
 # export OPTIONS="$OPTIONS -J-Xint"
 
 # Setup GC
-export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
+# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
 
 # Setup GC
 # export OPTIONS="$OPTIONS -J-XX:+UseZGC"

backend/scripts/start-dev

@@ -6,33 +6,90 @@ export PENPOT_FLAGS="\
        $PENPOT_FLAGS \
        enable-prepl-server \
        enable-urepl-server \
+       enable-nrepl-server \
        enable-webhooks \
        enable-backend-asserts \
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
-       enable-fdata-storage-pointer-map \
-       enable-fdata-storage-objets-map \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
        disable-secure-session-cookies \
        enable-smtp \
-       enable-access-tokens";
+       enable-access-tokens \
+       disable-feature-components-v2 \
+       enable-file-validation \
+       enable-file-schema-validation \
+       disable-soft-file-schema-validation \
+       disable-soft-file-validation";
 
-set -ex
+export OPTIONS="
+       -A:jmx-remote -A:dev \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
+       -J-Dlog4j2.configurationFile=log4j2.xml \
+       -J-XX:+EnableDynamicAgentLoading \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints"
+
+# Setup HEAP
+# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
+# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
+
+# Increase virtual thread pool size
+# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
+
+# Disable C2 Compiler
+# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
+
+# Disable all compilers
+# export OPTIONS="$OPTIONS -J-Xint"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
+
+# Enable ImageMagick v7.x support
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
+
+
+# Initialize MINIO config
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
+export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
+export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
 
 if [ "$1" = "--watch" ]; then
+    trap "exit" INT TERM ERR
+    trap "kill 0" EXIT
+
     echo "Start Watch..."
 
-    clojure -A:dev -M -m app.main &
-    PID=$!
+    clojure $OPTIONS -A:dev -M -m app.main &
 
     npx nodemon \
         --watch src \
         --watch ../common \
         --ext "clj" \
         --signal SIGKILL \
-        --exec 'echo "(user/restart)" | nc -N localhost 6062'
+        --exec 'echo "(app.main/stop)\n\r(repl/refresh)\n\r(app.main/start)\n" | nc -N localhost 6062'
+
+    wait;
 
-    kill -9 $PID
 else
-    clojure -A:dev -M -m app.main
+    set -x
+    clojure $OPTIONS -A:dev -M -m app.main;
 fi

backend/src/app/auth.clj

@@ -8,14 +8,13 @@
   (:require
    [app.config :as cf]
    [buddy.hashers :as hashers]
-   [cuerdas.core :as str]
-   [promesa.exec :as px]))
+   [cuerdas.core :as str]))
 
 (def default-params
   {:alg :argon2id
-   :memory (* 32768 2) ;; 64 MiB
-   :iterations 7
-   :parallelism (px/get-available-processors)})
+   :memory 32768 ;; 32 MiB
+   :iterations 3
+   :parallelism 2})
 
 (defn derive-password
   [password]

backend/src/app/auth/oidc.clj

@@ -31,7 +31,7 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [yetti.response :as-alias yrs]))
+   [ring.response :as-alias rres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -353,8 +353,7 @@
           (get-name [props]
             (let [attr-kw (cf/get :oidc-name-attr "name")
                   attr-ph (parse-attr-path provider attr-kw)]
-              (get-in props attr-ph)))
-          ]
+              (get-in props attr-ph)))]
 
     (let [props (qualify-props provider info)
           email (get-email props)]
@@ -479,8 +478,8 @@
 
 (defn- redirect-response
   [uri]
-  {::yrs/status 302
-   ::yrs/headers {"location" (str uri)}})
+  {::rres/status 302
+   ::rres/headers {"location" (str uri)}})
 
 (defn- generate-error-redirect
   [_ cause]
@@ -557,8 +556,8 @@
                                 :props props
                                 :exp (dt/in-future "4h")})
         uri   (build-auth-uri cfg state)]
-    {::yrs/status 200
-     ::yrs/body {:redirect-uri uri}}))
+    {::rres/status 200
+     ::rres/body {:redirect-uri uri}}))
 
 (defn- callback-handler
   [cfg request]

backend/src/app/config.clj

@@ -203,6 +203,7 @@
 (s/def ::storage-assets-s3-bucket ::us/string)
 (s/def ::storage-assets-s3-region ::us/keyword)
 (s/def ::storage-assets-s3-endpoint ::us/string)
+(s/def ::storage-assets-s3-io-threads ::us/integer)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
@@ -294,6 +295,7 @@
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::rpc-rlimit-config
+                   ::rpc-climit-config
 
                    ::semaphore-process-font
                    ::semaphore-process-image
@@ -319,6 +321,7 @@
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
                    ::storage-assets-s3-endpoint
+                   ::storage-assets-s3-io-threads
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer

backend/src/app/db.clj

@@ -97,7 +97,7 @@
             :with-credentials (and (contains? cfg ::username)
                                    (contains? cfg ::password))
             :min-size (::min-size cfg)
-          :max-size (::max-size cfg))
+            :max-size (::max-size cfg))
     (create-pool cfg)))
 
 (defmethod ig/halt-key! ::pool
@@ -145,6 +145,10 @@
   [v]
   (instance? javax.sql.DataSource v))
 
+(defn connection?
+  [conn]
+  (instance? Connection conn))
+
 (s/def ::conn some?)
 (s/def ::nilable-pool (s/nilable ::pool))
 (s/def ::pool pool?)
@@ -227,49 +231,82 @@
     `(jdbc/with-transaction ~@args)))
 
 (defn open
-  [pool]
-  (jdbc/get-connection pool))
+  [system-or-pool]
+  (if (pool? system-or-pool)
+    (jdbc/get-connection system-or-pool)
+    (if (map? system-or-pool)
+      (open (::pool system-or-pool))
+      (ex/raise :type :internal
+                :code :unable-resolve-pool))))
+
+(defn get-connection
+  [cfg-or-conn]
+  (if (connection? cfg-or-conn)
+    cfg-or-conn
+    (if (map? cfg-or-conn)
+      (get-connection (::conn cfg-or-conn))
+      (ex/raise :type :internal
+                :code :unable-resolve-connection
+                :hint "expected conn or system map"))))
+
+(defn connection-map?
+  "Check if the provided value is a map like data structure that
+  contains a database connection."
+  [o]
+  (and (map? o) (connection? (::conn o))))
+
+(defn- get-connectable
+  [o]
+  (cond
+    (connection? o) o
+    (pool? o)       o
+    (map? o)        (get-connectable (or (::conn o) (::pool o)))
+    :else           (ex/raise :type :internal
+                              :code :unable-resolve-connectable
+                              :hint "expected conn, pool or system")))
 
 (def ^:private default-opts
   {:builder-fn sql/as-kebab-maps})
 
 (defn exec!
   ([ds sv]
-   (jdbc/execute! ds sv default-opts))
+   (-> (get-connectable ds)
+       (jdbc/execute! sv default-opts)))
   ([ds sv opts]
-   (jdbc/execute! ds sv (merge default-opts opts))))
+   (-> (get-connectable ds)
+       (jdbc/execute! sv (into default-opts (sql/adapt-opts opts))))))
 
 (defn exec-one!
   ([ds sv]
-   (jdbc/execute-one! ds sv default-opts))
+   (-> (get-connectable ds)
+       (jdbc/execute-one! sv default-opts)))
   ([ds sv opts]
-   (jdbc/execute-one! ds sv
-                      (-> (merge default-opts opts)
-                          (assoc :return-keys (::return-keys? opts false))))))
+   (-> (get-connectable ds)
+       (jdbc/execute-one! sv (into default-opts (sql/adapt-opts opts))))))
 
 (defn insert!
-  [ds table params & {:as opts}]
-  (exec-one! ds
-             (sql/insert table params opts)
-             (merge {::return-keys? true} opts)))
+  [ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
+  (-> (get-connectable ds)
+      (exec-one! (sql/insert table params opts)
+                 (assoc opts ::return-keys? return-keys?))))
 
 (defn insert-multi!
-  [ds table cols rows & {:as opts}]
-  (exec! ds
-         (sql/insert-multi table cols rows opts)
-         (merge {::return-keys? true} opts)))
+  [ds table cols rows & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
+  (-> (get-connectable ds)
+      (exec! (sql/insert-multi table cols rows opts)
+             (assoc opts ::return-keys? return-keys?))))
 
 (defn update!
-  [ds table params where & {:as opts}]
-  (exec-one! ds
-             (sql/update table params where opts)
-             (merge {::return-keys? true} opts)))
+  [ds table params where & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
+  (-> (get-connectable ds)
+      (exec-one! (sql/update table params where opts)
+                 (assoc opts ::return-keys? return-keys?))))
 
 (defn delete!
-  [ds table params & {:as opts}]
-  (exec-one! ds
-             (sql/delete table params opts)
-             (merge {::return-keys? true} opts)))
+  [ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
+  (-> (get-connectable ds)
+      (exec-one! (sql/delete table params opts)
+                 (assoc opts ::return-keys? return-keys?))))
 
 (defn is-row-deleted?
   [{:keys [deleted-at]}]
@@ -301,7 +338,8 @@
 
 (defn plan
   [ds sql]
-  (jdbc/plan ds sql sql/default-opts))
+  (-> (get-connectable ds)
+      (jdbc/plan sql sql/default-opts)))
 
 (defn get-by-id
   [ds table id & {:as opts}]
@@ -371,10 +409,6 @@
   [data]
   (org.postgresql.util.PGInterval. ^String data))
 
-(defn connection?
-  [conn]
-  (instance? Connection conn))
-
 (defn savepoint
   ([^Connection conn]
    (.setSavepoint conn))
@@ -382,57 +416,65 @@
    (.setSavepoint conn (name label))))
 
 (defn release!
-  [^Connection conn ^Savepoint sp ]
+  [^Connection conn ^Savepoint sp]
   (.releaseSavepoint conn sp))
 
 (defn rollback!
-  ([^Connection conn]
-   (.rollback conn))
-  ([^Connection conn ^Savepoint sp]
-   (.rollback conn sp)))
+  ([conn]
+   (let [^Connection conn (get-connection conn)]
+     (l/trc :hint "explicit rollback requested")
+     (.rollback conn)))
+  ([conn ^Savepoint sp]
+   (let [^Connection conn (get-connection conn)]
+     (l/trc :hint "explicit rollback requested")
+     (.rollback conn sp))))
 
 (defn tx-run!
-  [cfg f]
+  [system f & params]
   (cond
-    (connection? cfg)
-    (tx-run! {::conn cfg} f)
+    (connection? system)
+    (tx-run! {::conn system} f)
 
-    (pool? cfg)
-    (tx-run! {::pool cfg} f)
+    (pool? system)
+    (tx-run! {::pool system} f)
 
-    (::conn cfg)
-    (let [conn (::conn cfg)
+    (::conn system)
+    (let [conn (::conn system)
           sp   (savepoint conn)]
       (try
-        (let [result (f cfg)]
+        (let [result (apply f system params)]
           (release! conn sp)
           result)
         (catch Throwable cause
-          (rollback! sp)
+          (rollback! conn sp)
           (throw cause))))
 
-    (::pool cfg)
-    (with-atomic [conn (::pool cfg)]
-      (f (assoc cfg ::conn conn)))
+    (::pool system)
+    (with-atomic [conn (::pool system)]
+      (let [system (assoc system ::conn conn)
+            result (apply f system params)]
+        (when (::rollback system)
+          (rollback! conn))
+        result))
 
     :else
     (throw (IllegalArgumentException. "invalid arguments"))))
 
 (defn run!
-  [cfg f]
+  [system f & params]
   (cond
-    (connection? cfg)
-    (run! {::conn cfg} f)
+    (connection? system)
+    (run! {::conn system} f)
 
-    (pool? cfg)
-    (run! {::pool cfg} f)
+    (pool? system)
+    (run! {::pool system} f)
 
-    (::conn cfg)
-    (f cfg)
+    (::conn system)
+    (apply f system params)
 
-    (::pool cfg)
-    (with-open [^Connection conn (open (::pool cfg))]
-      (f (assoc cfg ::conn conn)))
+    (::pool system)
+    (with-open [^Connection conn (open (::pool system))]
+      (apply f (assoc system ::conn conn) params))
 
     :else
     (throw (IllegalArgumentException. "invalid arguments"))))

backend/src/app/db/sql.clj

@@ -8,6 +8,7 @@
   (:refer-clojure :exclude [update])
   (:require
    [app.db :as-alias db]
+   [clojure.set :as set]
    [clojure.string :as str]
    [next.jdbc.optional :as jdbc-opt]
    [next.jdbc.sql.builder :as sql]))
@@ -19,6 +20,14 @@
   {:table-fn snake-case
    :column-fn snake-case})
 
+(def params-mapping
+  {::db/return-keys? :return-keys
+   ::db/columns :columns})
+
+(defn adapt-opts
+  [opts]
+  (set/rename-keys opts params-mapping))
+
 (defn as-kebab-maps
   [rs opts]
   (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
@@ -29,7 +38,7 @@
   ([table key-map opts]
    (let [opts (merge default-opts opts)
          opts (cond-> opts
-                (:on-conflict-do-nothing opts)
+                (::db/on-conflict-do-nothing? opts)
                 (assoc :suffix "ON CONFLICT DO NOTHING"))]
      (sql/for-insert table key-map opts))))
 
@@ -44,6 +53,7 @@
   ([table where-params opts]
    (let [opts (merge default-opts opts)
          opts (cond-> opts
+                (::db/columns opts)     (assoc :columns (::db/columns opts))
                 (::db/for-update? opts) (assoc :suffix "FOR UPDATE")
                 (::db/for-share? opts)  (assoc :suffix "FOR KEY SHARE")
                 (:for-update opts)      (assoc :suffix "FOR UPDATE")
@@ -54,7 +64,13 @@
   ([table key-map where-params]
    (update table key-map where-params nil))
   ([table key-map where-params opts]
-   (let [opts (merge default-opts opts)]
+   (let [opts (into default-opts opts)
+         opts (if-let [columns (::db/columns opts)]
+                (let [columns   (if (seq columns)
+                                  (sql/as-cols columns opts)
+                                  "*")]
+                  (assoc opts :suffix (str "RETURNING " columns)))
+                opts)]
      (sql/for-update table key-map where-params opts))))
 
 (defn delete

backend/src/app/email.clj

@@ -311,6 +311,12 @@
                     ^String (::password cfg))
 
           (let [^MimeMessage message (create-smtp-message cfg session params)]
+            (l/dbg :hint "sendmail"
+                   :id (:id params)
+                   :to (:to params)
+                   :subject (str/trim (:subject params))
+                   :body (str/join "," (map :type (:body params))))
+
             (.sendMessage ^Transport transport
                           ^MimeMessage message
                           (.getAllRecipients message))))))

backend/src/app/features/components_v2.clj

@@ -0,0 +1,928 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.components-v2
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as cp]
+   [app.common.files.changes-builder :as fcb]
+   [app.common.files.helpers :as cfh]
+   [app.common.files.libraries-helpers :as cflh]
+   [app.common.files.migrations :as fmg]
+   [app.common.files.shapes-helpers :as cfsh]
+   [app.common.files.validate :as cfv]
+   [app.common.geom.point :as gpt]
+   [app.common.geom.rect :as grc]
+   [app.common.geom.shapes :as gsh]
+   [app.common.logging :as l]
+   [app.common.svg :as csvg]
+   [app.common.svg.shapes-builder :as sbuilder]
+   [app.common.types.component :as ctk]
+   [app.common.types.components-list :as ctkl]
+   [app.common.types.container :as ctn]
+   [app.common.types.file :as ctf]
+   [app.common.types.pages-list :as ctpl]
+   [app.common.types.shape :as cts]
+   [app.common.types.shape-tree :as ctst]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.features.fdata :as fdata]
+   [app.http.sse :as sse]
+   [app.media :as media]
+   [app.rpc.commands.files :as files]
+   [app.rpc.commands.files-snapshot :as fsnap]
+   [app.rpc.commands.media :as cmd.media]
+   [app.storage :as sto]
+   [app.storage.tmp :as tmp]
+   [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
+   [app.util.time :as dt]
+   [buddy.core.codecs :as bc]
+   [cuerdas.core :as str]
+   [datoteka.io :as io]
+   [promesa.exec :as px]
+   [promesa.exec.semaphore :as ps]
+   [promesa.util :as pu]))
+
+(def ^:dynamic *system* nil)
+(def ^:dynamic *stats* nil)
+(def ^:dynamic *file-stats* nil)
+(def ^:dynamic *team-stats* nil)
+(def ^:dynamic *semaphore* nil)
+(def ^:dynamic *skip-on-error* true)
+
+(def grid-gap 50)
+(def frame-gap 200)
+(def max-group-size 50)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FILE PREPARATION BEFORE MIGRATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- prepare-file-data
+  "Apply some specific migrations or fixes to things that are allowed in v1 but not in v2,
+   or that are the result of old bugs."
+  [file-data libraries]
+  (let [detached-ids  (volatile! #{})
+
+        detach-shape
+        (fn [container shape]
+          ;; Detach a shape. If it's inside a component, add it to detached-ids, for further use.
+          (let [is-component? (let [root-shape (ctst/get-shape container (:id container))]
+                                (and (some? root-shape) (nil? (:parent-id root-shape))))]
+            (when is-component?
+              (vswap! detached-ids conj (:id shape)))
+            (ctk/detach-shape shape)))
+
+        fix-orphan-shapes
+        (fn [file-data]
+          ;; Find shapes that are not listed in their parent's children list.
+          ;; Remove them, and also their children
+          (letfn [(fix-container [container]
+                    (reduce fix-shape container (ctn/shapes-seq container)))
+
+                  (fix-shape
+                    [container shape]
+                    (if-not (or (= (:id shape) uuid/zero)
+                                (nil? (:parent-id shape)))
+                      (let [parent (ctst/get-shape container (:parent-id shape))
+                            exists? (d/index-of (:shapes parent) (:id shape))]
+                        (if (nil? exists?)
+                          (let [ids (cfh/get-children-ids-with-self (:objects container) (:id shape))]
+                            (update container :objects #(reduce dissoc % ids)))
+                          container))
+                      container))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        remove-nested-roots
+        (fn [file-data]
+          ;; Remove :component-root in head shapes that are nested.
+          (letfn [(fix-container [container]
+                    (update container :objects update-vals (partial fix-shape container)))
+
+                  (fix-shape [container shape]
+                    (let [parent (ctst/get-shape container (:parent-id shape))]
+                      (if (and (ctk/instance-root? shape)
+                               (ctn/in-any-component? (:objects container) parent))
+                        (dissoc shape :component-root)
+                        shape)))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        add-not-nested-roots
+        (fn [file-data]
+          ;; Add :component-root in head shapes that are not nested.
+          (letfn [(fix-container [container]
+                    (update container :objects update-vals (partial fix-shape container)))
+
+                  (fix-shape [container shape]
+                    (let [parent (ctst/get-shape container (:parent-id shape))]
+                      (if (and (ctk/subinstance-head? shape)
+                               (not (ctn/in-any-component? (:objects container) parent)))
+                        (assoc shape :component-root true)
+                        shape)))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        fix-orphan-copies
+        (fn [file-data]
+          ;; Detach shapes that were inside a copy (have :shape-ref) but now they aren't.
+          (letfn [(fix-container [container]
+                    (update container :objects update-vals (partial fix-shape container)))
+
+                  (fix-shape [container shape]
+                    (let [parent (ctst/get-shape container (:parent-id shape))]
+                      (if (and (ctk/in-component-copy? shape)
+                               (not (ctk/instance-head? shape))
+                               (not (ctk/in-component-copy? parent)))
+                        (detach-shape container shape)
+                        shape)))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        remap-refs
+        (fn [file-data]
+          ;; Remap shape-refs so that they point to the near main.
+          ;; At the same time, if there are any dangling ref, detach the shape and its children.
+          (letfn [(fix-container [container]
+                    (reduce fix-shape container (ctn/shapes-seq container)))
+
+                  (fix-shape [container shape]
+                    (if (ctk/in-component-copy? shape)
+                      ;; First look for the direct shape.
+                      (let [root         (ctn/get-component-shape (:objects container) shape)
+                            libraries    (assoc-in libraries [(:id file-data) :data] file-data)
+                            library      (get libraries (:component-file root))
+                            component    (ctkl/get-component (:data library) (:component-id root) true)
+                            direct-shape (ctf/get-component-shape (:data library) component (:shape-ref shape))]
+                        (if (some? direct-shape)
+                          ;; If it exists, there is nothing else to do.
+                          container
+                          ;; If not found, find the near shape.
+                          (let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape))
+                                                   (ctf/get-component-shapes (:data library) component))]
+                            (if (some? near-shape)
+                              ;; If found, update the ref to point to the near shape.
+                              (ctn/update-shape container (:id shape) #(assoc % :shape-ref (:id near-shape)))
+                              ;; If not found, it may be a fostered component. Try to locate a direct shape
+                              ;; in the head component.
+                              (let [head           (ctn/get-head-shape (:objects container) shape)
+                                    library-2      (get libraries (:component-file head))
+                                    component-2    (ctkl/get-component (:data library-2) (:component-id head) true)
+                                    direct-shape-2 (ctf/get-component-shape (:data library-2) component-2 (:shape-ref shape))]
+                                (if (some? direct-shape-2)
+                                  ;; If it exists, there is nothing else to do.
+                                  container
+                                  ;; If not found, detach shape and all children (stopping if a nested instance is reached)
+                                  (let [children (ctn/get-children-in-instance (:objects container) (:id shape))]
+                                    (reduce #(ctn/update-shape %1 (:id %2) (partial detach-shape %1))
+                                            container
+                                            children))))))))
+                      container))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        fix-copies-of-detached
+        (fn [file-data]
+          ;; Find any copy that is referencing a detached shape inside a component, and
+          ;; undo the nested copy, converting it into a direct copy.
+          (letfn [(fix-container [container]
+                    (update container :objects update-vals fix-shape))
+
+                  (fix-shape [shape]
+                    (cond-> shape
+                      (@detached-ids (:shape-ref shape))
+                      (dissoc shape
+                              :component-id
+                              :component-file
+                              :component-root)))]
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        transform-to-frames
+        (fn [file-data]
+          ;; Transform component and copy heads to frames, and set the
+          ;; frame-id of its childrens
+          (letfn [(fix-container
+                    [container]
+                    (update container :objects update-vals fix-shape))
+
+                  (fix-shape
+                    [shape]
+                    (if (or (nil? (:parent-id shape)) (ctk/instance-head? shape))
+                      (assoc shape
+                        :type :frame                  ; Old groups must be converted
+                        :fills (or (:fills shape) []) ; to frames and conform to spec
+                        :hide-in-viewer (or (:hide-in-viewer shape) true)
+                        :rx (or (:rx shape) 0)
+                        :ry (or (:ry shape) 0))
+                      shape))]
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        remap-frame-ids
+        (fn [file-data]
+          ;; Remap the frame-ids of the primary childs of the head instances
+          ;; to point to the head instance.
+          (letfn [(fix-container
+                    [container]
+                    (update container :objects update-vals (partial fix-shape container)))
+
+                  (fix-shape
+                    [container shape]
+                    (let [parent (ctst/get-shape container (:parent-id shape))]
+                      (if (ctk/instance-head? parent)
+                        (assoc shape :frame-id (:id parent))
+                        shape)))]
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        fix-frame-ids
+        (fn [file-data]
+          ;; Ensure that frame-id of all shapes point to the parent or to the frame-id
+          ;; of the parent, and that the destination is indeed a frame.
+          (letfn [(fix-container [container]
+                    (update container :objects #(cfh/reduce-objects % fix-shape %)))
+
+                  (fix-shape [objects shape]
+                    (let [parent (when (:parent-id shape)
+                                   (get objects (:parent-id shape)))
+                          error? (when (some? parent)
+                                   (if (= (:type parent) :frame)
+                                     (not= (:frame-id shape) (:id parent))
+                                     (not= (:frame-id shape) (:frame-id parent))))]
+                      (if error?
+                        (let [nearest-frame (cfh/get-frame objects (:parent-id shape))
+                              frame-id      (or (:id nearest-frame) uuid/zero)]
+                          (update objects (:id shape) assoc :frame-id frame-id))
+                        objects)))]
+
+            (-> file-data
+                (update :pages-index update-vals fix-container)
+                (update :components update-vals fix-container))))
+
+        fix-component-nil-objects
+        (fn [file-data]
+          ;; Ensure that objects of all components is not null
+          (letfn [(fix-component [component]
+                    (if (and (contains? component :objects) (nil? (:objects component)))
+                      (if (:deleted component)
+                        (assoc component :objects {})
+                        (dissoc component :objects))
+                      component))]
+            (-> file-data
+                (update :components update-vals fix-component))))]
+
+    (-> file-data
+        (fix-orphan-shapes)
+        (remove-nested-roots)
+        (add-not-nested-roots)
+        (fix-orphan-copies)
+        (remap-refs)
+        (fix-copies-of-detached)
+        (transform-to-frames)
+        (remap-frame-ids)
+        (fix-frame-ids)
+        (fix-component-nil-objects))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; COMPONENTS MIGRATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- get-asset-groups
+  [assets generic-name]
+  (let [; Group by first element of the path.
+        groups (d/group-by #(first (cfh/split-path (:path %))) assets)
+
+        ; Split large groups in chunks of max-group-size elements
+        groups (loop [groups (seq groups)
+                      result {}]
+                 (if (empty? groups)
+                   result
+                   (let [[group-name assets] (first groups)
+                         group-name (if (or (nil? group-name) (str/empty? group-name))
+                                      generic-name
+                                      group-name)]
+                     (if (<= (count assets) max-group-size)
+                       (recur (next groups)
+                              (assoc result group-name assets))
+                       (let [splits (-> (partition-all max-group-size assets)
+                                        (d/enumerate))]
+                         (recur (next groups)
+                                (reduce (fn [result [index split]]
+                                          (let [split-name (str group-name " " (inc index))]
+                                            (assoc result split-name split)))
+                                        result
+                                        splits)))))))
+
+        ; Sort assets in each group by path
+        groups (update-vals groups (fn [assets]
+                                     (sort-by (fn [{:keys [path name]}]
+                                                (str/lower (cfh/merge-path-item path name)))
+                                              assets)))
+
+        ; Sort groups by name
+        groups (into (sorted-map) groups)]
+    groups))
+
+(defn- create-frame
+  [name position width height]
+  (cts/setup-shape
+   {:type :frame
+    :x (:x position)
+    :y (:y position)
+    :width (+ width (* 2 grid-gap))
+    :height (+ height (* 2 grid-gap))
+    :name name
+    :frame-id uuid/zero
+    :parent-id uuid/zero}))
+
+(defn- migrate-components
+  "If there is any component in the file library, add a new 'Library
+  backup', generate main instances for all components there and remove
+  shapes from library components.  Mark the file with
+  the :components-v2 option."
+  [file-data libraries]
+  (sse/tap {:type :migration-progress
+            :section :components})
+  (let [components (ctkl/components-seq file-data)]
+    (if (empty? components)
+      (assoc-in file-data [:options :components-v2] true)
+      (let [[file-data page-id start-pos]
+            (ctf/get-or-add-library-page file-data frame-gap)
+
+            migrate-component-shape
+            (fn [shape delta component-file component-id frame-id]
+              (cond-> shape
+                (nil? (:parent-id shape))
+                (assoc :parent-id frame-id
+                       :main-instance true
+                       :component-root true
+                       :component-file component-file
+                       :component-id component-id)
+
+                (nil? (:frame-id shape))
+                (assoc :frame-id frame-id)
+
+                :always
+                (gsh/move delta)))
+
+            add-main-instance
+            (fn [file-data component frame-id position]
+              (let [shapes (cfh/get-children-with-self (:objects component)
+                                                       (:id component))
+
+                    root-shape (first shapes)
+                    orig-pos   (gpt/point (:x root-shape) (:y root-shape))
+                    delta      (gpt/subtract position orig-pos)
+
+                    xf-shape (map #(migrate-component-shape %
+                                                            delta
+                                                            (:id file-data)
+                                                            (:id component)
+                                                            frame-id))
+                    new-shapes
+                    (into [] xf-shape shapes)
+
+                    find-frame-id ; if its parent is a frame, the frame-id should be the parent-id
+                    (fn [page shape]
+                      (let [parent (ctst/get-shape page (:parent-id shape))]
+                        (if (= :frame (:type parent))
+                          (:id parent)
+                          (:frame-id parent))))
+
+                    add-shapes
+                    (fn [page]
+                      (reduce (fn [page shape]
+                                (ctst/add-shape (:id shape)
+                                                shape
+                                                page
+                                                (find-frame-id page shape)
+                                                (:parent-id shape)
+                                                nil     ; <- As shapes are ordered, we can safely add each
+                                                true))  ;    one at the end of the parent's children list.
+                              page
+                              new-shapes))
+
+                    update-component
+                    (fn [component]
+                      (-> component
+                          (assoc :main-instance-id (:id root-shape)
+                                 :main-instance-page page-id)
+                          (dissoc :objects)))]
+
+                (-> file-data
+                    (ctpl/update-page page-id add-shapes)
+                    (ctkl/update-component (:id component) update-component))))
+
+            add-instance-grid
+            (fn [fdata frame-id grid assets]
+              (reduce (fn [result [component position]]
+                        (sse/tap {:type :migration-progress
+                                  :section :components
+                                  :name (:name component)})
+                        (add-main-instance result component frame-id (gpt/add position
+                                                                              (gpt/point grid-gap grid-gap))))
+                      fdata
+                      (d/zip assets grid)))
+
+            add-instance-grids
+            (fn [fdata]
+              (let [components (ctkl/components-seq fdata)
+                    groups     (get-asset-groups components "Components")]
+                (loop [groups   (seq groups)
+                       fdata    fdata
+                       position start-pos]
+                  (if (empty? groups)
+                    fdata
+                    (let [[group-name assets]    (first groups)
+                          grid                   (ctst/generate-shape-grid
+                                                  (map (partial ctf/get-component-root fdata) assets)
+                                                  position
+                                                  grid-gap)
+                          {:keys [width height]} (meta grid)
+                          frame                  (create-frame group-name position width height)
+                          fdata                  (ctpl/update-page fdata
+                                                                   page-id
+                                                                   #(ctst/add-shape (:id frame)
+                                                                                    frame
+                                                                                    %
+                                                                                    (:id frame)
+                                                                                    (:id frame)
+                                                                                    nil
+                                                                                    true))]
+                      (recur (next groups)
+                             (add-instance-grid fdata (:id frame) grid assets)
+                             (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))]
+
+        (let [total (count components)]
+          (some-> *stats* (swap! update :processed/components (fnil + 0) total))
+          (some-> *team-stats* (swap! update :processed/components (fnil + 0) total))
+          (some-> *file-stats* (swap! assoc :processed/components total)))
+
+        (-> file-data
+            (prepare-file-data libraries)
+            (add-instance-grids))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GRAPHICS MIGRATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- create-shapes-for-bitmap
+  "Convert a media object that contains a bitmap image into shapes,
+  one shape of type :image and one group that contains it."
+  [{:keys [name width height id mtype]} frame-id position]
+  (let [frame-shape (cts/setup-shape
+                     {:type :frame
+                      :x (:x position)
+                      :y (:y position)
+                      :width width
+                      :height height
+                      :name name
+                      :frame-id frame-id
+                      :parent-id frame-id})
+
+        img-shape   (cts/setup-shape
+                     {:type :image
+                      :x (:x position)
+                      :y (:y position)
+                      :width width
+                      :height height
+                      :metadata {:id id
+                                 :width width
+                                 :height height
+                                 :mtype mtype}
+                      :name name
+                      :frame-id (:id frame-shape)
+                      :parent-id (:id frame-shape)})]
+    [frame-shape [img-shape]]))
+
+(defn- parse-datauri
+  [data]
+  (let [[mtype b64-data] (str/split data ";base64," 2)
+        mtype (subs mtype (inc (str/index-of mtype ":")))
+        data  (-> b64-data bc/str->bytes bc/b64->bytes)]
+    [mtype data]))
+
+(defn- extract-name
+  [href]
+  (let [query-idx (d/nilv (str/last-index-of href "?") 0)
+        href (if (> query-idx 0) (subs href 0 query-idx) href)
+        filename (->> (str/split href "/") (last))
+        ext-idx (str/last-index-of filename ".")]
+    (if (> ext-idx 0) (subs filename 0 ext-idx) filename)))
+
+(defn- collect-and-persist-images
+  [svg-data file-id]
+  (letfn [(process-image [{:keys [href] :as item}]
+            (try
+              (let [item (if (str/starts-with? href "data:")
+                           (let [[mtype data] (parse-datauri href)
+                                 size         (alength data)
+                                 path         (tmp/tempfile :prefix "penpot.media.download.")
+                                 written      (io/write-to-file! data path :size size)]
+
+                             (when (not= written size)
+                               (ex/raise :type :internal
+                                         :code :mismatch-write-size
+                                         :hint "unexpected state: unable to write to file"))
+
+                             (-> item
+                                 (assoc :size size)
+                                 (assoc :path path)
+                                 (assoc :filename "tempfile")
+                                 (assoc :mtype mtype)))
+
+                           (let [result (cmd.media/download-image *system* href)]
+                             (-> (merge item result)
+                                 (assoc :name (extract-name href)))))]
+
+                ;; The media processing adds the data to the
+                ;; input map and returns it.
+                (media/run {:cmd :info :input item}))
+
+              (catch Throwable cause
+                (l/warn :hint "unexpected exception on processing internal image shape (skiping)"
+                        :cause cause)
+                (when-not *skip-on-error*
+                  (throw cause)))))
+
+          (persist-image [acc {:keys [path size width height mtype href] :as item}]
+            (let [storage (::sto/storage *system*)
+                  conn    (::db/conn *system*)
+                  hash    (sto/calculate-hash path)
+                  content (-> (sto/content path size)
+                              (sto/wrap-with-hash hash))
+                  params  {::sto/content content
+                           ::sto/deduplicate? true
+                           ::sto/touched-at (:ts item)
+                           :content-type mtype
+                           :bucket "file-media-object"}
+                  image   (sto/put-object! storage params)
+                  fmo-id  (uuid/next)]
+
+              (db/exec-one! conn
+                            [cmd.media/sql:create-file-media-object
+                             fmo-id
+                             file-id true (:name item "image")
+                             (:id image)
+                             nil
+                             width
+                             height
+                             mtype])
+
+              (assoc acc href {:id fmo-id
+                               :mtype mtype
+                               :width width
+                               :height height})))]
+
+    (let [images (->> (csvg/collect-images svg-data)
+                      (transduce (keep process-image)
+                                 (completing persist-image) {}))]
+      (assoc svg-data :image-data images))))
+
+(defn- get-svg-content
+  [id]
+  (let [storage  (::sto/storage *system*)
+        conn     (::db/conn *system*)
+        fmobject (db/get conn :file-media-object {:id id})
+        sobject  (sto/get-object storage (:media-id fmobject))]
+
+    (with-open [stream (sto/get-object-data storage sobject)]
+      (slurp stream))))
+
+(defn- create-shapes-for-svg
+  [{:keys [id] :as mobj} file-id objects frame-id position]
+  (let [svg-text  (get-svg-content id)
+
+        optimizer (::csvg/optimizer *system*)
+        svg-text  (csvg/optimize optimizer svg-text)
+
+        svg-data  (-> (csvg/parse svg-text)
+                      (assoc :name (:name mobj))
+                      (collect-and-persist-images file-id))]
+
+    (sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
+
+(defn- process-media-object
+  [fdata page-id frame-id mobj position]
+  (let [page    (ctpl/get-page fdata page-id)
+        file-id (get fdata :id)
+
+        [shape children]
+        (if (= (:mtype mobj) "image/svg+xml")
+          (create-shapes-for-svg mobj file-id (:objects page) frame-id position)
+          (create-shapes-for-bitmap mobj frame-id position))
+
+        shape (assoc shape :name (-> "Graphics"
+                                     (cfh/merge-path-item (:path mobj))
+                                     (cfh/merge-path-item (:name mobj))))
+
+        changes
+        (-> (fcb/empty-changes nil)
+            (fcb/set-save-undo? false)
+            (fcb/with-page page)
+            (fcb/with-objects (:objects page))
+            (fcb/with-library-data fdata)
+            (fcb/delete-media (:id mobj))
+            (fcb/add-objects (cons shape children)))
+
+        ;; NOTE: this is a workaround for `generate-add-component`, it
+        ;; is needed because that function always starts from empty
+        ;; changes; so in this case we need manually add all shapes to
+        ;; the page and then use that page for the
+        ;; `generate-add-component` function
+        page
+        (reduce (fn [page shape]
+                  (ctst/add-shape (:id shape)
+                                  shape
+                                  page
+                                  frame-id
+                                  frame-id
+                                  nil
+                                  true))
+                page
+                (cons shape children))
+
+        [_ _ changes2]
+        (cflh/generate-add-component nil
+                                     [shape]
+                                     (:objects page)
+                                     (:id page)
+                                     file-id
+                                     true
+                                     nil
+                                     cfsh/prepare-create-artboard-from-selection)
+        changes (fcb/concat-changes changes changes2)]
+
+    (:redo-changes changes)))
+
+(defn- create-media-grid
+  [fdata page-id frame-id grid media-group]
+  (let [factory  (px/thread-factory :virtual true)
+        executor (px/fixed-executor :parallelism 10 :factory factory)
+        process  (fn [mobj position]
+                   (let [position (gpt/add position (gpt/point grid-gap grid-gap))
+                         tp1      (dt/tpoint)]
+                     (try
+                       (process-media-object fdata page-id frame-id mobj position)
+                       (catch Throwable cause
+                         (l/wrn :hint "unable to process file media object (skiping)"
+                                :file-id (str (:id fdata))
+                                :id (str (:id mobj))
+                                :cause cause)
+
+                         (if-not *skip-on-error*
+                           (throw cause)
+                           nil))
+                       (finally
+                         (l/trc :hint "graphic processed"
+                                :file-id (str (:id fdata))
+                                :media-id (str (:id mobj))
+                                :elapsed (dt/format-duration (tp1)))))))]
+    (try
+      (->> (d/zip media-group grid)
+           (map (fn [[mobj position]]
+                  (sse/tap {:type :migration-progress
+                            :section :graphics
+                            :name (:name mobj)})
+                  (px/submit! executor (partial process mobj position))))
+           (reduce (fn [fdata promise]
+                     (if-let [changes (deref promise)]
+                       (-> (assoc-in fdata [:options :components-v2] true)
+                           (cp/process-changes changes false))
+                       fdata))
+                   fdata))
+      (finally
+        (pu/close! executor)))))
+
+(defn- migrate-graphics
+  [fdata]
+  (sse/tap {:type :migration-progress
+            :section :graphics})
+  (if (empty? (:media fdata))
+    fdata
+    (let [[fdata page-id start-pos]
+          (ctf/get-or-add-library-page fdata frame-gap)
+
+          media (->> (vals (:media fdata))
+                     (map (fn [{:keys [width height] :as media}]
+                            (let [points (-> (grc/make-rect 0 0 width height)
+                                             (grc/rect->points))]
+                              (assoc media :points points)))))
+
+          groups (get-asset-groups media "Graphics")]
+
+      (let [total (count media)]
+        (some-> *stats* (swap! update :processed/graphics (fnil + 0) total))
+        (some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total))
+        (some-> *file-stats* (swap! assoc :processed/graphics total)))
+
+      (loop [groups (seq groups)
+             fdata fdata
+             position start-pos]
+        (if (empty? groups)
+          fdata
+          (let [[group-name assets]    (first groups)
+                grid                   (ctst/generate-shape-grid assets position grid-gap)
+                {:keys [width height]} (meta grid)
+                frame                  (create-frame group-name position width height)
+                fdata                  (ctpl/update-page fdata
+                                                         page-id
+                                                         #(ctst/add-shape (:id frame)
+                                                                          frame
+                                                                          %
+                                                                          (:id frame)
+                                                                          (:id frame)
+                                                                          nil
+                                                                          true))]
+            (recur (next groups)
+                   (create-media-grid fdata page-id (:id frame) grid assets)
+                   (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap))))))))))
+
+(defn- migrate-fdata
+  [fdata libs]
+  (let [migrated? (dm/get-in fdata [:options :components-v2])]
+    (if migrated?
+      fdata
+      (let [fdata (migrate-components fdata libs)
+            fdata (migrate-graphics fdata)]
+        (update fdata :options assoc :components-v2 true)))))
+
+(defn- process-fdata
+  [fdata id]
+  (-> fdata
+      (assoc :id id)
+      (fdata/process-pointers deref)
+      (fmg/migrate-data)))
+
+(defn- process-file
+  [{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}]
+  (binding [pmap/*tracked* (pmap/create-tracked)
+            pmap/*load-fn* (partial fdata/load-pointer *system* id)]
+
+    (let [file  (binding [cfeat/*new* (atom #{})]
+                  (-> (files/get-file system id :migrate? false)
+                      (update :data process-fdata id)
+                      (update :features into (deref cfeat/*new*))
+                      (update :features cfeat/migrate-legacy-features)))
+
+          libs  (->> (files/get-file-libraries conn id)
+                     (into [file] (map (fn [{:keys [id]}]
+                                         (binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
+                                           (-> (files/get-file system id :migrate? false)
+                                               (update :data process-fdata id))))))
+                     (d/index-by :id))
+
+          pmap? (contains? (:features file) "fdata/pointer-map")
+
+          file  (-> file
+                    (update :data migrate-fdata libs)
+                    (update :features conj "components/v2")
+                    (cond-> pmap? (fdata/enable-pointer-map)))
+          ]
+
+      (when validate?
+        (if throw-on-validate?
+          (cfv/validate-file! file libs)
+          (doseq [error (cfv/validate-file file libs)]
+            (l/wrn :hint "migrate:file:validation-error"
+                   :file-id (str (:id file))
+                   :file-name (:name file)
+                   :error error))))
+
+      (db/update! conn :file
+                  {:data (blob/encode (:data file))
+                   :features (db/create-array conn "text" (:features file))
+                   :revn (:revn file)}
+                  {:id (:id file)})
+
+      (when pmap?
+        (fdata/persist-pointers! system id))
+
+      (dissoc file :data))))
+
+(defn migrate-file!
+  [system file-id & {:keys [validate? throw-on-validate?]}]
+  (let [tpoint  (dt/tpoint)
+        file-id (if (string? file-id)
+                  (parse-uuid file-id)
+                  file-id)]
+    (binding [*file-stats* (atom {})]
+      (try
+        (l/dbg :hint "migrate:file:start" :file-id (str file-id))
+
+        (let [system (update system ::sto/storage media/configure-assets-storage)]
+          (db/tx-run! system
+                      (fn [system]
+                        (binding [*system* system]
+                          (fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
+                          (process-file system file-id
+                                        :validate? validate?
+                                        :throw-on-validate? throw-on-validate?)))))
+
+        (finally
+          (let [elapsed    (tpoint)
+                components (get @*file-stats* :processed/components 0)
+                graphics   (get @*file-stats* :processed/graphics 0)]
+
+            (l/dbg :hint "migrate:file:end"
+                   :file-id (str file-id)
+                   :graphics graphics
+                   :components components
+                   :elapsed (dt/format-duration elapsed))
+
+            (some-> *stats* (swap! update :processed/files (fnil inc 0)))
+            (some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
+
+(defn migrate-team!
+  [system team-id & {:keys [validate? throw-on-validate?]}]
+  (let [tpoint  (dt/tpoint)
+        team-id (if (string? team-id)
+                  (parse-uuid team-id)
+                  team-id)]
+    (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id))
+    (binding [*team-stats* (atom {})]
+      (try
+        ;; We execute this out of transaction because we want this
+        ;; change to be visible to all other sessions before starting
+        ;; the migration
+        (let [sql (str "UPDATE team SET features = "
+                       "    array_append(features, 'ephimeral/v2-migration') "
+                       " WHERE id = ?")]
+          (db/exec-one! system [sql team-id]))
+
+        (db/tx-run! system
+                    (fn [{:keys [::db/conn] :as system}]
+                      ;; Lock the team
+                      (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
+
+                      (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
+                                                            (update :features db/decode-pgarray #{}))]
+
+                        (if (contains? features "components/v2")
+                          (l/dbg :hint "team already migrated")
+                          (let [sql  (str/concat
+                                      "SELECT f.id FROM file AS f "
+                                      "  JOIN project AS p ON (p.id = f.project_id) "
+                                      "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
+                                      "FOR UPDATE")]
+
+                            (doseq [file-id (->> (db/exec! conn [sql team-id])
+                                                 (map :id))]
+                              (migrate-file! system file-id
+                                             :validate? validate?
+                                             :throw-on-validate? throw-on-validate?))
+
+                            (let [features (-> features
+                                               (disj "ephimeral/v2-migration")
+                                               (conj "components/v2")
+                                               (conj "layout/grid")
+                                               (conj "styles/v2"))]
+                              (db/update! conn :team
+                                          {:features (db/create-array conn "text" features)}
+                                          {:id team-id})))))))
+        (finally
+          (some-> *semaphore* ps/release!)
+          (let [elapsed (tpoint)]
+            (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
+
+            ;; We execute this out of transaction because we want this
+            ;; change to be visible to all other sessions before starting
+            ;; the migration
+            (let [sql (str "UPDATE team SET features = "
+                           "    array_remove(features, 'ephimeral/v2-migration') "
+                           " WHERE id = ?")]
+              (db/exec-one! system [sql team-id]))
+
+            (let [components (get @*team-stats* :processed/components 0)
+                  graphics   (get @*team-stats* :processed/graphics 0)
+                  files      (get @*team-stats* :processed/files 0)]
+              (l/dbg :hint "migrate:team:end"
+                     :team-id (dm/str team-id)
+                     :files files
+                     :components components
+                     :graphics graphics
+                     :elapsed (dt/format-duration elapsed)))))))))
+
+

backend/src/app/features/fdata.clj

@@ -0,0 +1,96 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.fdata
+  "A `fdata/*` related feature migration helpers"
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.db :as db]
+   [app.util.blob :as blob]
+   [app.util.objects-map :as omap]
+   [app.util.pointer-map :as pmap]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OBJECTS-MAP
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn enable-objects-map
+  [file]
+  (let [update-fn #(d/update-when % :objects omap/wrap)]
+    (-> file
+        (update :data (fn [fdata]
+                        (-> fdata
+                            (update :pages-index update-vals update-fn)
+                            (update :components update-vals update-fn))))
+        (update :features conj "fdata/objects-map"))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; POINTER-MAP
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn load-pointer
+  "A database loader pointer helper"
+  [system file-id id]
+  (let [{:keys [content]} (db/get system :file-data-fragment
+                                  {:id id :file-id file-id}
+                                  {::db/columns [:content]
+                                   ::db/check-deleted? false})]
+    (when-not content
+      (ex/raise :type :internal
+                :code :fragment-not-found
+                :hint "fragment not found"
+                :file-id file-id
+                :fragment-id id))
+
+    (blob/decode content)))
+
+(defn persist-pointers!
+  "Given a database connection and the final file-id, persist all
+  pointers to the underlying storage (the database)."
+  [system file-id]
+  (doseq [[id item] @pmap/*tracked*]
+    (when (pmap/modified? item)
+      (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
+      (let [content (-> item deref blob/encode)]
+        (db/insert! system :file-data-fragment
+                    {:id id
+                     :file-id file-id
+                     :content content})))))
+
+(defn process-pointers
+  "Apply a function to all pointers on the file. Usuly used for
+  dereference the pointer to a plain value before some processing."
+  [fdata update-fn]
+  (cond-> fdata
+    (contains? fdata :pages-index)
+    (update :pages-index process-pointers update-fn)
+
+    :always
+    (update-vals (fn [val]
+                   (if (pmap/pointer-map? val)
+                     (update-fn val)
+                     val)))))
+
+(defn get-used-pointer-ids
+  "Given a file, return all pointer ids used in the data."
+  [fdata]
+  (->> (concat (vals fdata)
+               (vals (:pages-index fdata)))
+       (into #{} (comp (filter pmap/pointer-map?)
+                       (map pmap/get-id)))))
+
+(defn enable-pointer-map
+  "Enable the fdata/pointer-map feature on the file."
+  [file]
+  (-> file
+      (update :data (fn [fdata]
+                      (-> fdata
+                          (update :pages-index update-vals pmap/wrap)
+                          (update :components pmap/wrap))))
+
+      (update :features conj "fdata/pointer-map")))

backend/src/app/http.clj

@@ -23,15 +23,14 @@
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
    [app.rpc.doc :as-alias rpc.doc]
-   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
-   [yetti.adapter :as yt]
-   [yetti.request :as yrq]
-   [yetti.response :as-alias yrs]))
+   [ring.request :as rreq]
+   [ring.response :as-alias rres]
+   [yetti.adapter :as yt]))
 
 (declare router-handler)
 
@@ -63,8 +62,7 @@
                 ::max-multipart-body-size
                 ::router
                 ::handler
-                ::io-threads
-                ::wrk/executor]))
+                ::io-threads]))
 
 (defmethod ig/init-key ::server
   [_ {:keys [::handler ::router ::host ::port] :as cfg}]
@@ -75,11 +73,9 @@
                  :http/max-multipart-body-size (::max-multipart-body-size cfg)
                  :xnio/io-threads (or (::io-threads cfg)
                                       (max 3 (px/get-available-processors)))
-                 :xnio/worker-threads (or (::worker-threads cfg)
-                                          (max 6 (px/get-available-processors)))
-                 :xnio/dispatch true
-                 :socket/backlog 4069
-                 :ring/async true}
+                 :xnio/dispatch :virtual
+                 :ring/compat :ring2
+                 :socket/backlog 4069}
 
         handler (cond
                   (some? router)
@@ -102,13 +98,13 @@
   (yt/stop! server))
 
 (defn- not-found-handler
-  [_ respond _]
-  (respond {::yrs/status 404}))
+  [_]
+  {::rres/status 404})
 
 (defn- router-handler
   [router]
   (letfn [(resolve-handler [request]
-            (if-let [match (r/match-by-path router (yrq/path request))]
+            (if-let [match (r/match-by-path router (rreq/path request))]
               (let [params  (:path-params match)
                     result  (:result match)
                     handler (or (:handler result) not-found-handler)
@@ -120,18 +116,15 @@
             (let [{:keys [body] :as response} (errors/handle cause request)]
               (cond-> response
                 (map? body)
-                (-> (update ::yrs/headers assoc "content-type" "application/transit+json")
-                    (assoc ::yrs/body (t/encode-str body {:type :json-verbose}))))))]
+                (-> (update ::rres/headers assoc "content-type" "application/transit+json")
+                    (assoc ::rres/body (t/encode-str body {:type :json-verbose}))))))]
 
-    (fn [request respond _]
-      (let [handler  (resolve-handler request)
-            exchange (yrq/exchange request)]
-        (handler
-         (fn [response]
-           (yt/dispatch! exchange (partial respond response)))
-         (fn [cause]
-           (let [response (on-error cause request)]
-             (yt/dispatch! exchange (partial respond response)))))))))
+    (fn [request]
+      (let [handler (resolve-handler request)]
+        (try
+          (handler)
+          (catch Throwable cause
+            (on-error cause request)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HTTP ROUTER
@@ -160,8 +153,7 @@
                       [session/soft-auth cfg]
                       [actoken/soft-auth cfg]
                       [mw/errors errors/handle]
-                      [mw/restrict-methods]
-                      [mw/with-dispatch :vthread]]}
+                      [mw/restrict-methods]]}
 
      (::mtx/routes cfg)
      (::assets/routes cfg)

backend/src/app/http/access_token.clj

@@ -11,13 +11,13 @@
    [app.db :as db]
    [app.main :as-alias main]
    [app.tokens :as tokens]
-   [yetti.request :as yrq]))
+   [ring.request :as rreq]))
 
 (def header-re #"^Token\s+(.*)")
 
 (defn- get-token
   [request]
-  (some->> (yrq/get-header request "authorization")
+  (some->> (rreq/get-header request "authorization")
            (re-matches header-re)
            (second)))
 
@@ -30,7 +30,7 @@
   "SELECT perms, profile_id, expires_at
      FROM access_token
     WHERE id = ?
-      AND (expires_at IS NULL 
+      AND (expires_at IS NULL
            OR (expires_at > now()));")
 
 (defn- get-token-data
@@ -54,9 +54,8 @@
                 (l/trace :hint "exception on decoding malformed token" :cause cause)
                 request)))]
 
-    (fn [request respond raise]
-      (let [request (handle-request request)]
-        (handler request respond raise)))))
+    (fn [request]
+      (handler (handle-request request)))))
 
 (defn- wrap-authz
   "Authorization middleware, will be executed synchronously on vthread."

backend/src/app/http/assets.clj

@@ -16,7 +16,7 @@
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
-   [yetti.response :as-alias yrs]))
+   [ring.response :as-alias rres]))
 
 (def ^:private cache-max-age
   (dt/duration {:hours 24}))
@@ -37,11 +37,11 @@
 (defn- serve-object-from-s3
   [{:keys [::sto/storage] :as cfg} obj]
   (let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
-    {::yrs/status  307
-     ::yrs/headers {"location" (str url)
-                    "x-host"   (cond-> host port (str ":" port))
-                    "x-mtype"  (-> obj meta :content-type)
-                    "cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
+    {::rres/status  307
+     ::rres/headers {"location" (str url)
+                     "x-host"   (cond-> host port (str ":" port))
+                     "x-mtype"  (-> obj meta :content-type)
+                     "cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
 
 (defn- serve-object-from-fs
   [{:keys [::path]} obj]
@@ -51,8 +51,8 @@
         headers {"x-accel-redirect" (:path purl)
                  "content-type" (:content-type mdata)
                  "cache-control" (str "max-age=" (inst-ms cache-max-age))}]
-    {::yrs/status 204
-     ::yrs/headers headers}))
+    {::rres/status 204
+     ::rres/headers headers}))
 
 (defn- serve-object
   "Helper function that returns the appropriate response depending on
@@ -70,7 +70,7 @@
         obj (sto/get-object storage id)]
     (if obj
       (serve-object cfg obj)
-      {::yrs/status 404})))
+      {::rres/status 404})))
 
 (defn- generic-handler
   "A generic handler helper/common code for file-media based handlers."
@@ -81,7 +81,7 @@
         sobj (sto/get-object storage (kf mobj))]
     (if sobj
       (serve-object cfg sobj)
-      {::yrs/status 404})))
+      {::rres/status 404})))
 
 (defn file-objects-handler
   "Handler that serves storage objects by file media id."

backend/src/app/http/awsns.clj

@@ -20,8 +20,8 @@
    [integrant.core :as ig]
    [jsonista.core :as j]
    [promesa.exec :as px]
-   [yetti.request :as yrq]
-   [yetti.response :as-alias yrs]))
+   [ring.request :as rreq]
+   [ring.response :as-alias rres]))
 
 (declare parse-json)
 (declare handle-request)
@@ -31,15 +31,14 @@
 (defmethod ig/pre-init-spec ::routes [_]
   (s/keys :req [::http/client
                 ::main/props
-                ::db/pool
-                ::wrk/executor]))
+                ::db/pool]))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [::wrk/executor] :as cfg}]
+  [_ cfg]
   (letfn [(handler [request]
-            (let [data (-> request yrq/body slurp)]
-              (px/run! executor #(handle-request cfg data)))
-            {::yrs/status 200})]
+            (let [data (-> request rreq/body slurp)]
+              (px/run! :vthread (partial handle-request cfg data)))
+            {::rres/status 200})]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 

backend/src/app/http/client.clj

@@ -8,7 +8,6 @@
   "Http client abstraction layer."
   (:require
    [app.common.spec :as us]
-   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [java-http-clj.core :as http]
@@ -21,12 +20,11 @@
   (s/keys :req [::client]))
 
 (defmethod ig/pre-init-spec ::client [_]
-  (s/keys :req [::wrk/executor]))
+  (s/keys :req []))
 
 (defmethod ig/init-key ::client
-  [_ {:keys [::wrk/executor] :as cfg}]
-  (http/build-client {:executor executor
-                      :connect-timeout 30000 ;; 10s
+  [_ _]
+  (http/build-client {:connect-timeout 30000 ;; 10s
                       :follow-redirects :always}))
 
 (defn send!

backend/src/app/http/debug.clj

@@ -7,6 +7,7 @@
 (ns app.http.debug
   (:refer-clojure :exclude [error-handler])
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
@@ -14,9 +15,12 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http.session :as session]
+   [app.main :as-alias main]
+   [app.rpc.commands.auth :as auth]
    [app.rpc.commands.binfile :as binf]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
+   [app.srepl.helpers :as srepl]
    [app.storage :as-alias sto]
    [app.util.blob :as blob]
    [app.util.template :as tmpl]
@@ -28,55 +32,41 @@
    [integrant.core :as ig]
    [markdown.core :as md]
    [markdown.transformers :as mdt]
-   [yetti.request :as yrq]
-   [yetti.response :as yrs]))
+   [ring.request :as rreq]
+   [ring.response :as rres]))
 
 ;; (selmer.parser/cache-off!)
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; HELPERS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn authorized?
-  [pool {:keys [::session/profile-id]}]
-  (or (= "devenv" (cf/get :host))
-      (let [profile (ex/ignoring (profile/get-profile pool profile-id))
-            admins  (or (cf/get :admins) #{})]
-        (contains? admins (:email profile)))))
-
-(defn prepare-response
-  [body]
-  (let [headers {"content-type" "application/transit+json"}]
-    {::yrs/status 200
-     ::yrs/body body
-     ::yrs/headers headers}))
-
-(defn prepare-download-response
-  [body filename]
-  (let [headers {"content-disposition" (str "attachment; filename=" filename)
-                 "content-type" "application/octet-stream"}]
-    {::yrs/status 200
-     ::yrs/body body
-     ::yrs/headers headers}))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; INDEX
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn index-handler
-  [{:keys [::db/pool]} request]
-  (when-not (authorized? pool request)
-    (ex/raise :type :authentication
-              :code :only-admins-allowed))
-  {::yrs/status  200
-   ::yrs/headers {"content-type" "text/html"}
-   ::yrs/body    (-> (io/resource "app/templates/debug.tmpl")
-                     (tmpl/render {}))})
+  [_cfg _request]
+  {::rres/status  200
+   ::rres/headers {"content-type" "text/html"}
+   ::rres/body    (-> (io/resource "app/templates/debug.tmpl")
+                      (tmpl/render {}))})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE CHANGES
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn prepare-response
+  [body]
+  (let [headers {"content-type" "application/transit+json"}]
+    {::rres/status 200
+     ::rres/body body
+     ::rres/headers headers}))
+
+(defn prepare-download-response
+  [body filename]
+  (let [headers {"content-disposition" (str "attachment; filename=" filename)
+                 "content-type" "application/octet-stream"}]
+    {::rres/status 200
+     ::rres/body body
+     ::rres/headers headers}))
+
 (def sql:retrieve-range-of-changes
   "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
 
@@ -85,10 +75,6 @@
 
 (defn- retrieve-file-data
   [{:keys [::db/pool]} {:keys [params ::session/profile-id] :as request}]
-  (when-not (authorized? pool request)
-    (ex/raise :type :authentication
-              :code :only-admins-allowed))
-
   (let [file-id  (some-> params :file-id parse-uuid)
         revn     (some-> params :revn parse-long)
         filename (str file-id)]
@@ -111,15 +97,18 @@
 
         (contains? params :clone)
         (let [profile    (profile/get-profile pool profile-id)
-              project-id (:default-project-id profile)
-              data       (blob/decode data)]
-          (create-file pool {:id (uuid/next)
-                             :name (str "Cloned file: " filename)
-                             :project-id project-id
-                             :profile-id profile-id
-                             :data data})
-          {::yrs/status 201
-           ::yrs/body "OK CREATED"})
+              project-id (:default-project-id profile)]
+
+          (db/run! pool (fn [{:keys [::db/conn]}]
+                          (create-file conn {:id file-id
+                                             :name (str "Cloned file: " filename)
+                                             :project-id project-id
+                                             :profile-id profile-id})
+                          (db/update! conn :file
+                                      {:data data}
+                                      {:id file-id})
+                          {::rres/status 201
+                           ::rres/body "OK CREATED"})))
 
         :else
         (prepare-response (blob/decode data))))))
@@ -133,38 +122,41 @@
   [{:keys [::db/pool]} {:keys [::session/profile-id params] :as request}]
   (let [profile    (profile/get-profile pool profile-id)
         project-id (:default-project-id profile)
-        data       (some-> params :file :path io/read-as-bytes blob/decode)]
+        data       (some-> params :file :path io/read-as-bytes)]
 
     (if (and data project-id)
-      (let [fname      (str "Imported file *: " (dt/now))
-            overwrite? (contains? params :overwrite?)
-            file-id    (or (and overwrite? (ex/ignoring (-> params :file :filename parse-uuid)))
-                           (uuid/next))]
+      (let [fname     (str "Imported file *: " (dt/now))
+            reuse-id? (contains? params :reuseid)
+            file-id   (or (and reuse-id? (ex/ignoring (-> params :file :filename parse-uuid)))
+                          (uuid/next))]
 
-        (if (and overwrite? file-id
+        (if (and reuse-id? file-id
                  (is-file-exists? pool file-id))
           (do
             (db/update! pool :file
-                        {:data (blob/encode data)}
+                        {:data data
+                         :deleted-at nil}
                         {:id file-id})
-            {::yrs/status 200
-             ::yrs/body "OK UPDATED"})
+            {::rres/status 200
+             ::rres/body "OK UPDATED"})
 
-          (do
-            (create-file pool {:id file-id
-                               :name fname
-                               :project-id project-id
-                               :profile-id profile-id
-                               :data data})
-            {::yrs/status 201
-             ::yrs/body "OK CREATED"})))
+          (db/run! pool (fn [{:keys [::db/conn]}]
+                          (create-file conn {:id file-id
+                                             :name fname
+                                             :project-id project-id
+                                             :profile-id profile-id})
+                          (db/update! conn :file
+                                      {:data data}
+                                      {:id file-id})
+                          {::rres/status 201
+                           ::rres/body "OK CREATED"}))))
 
-      {::yrs/status 500
-       ::yrs/body "ERROR"})))
+      {::rres/status 500
+       ::rres/body "ERROR"})))
 
 (defn file-data-handler
   [cfg request]
-  (case (yrq/method request)
+  (case (rreq/method request)
     :get (retrieve-file-data cfg request)
     :post (upload-file-data cfg request)
     (ex/raise :type :http
@@ -172,10 +164,6 @@
 
 (defn file-changes-handler
   [{:keys [::db/pool]} {:keys [params] :as request}]
-  (when-not (authorized? pool request)
-    (ex/raise :type :authentication
-              :code :only-admins-allowed))
-
   (letfn [(retrieve-changes [file-id revn]
             (if (str/includes? revn ":")
               (let [[start end] (->> (str/split revn #":")
@@ -242,24 +230,19 @@
             (-> (io/resource "app/templates/error-report.v3.tmpl")
                 (tmpl/render (-> content
                                  (assoc :id id)
-                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))
-          ]
-
-    (when-not (authorized? pool request)
-      (ex/raise :type :authentication
-                :code :only-admins-allowed))
+                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))]
 
     (if-let [report (get-report request)]
       (let [result (case (:version report)
                      1 (render-template-v1 report)
                      2 (render-template-v2 report)
                      3 (render-template-v3 report))]
-        {::yrs/status 200
-         ::yrs/body result
-         ::yrs/headers {"content-type" "text/html; charset=utf-8"
-                        "x-robots-tag" "noindex"}})
-      {::yrs/status 404
-       ::yrs/body "not found"})))
+        {::rres/status 200
+         ::rres/body result
+         ::rres/headers {"content-type" "text/html; charset=utf-8"
+                         "x-robots-tag" "noindex"}})
+      {::rres/status 404
+       ::rres/body "not found"})))
 
 (def sql:error-reports
   "SELECT id, created_at,
@@ -269,17 +252,14 @@
     LIMIT 200")
 
 (defn error-list-handler
-  [{:keys [::db/pool]} request]
-  (when-not (authorized? pool request)
-    (ex/raise :type :authentication
-              :code :only-admins-allowed))
+  [{:keys [::db/pool]} _request]
   (let [items (->> (db/exec! pool [sql:error-reports])
                    (map #(update % :created-at dt/format-instant :rfc1123)))]
-    {::yrs/status 200
-     ::yrs/body (-> (io/resource "app/templates/error-list.tmpl")
-                    (tmpl/render {:items items}))
-     ::yrs/headers {"content-type" "text/html; charset=utf-8"
-                    "x-robots-tag" "noindex"}}))
+    {::rres/status 200
+     ::rres/body (-> (io/resource "app/templates/error-list.tmpl")
+                     (tmpl/render {:items items}))
+     ::rres/headers {"content-type" "text/html; charset=utf-8"
+                     "x-robots-tag" "noindex"}}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; EXPORT/IMPORT
@@ -315,14 +295,14 @@
                   ::binf/profile-id profile-id
                   ::binf/project-id project-id))
 
-          {::yrs/status  200
-           ::yrs/headers {"content-type" "text/plain"}
-           ::yrs/body    "OK CLONED"})
+          {::rres/status  200
+           ::rres/headers {"content-type" "text/plain"}
+           ::rres/body    "OK CLONED"})
 
-        {::yrs/status  200
-         ::yrs/body    (io/input-stream path)
-         ::yrs/headers {"content-type" "application/octet-stream"
-                        "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
+        {::rres/status  200
+         ::rres/body    (io/input-stream path)
+         ::rres/headers {"content-type" "application/octet-stream"
+                         "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
 
 
 
@@ -353,9 +333,96 @@
             ::binf/profile-id profile-id
             ::binf/project-id project-id))
 
-    {::yrs/status  200
-     ::yrs/headers {"content-type" "text/plain"}
-     ::yrs/body    "OK"}))
+    {::rres/status  200
+     ::rres/headers {"content-type" "text/plain"}
+     ::rres/body    "OK"}))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ACTIONS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resend-email-notification
+  [{:keys [::db/pool ::main/props] :as cfg} {:keys [params] :as request}]
+
+  (when-not (contains? params :force)
+    (ex/raise :type :validation
+              :code :missing-force
+              :hint "missing force checkbox"))
+
+  (let [profile  (some->> params :email (profile/get-profile-by-email pool))]
+
+    (when-not profile
+      (ex/raise :type :validation
+                :code :missing-profile
+                :hint "unable to find profile by email"))
+
+    (cond
+      (contains? params :block)
+      (do
+        (db/update! pool :profile {:is-blocked true} {:id (:id profile)})
+        (db/delete! pool :http-session {:profile-id (:id profile)})
+
+        {::rres/status  200
+         ::rres/headers {"content-type" "text/plain"}
+         ::rres/body    (str/ffmt "PROFILE '%' BLOCKED" (:email profile))})
+
+      (contains? params :unblock)
+      (do
+        (db/update! pool :profile {:is-blocked false} {:id (:id profile)})
+        {::rres/status  200
+         ::rres/headers {"content-type" "text/plain"}
+         ::rres/body    (str/ffmt "PROFILE '%' UNBLOCKED" (:email profile))})
+
+      (contains? params :resend)
+      (if (:is-blocked profile)
+        {::rres/status  200
+         ::rres/headers {"content-type" "text/plain"}
+         ::rres/body    "PROFILE ALREADY BLOCKED"}
+        (do
+          (auth/send-email-verification! pool props profile)
+          {::rres/status  200
+           ::rres/headers {"content-type" "text/plain"}
+           ::rres/body    (str/ffmt "RESENDED FOR '%'" (:email profile))}))
+
+      :else
+      (do
+        (db/update! pool :profile {:is-active true} {:id (:id profile)})
+        {::rres/status  200
+         ::rres/headers {"content-type" "text/plain"}
+         ::rres/body    (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))
+
+
+(defn- reset-file-data-version
+  [cfg {:keys [params] :as request}]
+  (let [file-id (some-> params :file-id d/parse-uuid)
+        version (some-> params :version d/parse-integer)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? file-id)
+      (ex/raise :type :validation
+                :code :invalid-file-id
+                :hint "provided invalid file id"))
+
+    (when (nil? version)
+      (ex/raise :type :validation
+                :code :invalid-version
+                :hint "provided invalid version"))
+
+    (srepl/update-file! cfg
+                        :id file-id
+                        :update-fn (fn [file]
+                                     (update file :data assoc :version version))
+                        :migrate? false
+                        :inc-revn? false
+                        :save? true)
+    {::rres/status  200
+     ::rres/headers {"content-type" "text/plain"}
+     ::rres/body    "OK"}))
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
@@ -366,13 +433,13 @@
   [{:keys [::db/pool]} _]
   (try
     (db/exec-one! pool ["select count(*) as count from server_prop;"])
-    {::yrs/status 200
-     ::yrs/body "OK"}
+    {::rres/status 200
+     ::rres/body "OK"}
     (catch Throwable cause
       (l/warn :hint "unable to execute query on health handler"
               :cause cause)
-      {::yrs/status 503
-       ::yrs/body "KO"})))
+      {::rres/status 503
+       ::rres/body "KO"})))
 
 (defn changelog-handler
   [_ _]
@@ -381,16 +448,23 @@
           (md->html [text]
             (md/md-to-html-string text :replacement-transformers (into [transform-emoji] mdt/transformer-vector)))]
     (if-let [clog (io/resource "changelog.md")]
-      {::yrs/status 200
-       ::yrs/headers {"content-type" "text/html; charset=utf-8"}
-       ::yrs/body (-> clog slurp md->html)}
-      {::yrs/status 404
-       ::yrs/body "NOT FOUND"})))
+      {::rres/status 200
+       ::rres/headers {"content-type" "text/html; charset=utf-8"}
+       ::rres/body (-> clog slurp md->html)}
+      {::rres/status 404
+       ::rres/body "NOT FOUND"})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; INIT
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn authorized?
+  [pool {:keys [::session/profile-id]}]
+  (or (= "devenv" (cf/get :host))
+      (let [profile (ex/ignoring (profile/get-profile pool profile-id))
+            admins  (or (cf/get :admins) #{})]
+        (contains? admins (:email profile)))))
+
 (def with-authorization
   {:compile
    (fn [& _]
@@ -414,6 +488,10 @@
     ["/changelog" {:handler (partial changelog-handler cfg)}]
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
+    ["/actions/resend-email-verification"
+     {:handler (partial resend-email-notification cfg)}]
+    ["/actions/reset-file-data-version"
+     {:handler (partial reset-file-data-version cfg)}]
     ["/file/export" {:handler (partial export-handler cfg)}]
     ["/file/import" {:handler (partial import-handler cfg)}]
     ["/file/data" {:handler (partial file-data-handler cfg)}]

backend/src/app/http/errors.clj

@@ -9,21 +9,21 @@
   (:require
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
-   [app.common.schema :as sm]
+   [app.common.schema :as-alias sm]
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
    [app.http.session :as-alias session]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [yetti.request :as yrq]
-   [yetti.response :as yrs]))
+   [ring.request :as rreq]
+   [ring.response :as rres]))
 
 (defn- parse-client-ip
   [request]
-  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
-      (yrq/get-header request "x-real-ip")
-      (yrq/remote-addr request)))
+  (or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
+      (rreq/get-header request "x-real-ip")
+      (rreq/remote-addr request)))
 
 (defn request->context
   "Extracts error report relevant context data from request."
@@ -34,184 +34,205 @@
     {:request/path       (:path request)
      :request/method     (:method request)
      :request/params     (:params request)
-     :request/user-agent (yrq/get-header request "user-agent")
+     :request/user-agent (rreq/get-header request "user-agent")
      :request/ip-addr    (parse-client-ip request)
      :request/profile-id (:uid claims)
-     :version/frontend   (or (yrq/get-header request "x-frontend-version") "unknown")
+     :version/frontend   (or (rreq/get-header request "x-frontend-version") "unknown")
      :version/backend    (:full cf/version)}))
 
+(defmulti handle-error
+  (fn [cause _ _]
+    (-> cause ex-data :type)))
+
 (defmulti handle-exception
-  (fn [err & _rest]
-    (let [edata (ex-data err)]
-      (or (:type edata)
-          (class err)))))
+  (fn [cause _ _]
+    (class cause)))
 
-(defmethod handle-exception :authentication
-  [err _]
-  {::yrs/status 401
-   ::yrs/body (ex-data err)})
+(defmethod handle-error :authentication
+  [err _ _]
+  {::rres/status 401
+   ::rres/body (ex-data err)})
 
-(defmethod handle-exception :authorization
-  [err _]
-  {::yrs/status 403
-   ::yrs/body (ex-data err)})
+(defmethod handle-error :authorization
+  [err _ _]
+  {::rres/status 403
+   ::rres/body (ex-data err)})
 
-(defmethod handle-exception :restriction
-  [err _]
-  {::yrs/status 400
-   ::yrs/body (ex-data err)})
+(defmethod handle-error :restriction
+  [err _ _]
+  {::rres/status 400
+   ::rres/body (ex-data err)})
 
-(defmethod handle-exception :rate-limit
-  [err _]
+(defmethod handle-error :rate-limit
+  [err _ _]
   (let [headers (-> err ex-data ::http/headers)]
-    {::yrs/status 429
-     ::yrs/headers headers}))
+    {::rres/status 429
+     ::rres/headers headers}))
 
-(defmethod handle-exception :concurrency-limit
-  [err _]
+(defmethod handle-error :concurrency-limit
+  [err _ _]
   (let [headers (-> err ex-data ::http/headers)]
-    {::yrs/status 429
-     ::yrs/headers headers}))
+    {::rres/status 429
+     ::rres/headers headers}))
 
-(defmethod handle-exception :validation
-  [err request]
+(defmethod handle-error :validation
+  [err request parent-cause]
   (let [{:keys [code] :as data} (ex-data err)]
     (cond
-      (= code :spec-validation)
+      (or (= code :spec-validation)
+          (= code :params-validation)
+          (= code :data-validation))
       (let [explain (ex/explain data)]
-        {::yrs/status 400
-         ::yrs/body   (-> data
-                          (dissoc ::s/problems ::s/value ::s/spec)
-                          (cond-> explain (assoc :explain explain)))})
-
-      (= code :params-validation)
-      (let [explain (::sm/explain data)
-            payload (sm/humanize-data explain)]
-        {::yrs/status 400
-         ::yrs/body   (-> data
-                          (dissoc ::sm/explain)
-                          (assoc :data payload))})
+        {::rres/status 400
+         ::rres/body   (-> data
+                           (dissoc ::s/problems ::s/value ::s/spec ::sm/explain)
+                           (cond-> explain (assoc :explain explain)))})
 
       (= code :request-body-too-large)
-      {::yrs/status 413 ::yrs/body data}
+      {::rres/status 413 ::rres/body data}
 
       (= code :invalid-image)
       (binding [l/*context* (request->context request)]
-        (l/error :hint "unexpected error on processing image" :cause err)
-        {::yrs/status 400 ::yrs/body data})
+        (let [cause (or parent-cause err)]
+          (l/error :hint "unexpected error on processing image" :cause cause)
+          {::rres/status 400 ::rres/body data}))
 
       :else
-      {::yrs/status 400 ::yrs/body data})))
+      {::rres/status 400 ::rres/body data})))
 
-(defmethod handle-exception :assertion
-  [error request]
+(defmethod handle-error :assertion
+  [error request parent-cause]
   (binding [l/*context* (request->context request)]
-    (let [{:keys [code] :as data} (ex-data error)]
+    (let [{:keys [code] :as data} (ex-data error)
+          cause (or parent-cause error)]
       (cond
         (= code :data-validation)
-        (let [explain (::sm/explain data)
-              payload (sm/humanize-data explain)]
-          (l/error :hint "Data assertion error" :message (ex-message error) :cause error)
-          {::yrs/status 500
-           ::yrs/body   {:type :server-error
-                         :code :assertion
-                         :data (-> data
-                                   (dissoc ::sm/explain)
-                                   (assoc :data payload))}})
+        (let [explain (ex/explain data)]
+          (l/error :hint "data assertion error" :cause cause)
+          {::rres/status 500
+           ::rres/body   {:type :server-error
+                          :code :assertion
+                          :data (-> data
+                                    (dissoc ::sm/explain)
+                                    (cond-> explain (assoc :explain explain)))}})
 
         (= code :spec-validation)
         (let [explain (ex/explain data)]
-          (l/error :hint "Spec assertion error" :message (ex-message error) :cause error)
-          {::yrs/status 500
-           ::yrs/body   {:type :server-error
-                         :code :assertion
-                         :data (-> data
-                                   (dissoc ::s/problems ::s/value ::s/spec)
-                                   (cond-> explain (assoc :explain explain)))}})
+          (l/error :hint "spec assertion error" :cause cause)
+          {::rres/status 500
+           ::rres/body   {:type :server-error
+                          :code :assertion
+                          :data (-> data
+                                    (dissoc ::s/problems ::s/value ::s/spec)
+                                    (cond-> explain (assoc :explain explain)))}})
 
         :else
         (do
-          (l/error :hint "Assertion error" :message (ex-message error) :cause error)
-          {::yrs/status 500
-           ::yrs/body   {:type :server-error
-                         :code :assertion
-                         :data data}})))))
+          (l/error :hint "assertion error" :cause cause)
+          {::rres/status 500
+           ::rres/body   {:type :server-error
+                          :code :assertion
+                          :data data}})))))
 
+(defmethod handle-error :not-found
+  [err _ _]
+  {::rres/status 404
+   ::rres/body (ex-data err)})
 
-(defmethod handle-exception :not-found
-  [err _]
-  {::yrs/status 404
-   ::yrs/body (ex-data err)})
-
-(defmethod handle-exception :internal
-  [error request]
+(defmethod handle-error :internal
+  [error request parent-cause]
   (binding [l/*context* (request->context request)]
-    (l/error :hint "Internal error" :message (ex-message error) :cause error)
-    {::yrs/status 500
-     ::yrs/body {:type :server-error
-                 :code :unhandled
-                 :hint (ex-message error)
-                 :data (ex-data error)}}))
+    (let [cause (or parent-cause error)]
+      (l/error :hint "internal error" :cause cause)
+      {::rres/status 500
+       ::rres/body {:type :server-error
+                    :code :unhandled
+                    :hint (ex-message error)
+                    :data (ex-data error)}})))
+
+(defmethod handle-error :default
+  [error request parent-cause]
+  (let [edata (ex-data error)]
+    ;; This is a special case for the idle-in-transaction error;
+    ;; when it happens, the connection is automatically closed and
+    ;; next-jdbc combines the two errors in a single ex-info. We
+    ;; only need the :handling error, because the :rollback error
+    ;; will be always "connection closed".
+    (if (and (ex/exception? (:rollback edata))
+             (ex/exception? (:handling edata)))
+      (handle-exception (:handling edata) request error)
+      (handle-exception error request parent-cause))))
 
 (defmethod handle-exception org.postgresql.util.PSQLException
-  [error request]
-  (let [state (.getSQLState ^java.sql.SQLException error)]
+  [error request parent-cause]
+  (let [state (.getSQLState ^java.sql.SQLException error)
+        cause (or parent-cause error)]
     (binding [l/*context* (request->context request)]
-      (l/error :hint "PSQL error" :message (ex-message error) :cause error)
+      (l/error :hint "PSQL error"
+               :cause cause)
       (cond
         (= state "57014")
-        {::yrs/status 504
-         ::yrs/body {:type :server-error
-                     :code :statement-timeout
-                     :hint (ex-message error)}}
+        {::rres/status 504
+         ::rres/body {:type :server-error
+                      :code :statement-timeout
+                      :hint (ex-message error)}}
 
         (= state "25P03")
-        {::yrs/status 504
-         ::yrs/body {:type :server-error
-                     :code :idle-in-transaction-timeout
-                     :hint (ex-message error)}}
+        {::rres/status 504
+         ::rres/body {:type :server-error
+                      :code :idle-in-transaction-timeout
+                      :hint (ex-message error)}}
 
         :else
-        {::yrs/status 500
-         ::yrs/body {:type :server-error
-                     :code :unexpected
-                     :hint (ex-message error)
-                     :state state}}))))
+        {::rres/status 500
+         ::rres/body {:type :server-error
+                      :code :unexpected
+                      :hint (ex-message error)
+                      :state state}}))))
 
 (defmethod handle-exception :default
-  [error request]
-  (let [edata (ex-data error)]
+  [error request parent-cause]
+  (let [edata (ex-data error)
+        cause (or parent-cause error)]
     (cond
       ;; This means that exception is not a controlled exception.
       (nil? edata)
       (binding [l/*context* (request->context request)]
-        (l/error :hint "Unexpected error" :message (ex-message error) :cause error)
-        {::yrs/status 500
-         ::yrs/body {:type :server-error
-                     :code :unexpected
-                     :hint (ex-message error)}})
-
-      ;; This is a special case for the idle-in-transaction error;
-      ;; when it happens, the connection is automatically closed and
-      ;; next-jdbc combines the two errors in a single ex-info. We
-      ;; only need the :handling error, because the :rollback error
-      ;; will be always "connection closed".
-      (and (ex/exception? (:rollback edata))
-           (ex/exception? (:handling edata)))
-      (handle-exception (:handling edata) request)
+        (l/error :hint "unexpected error" :cause cause)
+        {::rres/status 500
+         ::rres/body {:type :server-error
+                      :code :unexpected
+                      :hint (ex-message error)}})
 
       :else
       (binding [l/*context* (request->context request)]
-        (l/error :hint "Unhandled error" :message (ex-message error) :cause error)
-        {::yrs/status 500
-         ::yrs/body {:type :server-error
-                     :code :unhandled
-                     :hint (ex-message error)
-                     :data edata}}))))
+        (l/error :hint "unhandled error" :cause cause)
+        {::rres/status 500
+         ::rres/body {:type :server-error
+                      :code :unhandled
+                      :hint (ex-message error)
+                      :data edata}}))))
+
+(defmethod handle-exception java.util.concurrent.CompletionException
+  [cause request _]
+  (let [cause' (ex-cause cause)]
+    (if (ex/error? cause')
+      (handle-error cause' request cause)
+      (handle-exception cause' request cause))))
+
+(defmethod handle-exception java.util.concurrent.ExecutionException
+  [cause request _]
+  (let [cause' (ex-cause cause)]
+    (if (ex/error? cause')
+      (handle-error cause' request cause)
+      (handle-exception cause' request cause))))
 
 (defn handle
   [cause request]
-  (if (or (instance? java.util.concurrent.CompletionException cause)
-          (instance? java.util.concurrent.ExecutionException cause))
-    (handle-exception (ex-cause cause) request)
-    (handle-exception cause request)))
+  (if (ex/error? cause)
+    (handle-error cause request nil)
+    (handle-exception cause request nil)))
+
+(defn handle'
+  [cause request]
+  (::rres/body (handle cause request)))

backend/src/app/http/middleware.clj

@@ -12,13 +12,10 @@
    [app.config :as cf]
    [app.util.json :as json]
    [cuerdas.core :as str]
-   [promesa.core :as p]
-   [promesa.exec :as px]
-   [promesa.util :as pu]
+   [ring.request :as rreq]
+   [ring.response :as rres]
    [yetti.adapter :as yt]
-   [yetti.middleware :as ymw]
-   [yetti.request :as yrq]
-   [yetti.response :as yrs])
+   [yetti.middleware :as ymw])
   (:import
    com.fasterxml.jackson.core.JsonParseException
    com.fasterxml.jackson.core.io.JsonEOFException
@@ -46,17 +43,17 @@
 (defn wrap-parse-request
   [handler]
   (letfn [(process-request [request]
-            (let [header (yrq/get-header request "content-type")]
+            (let [header (rreq/get-header request "content-type")]
               (cond
                 (str/starts-with? header "application/transit+json")
-                (with-open [^InputStream is (yrq/body request)]
+                (with-open [^InputStream is (rreq/body request)]
                   (let [params (t/read! (t/reader is))]
                     (-> request
                         (assoc :body-params params)
                         (update :params merge params))))
 
                 (str/starts-with? header "application/json")
-                (with-open [^InputStream is (yrq/body request)]
+                (with-open [^InputStream is (rreq/body request)]
                   (let [params (json/decode is json-mapper)]
                     (-> request
                         (assoc :body-params params)
@@ -65,37 +62,36 @@
                 :else
                 request)))
 
-          (handle-error [raise cause]
+          (handle-error [cause]
             (cond
               (instance? RuntimeException cause)
               (if-let [cause (ex-cause cause)]
-                (handle-error raise cause)
-                (raise cause))
+                (handle-error cause)
+                (throw cause))
 
               (instance? RequestTooBigException cause)
-              (raise (ex/error :type :validation
-                               :code :request-body-too-large
-                               :hint (ex-message cause)))
-
+              (ex/raise :type :validation
+                        :code :request-body-too-large
+                        :hint (ex-message cause))
 
               (or (instance? JsonEOFException cause)
                   (instance? JsonParseException cause)
                   (instance? MismatchedInputException cause))
-              (raise (ex/error :type :validation
-                               :code :malformed-json
-                               :hint (ex-message cause)
-                               :cause cause))
+              (ex/raise :type :validation
+                        :code :malformed-json
+                        :hint (ex-message cause)
+                        :cause cause)
 
               :else
-              (raise cause)))]
+              (throw cause)))]
 
-    (fn [request respond raise]
-      (if (= (yrq/method request) :post)
+    (fn [request]
+      (if (= (rreq/method request) :post)
         (let [request (ex/try! (process-request request))]
           (if (ex/exception? request)
-            (handle-error raise request)
-            (handler request respond raise)))
-        (handler request respond raise)))))
+            (handle-error request)
+            (handler request)))
+        (handler request)))))
 
 (def parse-request
   {:name ::parse-request
@@ -113,7 +109,7 @@
 (defn wrap-format-response
   [handler]
   (letfn [(transit-streamable-body [data opts]
-            (reify yrs/StreamableResponseBody
+            (reify rres/StreamableResponseBody
               (-write-body-to-stream [_ _ output-stream]
                 (try
                   (with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
@@ -128,7 +124,7 @@
                     (.close ^OutputStream output-stream))))))
 
           (json-streamable-body [data]
-            (reify yrs/StreamableResponseBody
+            (reify rres/StreamableResponseBody
               (-write-body-to-stream [_ _ output-stream]
                 (try
                   (with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
@@ -143,24 +139,24 @@
                     (.close ^OutputStream output-stream))))))
 
           (format-response-with-json [response _]
-            (let [body (::yrs/body response)]
+            (let [body (::rres/body response)]
               (if (or (boolean? body) (coll? body))
                 (-> response
-                    (update ::yrs/headers assoc "content-type" "application/json")
-                    (assoc ::yrs/body (json-streamable-body body)))
+                    (update ::rres/headers assoc "content-type" "application/json")
+                    (assoc ::rres/body (json-streamable-body body)))
                 response)))
 
           (format-response-with-transit [response request]
-            (let [body (::yrs/body response)]
+            (let [body (::rres/body response)]
               (if (or (boolean? body) (coll? body))
-                (let [qs   (yrq/query request)
+                (let [qs   (rreq/query request)
                       opts (if (or (contains? cf/flags :transit-readable-response)
                                    (str/includes? qs "transit_verbose"))
                              {:type :json-verbose}
                              {:type :json})]
                   (-> response
-                      (update ::yrs/headers assoc "content-type" "application/transit+json")
-                      (assoc ::yrs/body (transit-streamable-body body opts))))
+                      (update ::rres/headers assoc "content-type" "application/transit+json")
+                      (assoc ::rres/body (transit-streamable-body body opts))))
                 response)))
 
           (format-from-params [{:keys [query-params] :as request}]
@@ -169,7 +165,7 @@
 
           (format-response [response request]
             (let [accept (or (format-from-params request)
-                             (yrq/get-header request "accept"))]
+                             (rreq/get-header request "accept"))]
               (cond
                 (or (= accept "application/transit+json")
                     (str/includes? accept "application/transit+json"))
@@ -186,11 +182,9 @@
             (cond-> response
               (map? response) (format-response request)))]
 
-    (fn [request respond raise]
-      (handler request
-               (fn [response]
-                 (respond (process-response response request)))
-               raise))))
+    (fn [request]
+      (let [response (handler request)]
+        (process-response response request)))))
 
 (def format-response
   {:name ::format-response
@@ -198,12 +192,11 @@
 
 (defn wrap-errors
   [handler on-error]
-  (fn [request respond raise]
-    (handler request respond (fn [cause]
-                               (try
-                                 (respond (on-error cause request))
-                                 (catch Throwable cause
-                                   (raise cause)))))))
+  (fn [request]
+    (try
+      (handler request)
+      (catch Throwable cause
+        (on-error cause request)))))
 
 (def errors
   {:name ::errors
@@ -221,11 +214,11 @@
 (defn wrap-cors
   [handler]
   (fn [request]
-    (let [response (if (= (yrq/method request) :options)
-                     {::yrs/status 200}
+    (let [response (if (= (rreq/method request) :options)
+                     {::rres/status 200}
                      (handler request))
-          origin   (yrq/get-header request "origin")]
-      (update response ::yrs/headers with-cors-headers origin))))
+          origin   (rreq/get-header request "origin")]
+      (update response ::rres/headers with-cors-headers origin))))
 
 (def cors
   {:name ::cors
@@ -239,18 +232,8 @@
    (fn [data _]
      (when-let [allowed (:allowed-methods data)]
        (fn [handler]
-         (fn [request respond raise]
-           (let [method (yrq/method request)]
+         (fn [request]
+           (let [method (rreq/method request)]
              (if (contains? allowed method)
-               (handler request respond raise)
-               (respond {::yrs/status 405})))))))})
-
-(def with-dispatch
-  {:name ::with-dispatch
-   :compile
-   (fn [& _]
-     (fn [handler executor]
-       (let [executor (px/resolve-executor executor)]
-         (fn [request respond raise]
-           (->> (px/submit! executor (partial handler request))
-                (p/fnly (pu/handler respond raise)))))))})
+               (handler request)
+               {::rres/status 405}))))))})

backend/src/app/http/session.clj

@@ -20,6 +20,7 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [ring.request :as rreq]
    [yetti.request :as yrq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -142,7 +143,7 @@
   (us/assert! ::us/uuid profile-id)
 
   (fn [request response]
-    (let [uagent  (yrq/get-header request "user-agent")
+    (let [uagent  (rreq/get-header request "user-agent")
           params  {:profile-id profile-id
                    :user-agent uagent
                    :created-at (dt/now)}
@@ -209,9 +210,8 @@
                 (l/trace :hint "exception on decoding malformed token" :cause cause)
                 request)))]
 
-    (fn [request respond raise]
-      (let [request (handle-request request)]
-        (handler request respond raise)))))
+    (fn [request]
+      (handler (handle-request request)))))
 
 (defn- wrap-authz
   [handler {:keys [::manager]}]
@@ -221,12 +221,15 @@
           request  (cond-> request
                      (some? session)
                      (assoc ::profile-id (:profile-id session)
-                            ::id (:id session)))]
+                            ::id (:id session)))
+          response (handler request)]
 
-      (cond-> (handler request)
-        (renew-session? session)
-        (-> (assign-auth-token-cookie session)
-            (assign-authenticated-cookie session))))))
+      (if (renew-session? session)
+        (let [session (update! manager session)]
+          (-> response
+              (assign-auth-token-cookie session)
+              (assign-authenticated-cookie session)))
+        response))))
 
 (def soft-auth
   {:name ::soft-auth

backend/src/app/http/sse.clj

@@ -0,0 +1,86 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.http.sse
+  "SSE (server sent events) helpers"
+  (:refer-clojure :exclude [tap])
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.transit :as t]
+   [app.http.errors :as errors]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [promesa.exec.csp :as sp]
+   [promesa.util :as pu]
+   [ring.response :as rres])
+  (:import
+   java.io.OutputStream))
+
+(def ^:dynamic *channel* nil)
+
+(defn- write!
+  [^OutputStream output ^bytes  data]
+  (l/trc :hint "writting data" :data data :length (alength data))
+  (.write output data)
+  (.flush output))
+
+(defn- create-writer-loop
+  [^OutputStream output]
+  (try
+    (loop []
+      (when-let [event (sp/take! *channel*)]
+        (let [result (ex/try! (write! output event))]
+          (if (ex/exception? result)
+            (l/wrn :hint "unexpected exception on sse writer" :cause result)
+            (recur)))))
+    (finally
+      (pu/close! output))))
+
+(defn- encode
+  [[name data]]
+  (try
+    (let [data (with-out-str
+                 (println "event:" (d/name name))
+                 (println "data:" (t/encode-str data {:type :json-verbose}))
+                 (println))]
+      (.getBytes data "UTF-8"))
+    (catch Throwable cause
+      (l/err :hint "unexpected error on encoding value on sse stream"
+             :cause cause)
+      nil)))
+
+;; ---- PUBLIC API
+
+(def default-headers
+  {"Content-Type" "text/event-stream;charset=UTF-8"
+   "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
+   "Pragma" "no-cache"})
+
+(defn tap
+  ([data] (tap "event" data))
+  ([name data]
+   (when-let [channel *channel*]
+     (sp/put! channel [name data])
+     nil)))
+
+(defn response
+  [handler & {:keys [buf] :or {buf 32} :as opts}]
+  (fn [request]
+    {::rres/headers default-headers
+     ::rres/status 200
+     ::rres/body (reify rres/StreamableResponseBody
+                   (-write-body-to-stream [_ _ output]
+                     (binding [*channel* (sp/chan :buf buf :xf (keep encode))]
+                       (let [writer (px/run! :virtual (partial create-writer-loop output))]
+                         (try
+                           (tap "end" (handler))
+                           (catch Throwable cause
+                             (tap "error" (errors/handle' cause request)))
+                           (finally
+                             (sp/close! *channel*)
+                             (p/await! writer)))))))}))

backend/src/app/http/websocket.clj

@@ -10,7 +10,7 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
-   [app.common.spec :as us]
+   [app.common.schema :as sm]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.http.session :as session]
@@ -21,6 +21,7 @@
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.exec.csp :as sp]
+   [ring.websocket :as rws]
    [yetti.websocket :as yws]))
 
 (def recv-labels
@@ -116,21 +117,21 @@
               :profile-id profile-id
               :session-id session-id}]
 
-      ;; Close profile subscription if exists
-      (when-let [ch (:channel psub)]
-        (sp/close! ch)
-        (mbus/purge! msgbus [ch]))
+    ;; Close profile subscription if exists
+    (when-let [ch (:channel psub)]
+      (sp/close! ch)
+      (mbus/purge! msgbus [ch]))
 
-      ;; Close team subscription if exists
-      (when-let [ch (:channel tsub)]
-        (sp/close! ch)
-        (mbus/purge! msgbus [ch]))
+    ;; Close team subscription if exists
+    (when-let [ch (:channel tsub)]
+      (sp/close! ch)
+      (mbus/purge! msgbus [ch]))
 
-      ;; Close file subscription if exists
-      (when-let [{:keys [topic channel]} fsub]
-        (sp/close! channel)
-        (mbus/purge! msgbus [channel])
-        (mbus/pub! msgbus :topic topic :message msg))))
+    ;; Close file subscription if exists
+    (when-let [{:keys [topic channel]} fsub]
+      (sp/close! channel)
+      (mbus/purge! msgbus [channel])
+      (mbus/pub! msgbus :topic topic :message msg))))
 
 (defmethod handle-message :subscribe-team
   [{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id]} {:keys [team-id] :as params}]
@@ -178,7 +179,7 @@
           (let [message {:type :presence
                          :file-id file-id
                          :session-id session-id
-                           :profile-id profile-id}]
+                         :profile-id profile-id}]
             (mbus/pub! msgbus
                        :topic file-id
                        :message message)))
@@ -277,19 +278,23 @@
             :inc 1)
   message)
 
-
-(s/def ::session-id ::us/uuid)
-(s/def ::handler-params
-  (s/keys :req-un [::session-id]))
+(def ^:private schema:params
+  (sm/define
+    [:map {:title "params"}
+     [:session-id ::sm/uuid]]))
 
 (defn- http-handler
   [cfg {:keys [params ::session/profile-id] :as request}]
-  (let [{:keys [session-id]} (us/conform ::handler-params params)]
+  (let [{:keys [session-id]} (sm/conform! schema:params params)]
     (cond
       (not profile-id)
       (ex/raise :type :authentication
                 :hint "Authentication required.")
 
+      ;; WORKAROUND: we use the adapter specific predicate for
+      ;; performance reasons; for now, the ring default impl for
+      ;; `upgrade-request?` parses all requests headers before perform
+      ;; any checking.
       (not (yws/upgrade-request? request))
       (ex/raise :type :validation
                 :code :websocket-request-expected
@@ -298,14 +303,13 @@
       :else
       (do
         (l/trace :hint "websocket request" :profile-id profile-id :session-id session-id)
-        (->> (ws/handler
-              ::ws/on-rcv-message (partial on-rcv-message cfg)
-              ::ws/on-snd-message (partial on-snd-message cfg)
-              ::ws/on-connect (partial on-connect cfg)
-              ::ws/handler (partial handle-message cfg)
-              ::profile-id profile-id
-              ::session-id session-id)
-             (yws/upgrade request))))))
+        {::rws/listener (ws/listener request
+                                     ::ws/on-rcv-message (partial on-rcv-message cfg)
+                                     ::ws/on-snd-message (partial on-snd-message cfg)
+                                     ::ws/on-connect (partial on-connect cfg)
+                                     ::ws/handler (partial handle-message cfg)
+                                     ::profile-id profile-id
+                                     ::session-id session-id)}))))
 
 (defmethod ig/pre-init-spec ::routes [_]
   (s/keys :req [::mbus/msgbus
@@ -318,5 +322,4 @@
 (defmethod ig/init-key ::routes
   [_ cfg]
   ["/ws/notifications" {:middleware [[session/authz cfg]]
-                        :handler (partial http-handler cfg)
-                        :allowed-methods #{:get}}])
+                        :handler (partial http-handler cfg)}])

backend/src/app/loggers/audit.clj

@@ -33,7 +33,7 @@
    [integrant.core :as ig]
    [lambdaisland.uri :as u]
    [promesa.exec :as px]
-   [yetti.request :as yrq]))
+   [ring.request :as rreq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -41,9 +41,9 @@
 
 (defn parse-client-ip
   [request]
-  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
-      (yrq/get-header request "x-real-ip")
-      (some-> (yrq/remote-addr request) str)))
+  (or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
+      (rreq/get-header request "x-real-ip")
+      (some-> (rreq/remote-addr request) str)))
 
 (defn extract-utm-params
   "Extracts additional data from params and namespace them under

backend/src/app/loggers/database.clj

@@ -39,33 +39,40 @@
 (defn record->report
   [{:keys [::l/context ::l/message ::l/props ::l/logger ::l/level ::l/cause] :as record}]
   (us/assert! ::l/record record)
+  (if (or (instance? java.util.concurrent.CompletionException cause)
+          (instance? java.util.concurrent.ExecutionException cause))
+    (-> record
+        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
+        (assoc ::l/cause (ex-cause cause))
+        (record->report))
 
-  (let [data (ex-data cause)
-        ctx  (-> context
-                 (assoc :tenant (cf/get :tenant))
-                 (assoc :host (cf/get :host))
-                 (assoc :public-uri (cf/get :public-uri))
-                 (assoc :logger/name logger)
-                 (assoc :logger/level level)
-                 (dissoc :request/params :value :params :data))]
-    (merge
-     {:context (-> (into (sorted-map) ctx)
-                   (pp/pprint-str :width 200 :length 50 :level 10))
-      :props   (pp/pprint-str props :width 200 :length 50)
-      :hint    (or (ex-message cause) @message)
-      :trace   (ex/format-throwable cause :data? false :explain? false :header? false :summary? false)}
+    (let [data (ex-data cause)
+          ctx  (-> context
+                   (assoc :tenant (cf/get :tenant))
+                   (assoc :host (cf/get :host))
+                   (assoc :public-uri (cf/get :public-uri))
+                   (assoc :logger/name logger)
+                   (assoc :logger/level level)
+                   (dissoc :request/params :value :params :data))]
+      (merge
+       {:context (-> (into (sorted-map) ctx)
+                     (pp/pprint-str :length 50))
+        :props   (pp/pprint-str props :length 50)
+        :hint    (or (ex-message cause) @message)
+        :trace   (or (::trace record)
+                     (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))}
 
-     (when-let [params (or (:request/params context) (:params context))]
-       {:params (pp/pprint-str params :width 200 :length 50 :level 10)})
+       (when-let [params (or (:request/params context) (:params context))]
+         {:params (pp/pprint-str params :length 30 :level 12)})
 
-     (when-let [value (:value context)]
-       {:value (pp/pprint-str value :width 200 :length 50 :level 10)})
+       (when-let [value (:value context)]
+         {:value (pp/pprint-str value :length 30 :level 12)})
 
-     (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-       {:data (pp/pprint-str data :width 200)})
+       (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
+         {:data (pp/pprint-str data :length 30 :level 12)})
 
-     (when-let [explain (ex/explain data {:level 10 :length 50})]
-       {:explain explain}))))
+       (when-let [explain (ex/explain data :length 30 :level 12)]
+         {:explain explain})))))
 
 (defn error-record?
   [{:keys [::l/level ::l/cause]}]
@@ -89,11 +96,11 @@
 
 (defmethod ig/init-key ::reporter
   [_ cfg]
-  (let [input (sp/chan :buf (sp/sliding-buffer 32)
+  (let [input (sp/chan :buf (sp/sliding-buffer 64)
                        :xf  (filter error-record?))]
     (add-watch l/log-record ::reporter #(sp/put! input %4))
 
-    (px/thread {:name "penpot/database-reporter" :virtual true}
+    (px/thread {:name "penpot/database-reporter"}
       (l/info :hint "initializing database error persistence")
       (try
         (loop []

backend/src/app/loggers/webhooks.clj

@@ -182,5 +182,4 @@
     "invalid-uri"
 
     (instance? java.net.http.HttpConnectTimeoutException cause)
-    "timeout"
-    ))
+    "timeout"))

backend/src/app/main.clj

@@ -10,6 +10,7 @@
    [app.auth.oidc :as-alias oidc]
    [app.auth.oidc.providers :as-alias oidc.providers]
    [app.common.logging :as l]
+   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.email :as-alias email]
@@ -36,8 +37,12 @@
    [app.storage.s3 :as-alias sto.s3]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
+   [cider.nrepl :refer [cider-nrepl-handler]]
+   [clojure.test :as test]
+   [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [nrepl.server :as nrepl]
    [promesa.exec :as px])
   (:gen-class))
 
@@ -155,12 +160,6 @@
    {::mdef/name "penpot_executors_running_threads"
     ::mdef/help "Current number of threads with state RUNNING."
     ::mdef/labels ["name"]
-    ::mdef/type :gauge}
-
-   :executors-queued-submissions
-   {::mdef/name "penpot_executors_queued_submissions"
-    ::mdef/help "Current number of queued submissions."
-    ::mdef/labels ["name"]
     ::mdef/type :gauge}})
 
 (def system-config
@@ -175,13 +174,12 @@
 
    ;; Default thread pool for IO operations
    ::wrk/executor
-   {::wrk/parallelism (cf/get :default-executor-parallelism
-                              (+ 3 (* (px/get-available-processors) 3)))}
+   {}
 
    ::wrk/monitor
    {::mtx/metrics  (ig/ref ::mtx/metrics)
-    ::wrk/name     "default"
-    ::wrk/executor (ig/ref ::wrk/executor)}
+    ::wrk/executor (ig/ref ::wrk/executor)
+    ::wrk/name     "default"}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -212,7 +210,7 @@
    {::db/pool (ig/ref ::db/pool)}
 
    ::http.client/client
-   {::wrk/executor (ig/ref ::wrk/executor)}
+   {}
 
    ::session/manager
    {::db/pool (ig/ref ::db/pool)}
@@ -223,14 +221,12 @@
    ::http.awsns/routes
    {::props              (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
-    ::http.client/client (ig/ref ::http.client/client)
-    ::wrk/executor       (ig/ref ::wrk/executor)}
+    ::http.client/client (ig/ref ::http.client/client)}
 
    ::http/server
    {::http/port                    (cf/get :http-server-port)
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
-    ::wrk/executor                 (ig/ref ::wrk/executor)
     ::http/io-threads              (cf/get :http-server-io-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
     ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
@@ -284,11 +280,11 @@
     ::http.ws/routes     (ig/ref ::http.ws/routes)
     ::http.awsns/routes  (ig/ref ::http.awsns/routes)}
 
-   :app.http.debug/routes
+   ::http.debug/routes
    {::db/pool         (ig/ref ::db/pool)
-    ::wrk/executor    (ig/ref ::wrk/executor)
     ::session/manager (ig/ref ::session/manager)
-    ::sto/storage     (ig/ref ::sto/storage)}
+    ::sto/storage     (ig/ref ::sto/storage)
+    ::props           (ig/ref ::setup/props)}
 
    ::http.ws/routes
    {::db/pool         (ig/ref ::db/pool)
@@ -300,12 +296,10 @@
    {::http.assets/path  (cf/get :assets-path)
     ::http.assets/cache-max-age (dt/duration {:hours 24})
     ::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
-    ::sto/storage  (ig/ref ::sto/storage)
-    ::wrk/executor (ig/ref ::wrk/executor)}
+    ::sto/storage  (ig/ref ::sto/storage)}
 
    :app.rpc/climit
-   {::mtx/metrics  (ig/ref ::mtx/metrics)
-    ::wrk/executor (ig/ref ::wrk/executor)}
+   {::mtx/metrics  (ig/ref ::mtx/metrics)}
 
    :app.rpc/rlimit
    {::wrk/executor (ig/ref ::wrk/executor)}
@@ -320,14 +314,14 @@
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
     ::rds/redis          (ig/ref ::rds/redis)
+    ::csvg/optimizer     (ig/ref ::csvg/optimizer)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
     ::setup/templates    (ig/ref ::setup/templates)
     ::props              (ig/ref ::setup/props)
 
-    :pool                (ig/ref ::db/pool)
-    }
+    :pool                (ig/ref ::db/pool)}
 
    :app.rpc.doc/routes
    {:methods (ig/ref :app.rpc/methods)}
@@ -335,7 +329,6 @@
    :app.rpc/routes
    {::rpc/methods     (ig/ref :app.rpc/methods)
     ::db/pool         (ig/ref ::db/pool)
-    ::wrk/executor    (ig/ref ::wrk/executor)
     ::session/manager (ig/ref ::session/manager)
     ::props           (ig/ref ::setup/props)}
 
@@ -410,6 +403,9 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
+   ::csvg/optimizer
+   {}
+
    ::audit.tasks/archive
    {::props              (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
@@ -434,20 +430,18 @@
 
    ::sto/storage
    {::db/pool      (ig/ref ::db/pool)
-    ::wrk/executor (ig/ref ::wrk/executor)
     ::sto/backends
     {:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
      :assets-fs (ig/ref [::assets :app.storage.fs/backend])}}
 
    [::assets :app.storage.s3/backend]
-   {::sto.s3/region   (cf/get :storage-assets-s3-region)
-    ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
-    ::sto.s3/bucket   (cf/get :storage-assets-s3-bucket)
-    ::wrk/executor    (ig/ref ::wrk/executor)}
+   {::sto.s3/region     (cf/get :storage-assets-s3-region)
+    ::sto.s3/endpoint   (cf/get :storage-assets-s3-endpoint)
+    ::sto.s3/bucket     (cf/get :storage-assets-s3-bucket)
+    ::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)}
 
    [::assets :app.storage.fs/backend]
-   {::sto.fs/directory (cf/get :storage-assets-fs-directory)}
-   })
+   {::sto.fs/directory (cf/get :storage-assets-fs-directory)}})
 
 
 (def worker-config
@@ -521,22 +515,65 @@
                                    (merge worker-config))
                                  (ig/prep)
                                  (ig/init))))
-  (l/info :hint "welcome to penpot"
-          :flags (str/join "," (map name cf/flags))
-          :worker? (contains? cf/flags :backend-worker)
-          :version (:full cf/version)))
+  (l/inf :hint "welcome to penpot"
+         :flags (str/join "," (map name cf/flags))
+         :worker? (contains? cf/flags :backend-worker)
+         :version (:full cf/version)))
 
 (defn stop
   []
   (alter-var-root #'system (fn [sys]
                              (when sys (ig/halt! sys))
                              nil)))
+(defn restart
+  []
+  (stop)
+  (repl/refresh :after 'app.main/start))
+
+(defn restart-all
+  []
+  (stop)
+  (repl/refresh-all :after 'app.main/start))
+
+(defmacro run-bench
+  [& exprs]
+  `(do
+     (require 'criterium.core)
+     (criterium.core/with-progress-reporting (crit/quick-bench (do ~@exprs) :verbose))))
+
+(defn run-tests
+  ([] (run-tests #"^backend-tests.*-test$"))
+  ([o]
+   (repl/refresh)
+   (cond
+     (instance? java.util.regex.Pattern o)
+     (test/run-all-tests o)
+
+     (symbol? o)
+     (if-let [sns (namespace o)]
+       (do (require (symbol sns))
+           (test/test-vars [(resolve o)]))
+       (test/test-ns o)))))
+
+(repl/disable-reload! (find-ns 'integrant.core))
 
 (defn -main
   [& _args]
   (try
-    (start)
+    (let [p (promise)]
+      (when (contains? cf/flags :nrepl-server)
+        (l/inf :hint "start nrepl server" :port 6064)
+        (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
+
+      (start)
+      (deref p))
     (catch Throwable cause
-      (l/error :hint (ex-message cause)
-               :cause cause)
+      (binding [*out* *err*]
+        (println "==== ERROR ===="))
+      (.printStackTrace cause)
+      (when-let [cause' (ex-cause cause)]
+        (binding [*out* *err*]
+          (println "==== CAUSE ===="))
+        (.printStackTrace cause'))
+      (px/sleep 500)
       (System/exit -1))))

backend/src/app/media.clj

@@ -14,11 +14,11 @@
    [app.common.schema.generators :as sg]
    [app.common.schema.openapi :as-alias oapi]
    [app.common.spec :as us]
+   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
-   [app.util.svg :as svg]
    [app.util.time :as dt]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
@@ -201,7 +201,7 @@
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file

backend/src/app/metrics.clj

@@ -94,7 +94,7 @@
         writer   (StringWriter.)]
     (TextFormat/write004 writer samples)
     {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
-              :body (.toString writer)}))
+     :body (.toString writer)}))
 
 
 

backend/src/app/migrations.clj

@@ -324,10 +324,20 @@
    {:name "0104-mod-file-thumbnail-table"
     :fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")}
 
+   {:name "0105-mod-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0105-mod-file-change-table.sql")}
+
    {:name "0105-mod-server-error-report-table"
     :fn (mg/resource "app/migrations/sql/0105-mod-server-error-report-table.sql")}
 
-   ])
+   {:name "0106-add-file-tagged-object-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0106-add-file-tagged-object-thumbnail-table.sql")}
+
+   {:name "0106-mod-team-table"
+    :fn (mg/resource "app/migrations/sql/0106-mod-team-table.sql")}
+
+   {:name "0107-mod-file-tagged-object-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0107-mod-file-tagged-object-thumbnail-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0105-mod-file-change-table.sql

@@ -0,0 +1,9 @@
+ALTER TABLE file_change
+  ADD COLUMN label text NULL;
+
+ALTER TABLE file_change
+  ALTER COLUMN label SET STORAGE external;
+
+CREATE INDEX file_change__label__idx
+    ON file_change (file_id, label)
+ WHERE label is not null;

backend/src/app/migrations/sql/0106-add-file-tagged-object-thumbnail-table.sql

@@ -0,0 +1,10 @@
+CREATE TABLE file_tagged_object_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
+  tag text DEFAULT 'frame',
+  object_id text NOT NULL,
+
+  media_id uuid NOT NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE,
+  created_at timestamptz NOT NULL DEFAULT now(),
+
+  PRIMARY KEY(file_id, tag, object_id)
+);

backend/src/app/migrations/sql/0106-mod-team-table.sql

@@ -0,0 +1 @@
+ALTER TABLE team ADD COLUMN features text[] NULL DEFAULT null;

backend/src/app/migrations/sql/0107-mod-file-tagged-object-thumbnail-table.sql

@@ -0,0 +1,2 @@
+CREATE INDEX file_tagged_object_thumbnail__media_id__idx
+    ON file_tagged_object_thumbnail (media_id);

backend/src/app/rpc.clj

@@ -30,12 +30,11 @@
    [app.storage :as-alias sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.core :as p]
-   [yetti.request :as yrq]
-   [yetti.response :as yrs]))
+   [ring.request :as rreq]
+   [ring.response :as rres]))
 
 (s/def ::profile-id ::us/uuid)
 
@@ -58,21 +57,23 @@
 
 (defn- handle-response
   [request result]
-  (if (fn? result)
-    (result request)
-    (let [mdata (meta result)]
-      (-> {::yrs/status  (::http/status mdata 200)
-           ::yrs/headers (::http/headers mdata {})
-           ::yrs/body    (rph/unwrap result)}
-          (handle-response-transformation request mdata)
-          (handle-before-comple-hook mdata)))))
+  (let [mdata    (meta result)
+        response (if (fn? result)
+                   (result request)
+                   (let [result (rph/unwrap result)]
+                     {::rres/status  (::http/status mdata 200)
+                      ::rres/headers (::http/headers mdata {})
+                      ::rres/body    result}))]
+    (-> response
+        (handle-response-transformation request mdata)
+        (handle-before-comple-hook mdata))))
 
 (defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
   [methods {:keys [params path-params] :as request}]
   (let [type       (keyword (:type path-params))
-        etag       (yrq/get-header request "if-none-match")
+        etag       (rreq/get-header request "if-none-match")
         profile-id (or (::session/profile-id request)
                        (::actoken/profile-id request))
 
@@ -138,17 +139,20 @@
         (f cfg (us/conform spec params)))
       f)))
 
+;; TODO: integrate with sm/define
+
 (defn- wrap-params-validation
   [_ f mdata]
   (if-let [schema (::sm/params mdata)]
-    (let [schema  (sm/schema schema)
-          valid?  (sm/validator schema)
-          explain (sm/explainer schema)
-          decode  (sm/decoder schema sm/default-transformer)]
-
+    (let [schema   (if (sm/lazy-schema? schema)
+                     schema
+                     (sm/define schema))
+          validate (sm/validator schema)
+          explain  (sm/explainer schema)
+          decode   (sm/decoder schema)]
       (fn [cfg params]
         (let [params (decode params)]
-          (if (valid? params)
+          (if (validate params)
             (f cfg params)
             (ex/raise :type :validation
                       :code :params-validation
@@ -159,13 +163,15 @@
   [_ f mdata]
   (if (contains? cf/flags :rpc-output-validation)
     (or (when-let [schema (::sm/result mdata)]
-          (let [schema  (sm/schema schema)
-                valid?  (sm/validator schema)
-                explain (sm/explainer schema)]
+          (let [schema   (if (sm/lazy-schema? schema)
+                           schema
+                           (sm/define schema))
+                validate (sm/validator schema)
+                explain  (sm/explainer schema)]
             (fn [cfg params]
               (let [response (f cfg params)]
                 (when (map? response)
-                  (when-not (valid? response)
+                  (when-not (validate response)
                     (ex/raise :type :validation
                               :code :data-validation
                               ::sm/explain (explain response))))
@@ -214,6 +220,7 @@
           'app.rpc.commands.files-share
           'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
+          'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails
           'app.rpc.commands.ldap
           'app.rpc.commands.management
@@ -236,8 +243,7 @@
                 ::ldap/provider
                 ::sto/storage
                 ::mtx/metrics
-                ::main/props
-                ::wrk/executor]
+                ::main/props]
           :opt [::climit
                 ::rlimit]
           :req-un [::db/pool]))
@@ -256,7 +262,6 @@
   (s/keys :req [::methods
                 ::db/pool
                 ::main/props
-                ::wrk/executor
                 ::session/manager]))
 
 (defmethod ig/init-key ::routes

backend/src/app/rpc/climit.clj

@@ -31,19 +31,24 @@
 
 (set! *warn-on-reflection* true)
 
+(defn- id->str
+  [id]
+  (-> (str id)
+      (subs 1)))
+
 (defn- create-bulkhead-cache
-  [{:keys [::wrk/executor]} config]
-  (letfn [(load-fn [key]
-            (let [config (get config (nth key 0))]
-              (l/trace :hint "insert into cache" :key key)
+  [config]
+  (letfn [(load-fn [[id skey]]
+            (when-let [config (get config id)]
+              (l/trc :hint "insert into cache" :id (id->str id) :key skey)
               (pbh/create :permits (or (:permits config) (:concurrency config))
                           :queue (or (:queue config) (:queue-size config))
                           :timeout (:timeout config)
-                          :executor executor
-                          :type (:type config :semaphore))))
+                          :type :semaphore)))
 
-          (on-remove [_ _ cause]
-            (l/trace :hint "evict from cache" :key key :reason (str cause)))]
+          (on-remove [key _ cause]
+            (let [[id skey] key]
+              (l/trc :hint "evict from cache" :id (id->str id) :key skey :reason (str cause))))]
 
     (cache/create :executor :same-thread
                   :on-remove on-remove
@@ -65,22 +70,21 @@
 
 (s/def ::path ::fs/path)
 (defmethod ig/pre-init-spec ::rpc/climit [_]
-  (s/keys :req [::wrk/executor ::mtx/metrics ::path]))
+  (s/keys :req [::mtx/metrics ::path]))
 
 (defmethod ig/init-key ::rpc/climit
-  [_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}]
+  [_ {:keys [::path ::mtx/metrics] :as cfg}]
   (when (contains? cf/flags :rpc-climit)
     (when-let [params (some->> path slurp edn/read-string)]
-      (l/info :hint "initializing concurrency limit" :config (str path))
+      (l/inf :hint "initializing concurrency limit" :config (str path))
       (us/verify! ::config params)
-      {::cache (create-bulkhead-cache cfg params)
+      {::cache (create-bulkhead-cache params)
        ::config params
-       ::wrk/executor executor
        ::mtx/metrics metrics})))
 
 (s/def ::cache cache/cache?)
 (s/def ::instance
-  (s/keys :req [::cache ::config ::wrk/executor]))
+  (s/keys :req [::cache ::config]))
 
 (s/def ::rpc/climit
   (s/nilable ::instance))
@@ -91,114 +95,92 @@
 
 (defn invoke!
   [cache metrics id key f]
-  (let [limiter (cache/get cache [id key])
-        tpoint  (dt/tpoint)
-        labels  (into-array String [(name id)])
+  (if-let [limiter (cache/get cache [id key])]
+    (let [tpoint  (dt/tpoint)
+          labels  (into-array String [(id->str id)])
+          wrapped (fn []
+                    (let [elapsed (tpoint)
+                          stats   (pbh/get-stats limiter)]
+                      (l/trc :hint "acquired"
+                             :id (id->str id)
+                             :key key
+                             :permits (:permits stats)
+                             :queue (:queue stats)
+                             :max-permits (:max-permits stats)
+                             :max-queue (:max-queue stats)
+                             :elapsed (dt/format-duration elapsed))
 
-        wrapped
-        (fn []
-          (let [elapsed (tpoint)
-                stats   (pbh/get-stats limiter)]
-            (l/trace :hint "executed"
-                     :id (name id)
-                     :key key
-                     :fnh (hash f)
-                     :permits (:permits stats)
-                     :queue (:queue stats)
-                     :max-permits (:max-permits stats)
-                     :max-queue (:max-queue stats)
-                     :elapsed (dt/format-duration elapsed))
+                      (mtx/run! metrics
+                                :id :rpc-climit-timing
+                                :val (inst-ms elapsed)
+                                :labels labels)
+                      (try
+                        (f)
+                        (finally
+                          (let [elapsed (tpoint)]
+                            (l/trc :hint "finished"
+                                   :id (id->str id)
+                                   :key key
+                                   :permits (:permits stats)
+                                   :queue (:queue stats)
+                                   :max-permits (:max-permits stats)
+                                   :max-queue (:max-queue stats)
+                                   :elapsed (dt/format-duration elapsed)))))))
+          measure!
+          (fn [stats]
             (mtx/run! metrics
-                      :id :rpc-climit-timing
-                      :val (inst-ms elapsed)
+                      :id :rpc-climit-queue
+                      :val (:queue stats)
                       :labels labels)
-            (try
-              (f)
-              (finally
-                (let [elapsed (tpoint)]
-                  (l/trace :hint "finished"
-                           :id (name id)
-                           :key key
-                           :fnh (hash f)
-                           :permits (:permits stats)
-                           :queue (:queue stats)
-                           :max-permits (:max-permits stats)
-                           :max-queue (:max-queue stats)
-                           :elapsed (dt/format-duration elapsed)))))))
-        measure!
-        (fn [stats]
-          (mtx/run! metrics
-                    :id :rpc-climit-queue
-                    :val (:queue stats)
-                    :labels labels)
-          (mtx/run! metrics
-                    :id :rpc-climit-permits
-                    :val (:permits stats)
-                    :labels labels))]
+            (mtx/run! metrics
+                      :id :rpc-climit-permits
+                      :val (:permits stats)
+                      :labels labels))]
 
-    (try
-      (let [stats (pbh/get-stats limiter)]
-        (measure! stats)
-        (l/trace :hint "enqueued"
-                 :id (name id)
+      (try
+        (let [stats (pbh/get-stats limiter)]
+          (measure! stats)
+          (l/trc :hint "enqueued"
+                 :id (id->str id)
                  :key key
-                 :fnh (hash f)
                  :permits (:permits stats)
                  :queue (:queue stats)
                  :max-permits (:max-permits stats)
                  :max-queue (:max-queue stats))
-        (pbh/invoke! limiter wrapped))
-      (catch ExceptionInfo cause
-        (let [{:keys [type code]} (ex-data cause)]
-          (if (= :bulkhead-error type)
-            (ex/raise :type :concurrency-limit
-                      :code code
-                      :hint "concurrency limit reached")
-            (throw cause))))
+          (pbh/invoke! limiter wrapped))
+        (catch ExceptionInfo cause
+          (let [{:keys [type code]} (ex-data cause)]
+            (if (= :bulkhead-error type)
+              (ex/raise :type :concurrency-limit
+                        :code code
+                        :hint "concurrency limit reached")
+              (throw cause))))
 
-      (finally
-        (measure! (pbh/get-stats limiter))))))
+        (finally
+          (measure! (pbh/get-stats limiter)))))
 
-
-(defn run!
-  [{:keys [::id ::cache ::mtx/metrics]} f]
-  (if (and cache id)
-    (invoke! cache metrics id nil f)
-    (f)))
-
-(defn submit!
-  [{:keys [::id ::cache ::wrk/executor ::mtx/metrics]} f]
-  (let [f (partial px/submit! executor (px/wrap-bindings f))]
-    (if (and cache id)
-      (p/await! (invoke! cache metrics id nil f))
-      (p/await! (f)))))
+    (do
+      (l/wrn :hint "unable to load limiter" :id (id->str id))
+      (f))))
 
 (defn configure
-  ([{:keys [::rpc/climit]} id]
-   (us/assert! ::rpc/climit climit)
-   (assoc climit ::id id))
-  ([{:keys [::rpc/climit]} id executor]
-   (us/assert! ::rpc/climit climit)
-   (-> climit
-       (assoc ::id id)
-       (assoc ::wrk/executor executor))))
+  [{:keys [::rpc/climit]} id]
+  (us/assert! ::rpc/climit climit)
+  (assoc climit ::id id))
 
-(defmacro with-dispatch!
-  "Dispatch blocking operation to a separated thread protected with the
-  specified concurrency limiter. If climit is not active, the function
-  will be scheduled to execute without concurrency monitoring."
-  [instance & body]
-  (if (vector? instance)
-    `(-> (app.rpc.climit/configure ~@instance)
-         (app.rpc.climit/run! (^:once fn* [] ~@body)))
-    `(run! ~instance (^:once fn* [] ~@body))))
+(defn run!
+  "Run a function in context of climit.
+  Intended to be used in virtual threads."
+  ([{:keys [::id ::cache ::mtx/metrics]} f]
+   (if (and cache id)
+     (invoke! cache metrics id nil f)
+     (f)))
 
-(defmacro with-dispatch
-  "Dispatch blocking operation to a separated thread protected with
-  the specified semaphore.
-  DEPRECATED"
-  [& params]
-  `(with-dispatch! ~@params))
+  ([{:keys [::id ::cache ::mtx/metrics]} f executor]
+   (let [f #(p/await! (px/submit! executor f))]
+     (if (and cache id)
+       (invoke! cache metrics id nil f)
+       (f)))))
 
 (def noop-fn (constantly nil))
 
@@ -207,18 +189,19 @@
   (if (and (some? climit) (some? id))
     (if-let [config (get-in climit [::config id])]
       (let [cache (::cache climit)]
-        (l/debug :hint "wrap: instrumenting method"
-                 :limit (name id)
-                 :service-name (::sv/name mdata)
-                 :timeout (:timeout config)
-                 :permits (:permits config)
-                 :queue (:queue config)
-                 :keyed? (some? key-fn))
+        (l/dbg :hint "instrumenting method"
+               :limit (id->str id)
+               :service-name (::sv/name mdata)
+               :timeout (:timeout config)
+               :permits (:permits config)
+               :queue (:queue config)
+               :keyed? (not= key-fn noop-fn))
+
         (fn [cfg params]
           (invoke! cache metrics id (key-fn params) (partial f cfg params))))
 
       (do
-        (l/warn :hint "no config found for specified queue" :id id)
+        (l/wrn :hint "no config found for specified queue" :id (id->str id))
         f))
 
     f))

backend/src/app/rpc/commands/audit.clj

@@ -64,7 +64,7 @@
    [:events [:vector schema:event]]])
 
 (sv/defmethod ::push-audit-events
-  {::climit/id :submit-audit-events-by-profile
+  {::climit/id :submit-audit-events/by-profile
    ::climit/key-fn ::rpc/profile-id
    ::sm/params schema:push-audit-events
    ::audit/skip true

backend/src/app/rpc/commands/auth.clj

@@ -10,6 +10,7 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.uuid :as uuid]
@@ -291,9 +292,12 @@
 
 (defn create-profile-rels!
   [conn {:keys [id] :as profile}]
-  (let [team (teams/create-team conn {:profile-id id
-                                      :name "Default"
-                                      :is-default true})]
+  (let [features (cfeat/get-enabled-features cf/flags)
+        team     (teams/create-team conn
+                                    {:profile-id id
+                                     :name "Default"
+                                     :features features
+                                     :is-default true})]
     (-> (db/update! conn :profile
                     {:default-team-id (:id team)
                      :default-project-id  (:default-project-id team)}

backend/src/app/rpc/commands/binfile.clj

@@ -8,38 +8,47 @@
   (:refer-clojure :exclude [assert])
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.files.features :as ffeat]
+   [app.common.features :as cfeat]
+   [app.common.files.defaults :as cfd]
+   [app.common.files.migrations :as pmg]
+   [app.common.files.validate :as fval]
    [app.common.fressian :as fres]
    [app.common.logging :as l]
-   [app.common.pages.migrations :as pmg]
+   [app.common.schema :as sm]
    [app.common.spec :as us]
+   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.features.components-v2 :as feat.compv2]
+   [app.features.fdata :as feat.fdata]
+   [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.helpers :as rph]
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [app.worker :as-alias wrk]
+   [clojure.set :as set]
    [clojure.spec.alpha :as s]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
    [datoteka.io :as io]
-   [yetti.adapter :as yt]
-   [yetti.response :as yrs])
+   [promesa.core :as p]
+   [promesa.util :as pu]
+   [ring.response :as rres]
+   [yetti.adapter :as yt])
   (:import
    com.github.luben.zstd.ZstdInputStream
    com.github.luben.zstd.ZstdOutputStream
@@ -296,29 +305,25 @@
 
 (defn- get-files
   [cfg ids]
-  (letfn [(get-files* [{:keys [::db/conn]}]
-            (let [sql (str "SELECT id FROM file "
-                           " WHERE id = ANY(?) ")
-                  ids (db/create-array conn "uuid" ids)]
-              (->> (db/exec! conn [sql ids])
-                   (into [] (map :id))
-                   (not-empty))))]
-
-    (db/run! cfg get-files*)))
+  (db/run! cfg (fn [{:keys [::db/conn]}]
+                 (let [sql (str "SELECT id FROM file "
+                                " WHERE id = ANY(?) ")
+                       ids (db/create-array conn "uuid" ids)]
+                   (->> (db/exec! conn [sql ids])
+                        (into [] (map :id))
+                        (not-empty))))))
 
 (defn- get-file
   [cfg file-id]
-  (letfn [(get-file* [{:keys [::db/conn]}]
-            (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
-              (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false})
-                      (files/decode-row)
-                      (files/process-pointers deref))))]
-
-    (db/run! cfg get-file*)))
+  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                 (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+                   (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false})
+                           (files/decode-row)
+                           (update :data feat.fdata/process-pointers deref))))))
 
 (defn- get-file-media
   [{:keys [::db/pool]} {:keys [data id] :as file}]
-  (dm/with-open [conn (db/open pool)]
+  (pu/with-open [conn (db/open pool)]
     (let [ids (app.tasks.file-gc/collect-used-media data)
           ids (db/create-array conn "uuid" ids)
           sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")]
@@ -330,6 +335,14 @@
       (->> (db/exec! conn [sql ids])
            (mapv #(assoc % :file-id id))))))
 
+(defn- get-file-thumbnails
+  "Return all file thumbnails for a given file."
+  [{:keys [::db/pool]} id]
+  (pu/with-open [conn (db/open pool)]
+    (let [sql "SELECT * FROM file_tagged_object_thumbnail WHERE file_id = ?"]
+      (->> (db/exec! conn [sql id])
+           (mapv #(dissoc % :file-id))))))
+
 (def ^:private storage-object-id-xf
   (comp
    (mapcat (juxt :media-id :thumbnail-id))
@@ -352,7 +365,7 @@
 
 (defn- get-libraries
   [{:keys [::db/pool]} ids]
-  (dm/with-open [conn (db/open pool)]
+  (pu/with-open [conn (db/open pool)]
     (let [ids (db/create-array conn "uuid" ids)]
       (map :id (db/exec! pool [sql:file-libraries ids])))))
 
@@ -364,7 +377,7 @@
                                 " WHERE flr.file_id = ANY(?)")]
                    (db/exec! conn [sql ids])))))
 
-(defn- create-or-update-file
+(defn- create-or-update-file!
   [conn params]
   (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
                  "VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
@@ -382,8 +395,8 @@
 
 ;; --- GENERAL PURPOSE DYNAMIC VARS
 
-(def ^:dynamic *state*)
-(def ^:dynamic *options*)
+(def ^:dynamic *state* nil)
+(def ^:dynamic *options* nil)
 
 ;; --- EXPORT WRITER
 
@@ -471,19 +484,19 @@
 (defmethod write-export :default
   [{:keys [::output] :as options}]
   (write-header! output :v1)
-  (with-open [output (zstd-output-stream output :level 12)]
-    (with-open [output (io/data-output-stream output)]
-      (binding [*state* (volatile! {})]
-        (run! (fn [section]
-                (l/debug :hint "write section" :section section ::l/sync? true)
-                (write-label! output section)
-                (let [options (-> options
-                                  (assoc ::output output)
-                                  (assoc ::section section))]
-                  (binding [*options* options]
-                    (write-section options))))
+  (pu/with-open [output (zstd-output-stream output :level 12)
+                 output (io/data-output-stream output)]
+    (binding [*state* (volatile! {})]
+      (run! (fn [section]
+              (l/dbg :hint "write section" :section section ::l/sync? true)
+              (write-label! output section)
+              (let [options (-> options
+                                (assoc ::output output)
+                                (assoc ::section section))]
+                (binding [*options* options]
+                  (write-section options))))
 
-              [:v1/metadata :v1/files :v1/rels :v1/sobjects])))))
+            [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
 
 (defmethod write-section :v1/metadata
   [{:keys [::output ::file-ids ::include-libraries?] :as cfg}]
@@ -498,39 +511,54 @@
               :hint "unable to retrieve files for export")))
 
 (defmethod write-section :v1/files
-  [{:keys [::output ::embed-assets?] :as cfg}]
+  [{:keys [::output ::embed-assets? ::include-libraries?] :as cfg}]
 
   ;; Initialize SIDS with empty vector
   (vswap! *state* assoc :sids [])
 
   (doseq [file-id (-> *state* deref :files)]
-    (let [file  (cond-> (get-file cfg file-id)
-                  embed-assets?
-                  (update :data embed-file-assets cfg file-id))
+    (let [detach?    (and (not embed-assets?) (not include-libraries?))
+          thumbnails (get-file-thumbnails cfg file-id)
+          file       (cond-> (get-file cfg file-id)
+                       detach?
+                       (-> (ctf/detach-external-references file-id)
+                           (dissoc :libraries))
 
-          media (get-file-media cfg file)]
+                       embed-assets?
+                       (update :data embed-file-assets cfg file-id)
 
-      (l/debug :hint "write penpot file"
-               :id file-id
-               :name (:name file)
-               :media (count media)
-               ::l/sync? true)
+                       :always
+                       (assoc :thumbnails thumbnails))
+
+          media      (get-file-media cfg file)]
+
+      (l/dbg :hint "write penpot file"
+             :id (str file-id)
+             :name (:name file)
+             :thumbnails (count thumbnails)
+             :features (:features file)
+             :media (count media)
+             ::l/sync? true)
 
       (doseq [item media]
-        (l/debug :hint "write penpot file media object" :id (:id item) ::l/sync? true))
+        (l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true))
+
+      (doseq [item thumbnails]
+        (l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true))
 
       (doto output
         (write-obj! file)
         (write-obj! media))
 
-      (vswap! *state* update :sids into storage-object-id-xf media))))
+      (vswap! *state* update :sids into storage-object-id-xf media)
+      (vswap! *state* update :sids into storage-object-id-xf thumbnails))))
 
 (defmethod write-section :v1/rels
   [{:keys [::output ::include-libraries?] :as cfg}]
   (let [ids  (-> *state* deref :files)
         rels (when include-libraries?
                (get-library-relations cfg ids))]
-    (l/debug :hint "found rels" :total (count rels) ::l/sync? true)
+    (l/dbg :hint "found rels" :total (count rels) ::l/sync? true)
     (write-obj! output rels)))
 
 (defmethod write-section :v1/sobjects
@@ -538,21 +566,22 @@
   (let [sids    (-> *state* deref :sids)
         storage (media/configure-assets-storage storage)]
 
-    (l/debug :hint "found sobjects"
-             :items (count sids)
-             ::l/sync? true)
+    (l/dbg :hint "found sobjects"
+           :items (count sids)
+           ::l/sync? true)
 
     ;; Write all collected storage objects
     (write-obj! output sids)
 
     (doseq [id sids]
       (let [{:keys [size] :as obj} (sto/get-object storage id)]
-        (l/debug :hint "write sobject" :id id ::l/sync? true)
+        (l/dbg :hint "write sobject" :id (str id) ::l/sync? true)
+
         (doto output
           (write-uuid! id)
           (write-obj! (meta obj)))
 
-        (with-open [^InputStream stream (sto/get-object-data storage obj)]
+        (pu/with-open [stream (sto/get-object-data storage obj)]
           (let [written (write-stream! output stream size)]
             (when (not= written size)
               (ex/raise :type :validation
@@ -569,15 +598,16 @@
 (defmulti read-import ::version)
 (defmulti read-section ::section)
 
+(s/def ::profile-id ::us/uuid)
 (s/def ::project-id ::us/uuid)
 (s/def ::input io/input-stream?)
 (s/def ::overwrite? (s/nilable ::us/boolean))
-(s/def ::migrate? (s/nilable ::us/boolean))
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
 
+;; FIXME: replace with schema
 (s/def ::read-import-options
-  (s/keys :req [::db/pool ::sto/storage ::project-id ::input]
-          :opt [::overwrite? ::migrate? ::ignore-index-errors?]))
+  (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
+          :opt [::overwrite? ::ignore-index-errors?]))
 
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
@@ -587,9 +617,6 @@
   `::overwrite?`: if true, instead of creating new files and remapping id references,
   it reuses all ids and updates existing objects; defaults to `false`.
 
-  `::migrate?`: if true, applies the migration before persisting the
-  file data; defaults to `false`.
-
   `::ignore-index-errors?`: if true, do not fail on index lookup errors, can
   happen with broken files; defaults to: `false`.
   "
@@ -599,53 +626,115 @@
   (let [version (read-header! input)]
     (read-import (assoc options ::version version ::timestamp timestamp))))
 
-(defmethod read-import :v1
-  [{:keys [::db/pool ::input] :as options}]
-  (with-open [input (zstd-input-stream input)]
-    (with-open [input (io/data-input-stream input)]
-      (db/with-atomic [conn pool]
-        (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
-        (binding [*state* (volatile! {:media [] :index {}})]
-          (run! (fn [section]
-                  (l/debug :hint "reading section" :section section ::l/sync? true)
-                  (assert-read-label! input section)
-                  (let [options (-> options
-                                    (assoc ::section section)
-                                    (assoc ::input input)
-                                    (assoc ::db/conn conn))]
-                    (binding [*options* options]
-                      (read-section options))))
-                [:v1/metadata :v1/files :v1/rels :v1/sobjects])
+(defn- read-import-v1
+  [{:keys [::db/conn ::project-id ::profile-id ::input] :as options}]
+  (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
+  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
 
-          ;; Knowing that the ids of the created files are in
-          ;; index, just lookup them and return it as a set
-          (let [files (-> *state* deref :files)]
-            (into #{} (keep #(get-in @*state* [:index %])) files)))))))
+  (pu/with-open [input (zstd-input-stream input)
+                 input (io/data-input-stream input)]
+    (binding [*state* (volatile! {:media [] :index {}})]
+      (let [team      (teams/get-team conn
+                                      :profile-id profile-id
+                                      :project-id project-id)
+
+            validate? (contains? cf/flags :file-validation)
+            features  (cfeat/get-team-enabled-features cf/flags team)]
+
+        (sse/tap {:type :import-progress
+                  :section :read-import})
+
+        ;; Process all sections
+        (run! (fn [section]
+                (l/dbg :hint "reading section" :section section ::l/sync? true)
+                (assert-read-label! input section)
+                (let [options (-> options
+                                  (assoc ::enabled-features features)
+                                  (assoc ::section section)
+                                  (assoc ::input input))]
+                  (binding [*options* options]
+                    (sse/tap {:type :import-progress
+                              :section section})
+                    (read-section options))))
+              [:v1/metadata :v1/files :v1/rels :v1/sobjects])
+
+        ;; Run all pending migrations
+        (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
+          (case feature
+            "components/v2"
+            (feat.compv2/migrate-file! options file-id
+                                       :validate? validate?
+                                       :throw-on-validate? true)
+
+            "fdata/shape-data-type"
+            nil
+
+            (ex/raise :type :internal
+                      :code :no-migration-defined
+                      :hint (str/ffmt "no migation for feature '%' on file importation" feature)
+                      :feature feature)))
+
+        ;; Knowing that the ids of the created files are in index,
+        ;; just lookup them and return it as a set
+        (let [files (-> *state* deref :files)]
+          (into #{} (keep #(get-in @*state* [:index %])) files))))))
+
+(defmethod read-import :v1
+  [options]
+  (db/tx-run! options read-import-v1))
 
 (defmethod read-section :v1/metadata
   [{:keys [::input]}]
   (let [{:keys [version files]} (read-obj! input)]
-    (l/debug :hint "metadata readed" :version (:full version) :files files ::l/sync? true)
+    (l/dbg :hint "metadata readed"
+           :version (:full version)
+           :files (mapv str files)
+           ::l/sync? true)
     (vswap! *state* update :index update-index files)
     (vswap! *state* assoc :version version :files files)))
 
 (defn- postprocess-file
-  [data]
-  (let [omap-wrap ffeat/*wrap-with-objects-map-fn*
-        pmap-wrap ffeat/*wrap-with-pointer-map-fn*]
-    (-> data
-        (update :pages-index update-vals #(update % :objects omap-wrap))
-        (update :pages-index update-vals pmap-wrap)
-        (update :components update-vals #(d/update-when % :objects omap-wrap))
-        (update :components pmap-wrap))))
+  [file]
+  (cond-> file
+    (and (contains? cfeat/*current* "fdata/objects-map")
+         (not (contains? cfeat/*previous* "fdata/objects-map")))
+    (feat.fdata/enable-objects-map)
+
+    (and (contains? cfeat/*current* "fdata/pointer-map")
+         (not (contains? cfeat/*previous* "fdata/pointer-map")))
+    (feat.fdata/enable-pointer-map)))
+
+(defn- get-remaped-thumbnails
+  [thumbnails file-id]
+  (mapv (fn [thumbnail]
+          (-> thumbnail
+              (assoc :file-id file-id)
+              (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/")))))
+        thumbnails))
 
 (defmethod read-section :v1/files
-  [{:keys [::db/conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
+  [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}]
+
   (doseq [expected-file-id (-> *state* deref :files)]
-    (let [file     (read-obj! input)
-          media'   (read-obj! input)
-          file-id  (:id file)
-          features (files/get-default-features)]
+    (let [file       (read-obj! input)
+          media      (read-obj! input)
+
+          file-id    (:id file)
+          file-id'   (lookup-index file-id)
+
+          thumbnails (:thumbnails file)
+          file       (update file :features cfeat/migrate-legacy-features)
+
+          features   (-> enabled-features
+                         (set/difference cfeat/frontend-only-features)
+                         (set/union (cfeat/check-supported-features! (:features file))))]
+
+      ;; All features that are enabled and requires explicit migration
+      ;; are added to the state for a posterior migration step
+      (doseq [feature (-> enabled-features
+                          (set/difference cfeat/no-migration-features)
+                          (set/difference (:features file)))]
+        (vswap! *state* update :pending-to-migrate (fnil conj []) [feature file-id']))
 
       (when (not= file-id expected-file-id)
         (ex/raise :type :validation
@@ -654,50 +743,85 @@
                   :expected-id expected-file-id
                   :hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
 
-      ;; Update index using with media
-      (l/debug :hint "update index with media" ::l/sync? true)
-      (vswap! *state* update :index update-index (map :id media'))
+      (l/dbg :hint "processing file"
+             :id (str file-id)
+             :features (:features file)
+             :version (-> file :data :version)
+             :media (count media)
+             :thumbnails (count thumbnails)
+             ::l/sync? true)
 
-      ;; Store file media for later insertion
-      (l/debug :hint "update media references" ::l/sync? true)
-      (vswap! *state* update :media into (map #(update % :id lookup-index)) media')
+      (when (seq thumbnails)
+        (let [thumbnails (get-remaped-thumbnails thumbnails file-id')]
+          (l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true)
+          (vswap! *state* update :thumbnails (fnil into []) thumbnails)))
 
-      (l/debug :hint "processing file" :file-id file-id ::features features ::l/sync? true)
+      (when (seq media)
+        ;; Update index with media
+        (l/dbg :hint "update index with media" :total (count media) ::l/sync? true)
+        (vswap! *state* update :index update-index (map :id media))
 
-      (binding [ffeat/*current* features
-                ffeat/*wrap-with-objects-map-fn* (if (features "storage/objects-map") omap/wrap identity)
-                ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)
+        ;; Store file media for later insertion
+        (l/dbg :hint "update media references" ::l/sync? true)
+        (vswap! *state* update :media into (map #(update % :id lookup-index)) media))
+
+      (binding [cfeat/*current* features
+                cfeat/*previous* (:features file)
                 pmap/*tracked* (atom {})]
 
-        (let [file-id' (lookup-index file-id)
-              data     (-> (:data file)
-                           (assoc :id file-id')
-                           (cond-> migrate? (pmg/migrate-data))
-                           (update :pages-index relink-shapes)
-                           (update :components relink-shapes)
-                           (update :media relink-media)
-                           (postprocess-file))
+        (let [params (-> file
+                         (assoc :id file-id')
+                         (assoc :features features)
+                         (assoc :project-id project-id)
+                         (assoc :created-at timestamp)
+                         (assoc :modified-at timestamp)
+                         (dissoc :thumbnails)
+                         (update :data (fn [data]
+                                         (-> data
+                                             (dissoc :recent-colors)
+                                             (assoc :id file-id')
+                                             (cond-> (> (:version data) cfd/version)
+                                               (assoc :version cfd/version))
 
-              params  {:id file-id'
-                       :project-id project-id
-                       :features (db/create-array conn "text" features)
-                       :name (:name file)
-                       :revn (:revn file)
-                       :is-shared (:is-shared file)
-                       :data (blob/encode data)
-                       :created-at timestamp
-                       :modified-at timestamp}]
+                                             ;; FIXME: We're temporarily activating all
+                                             ;; migrations because a problem in the
+                                             ;; environments messed up with the version
+                                             ;; numbers When this problem is fixed delete
+                                             ;; the following line
+                                             (assoc :version 22)
+                                             (update :pages-index relink-shapes)
+                                             (update :components relink-shapes)
+                                             (update :media relink-media)
+                                             (pmg/migrate-data)
+                                             (d/without-nils)))))
 
-          (l/debug :hint "create file" :id file-id' ::l/sync? true)
+              params (if (contains? cf/flags :file-schema-validation)
+                       (fval/validate-file-schema! params)
+                       params)
+
+              _      (when (contains? cf/flags :soft-file-schema-validation)
+                       (try
+                         (fval/validate-file-schema! params)
+                         (catch Throwable cause
+                           (l/error :hint "file schema validation error" :cause cause))))
+
+              params (-> params
+                         (postprocess-file)
+                         (update :features #(db/create-array conn "text" %))
+                         (update :data blob/encode))]
+
+          (l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
 
           (if overwrite?
-            (create-or-update-file conn params)
+            (create-or-update-file! conn params)
             (db/insert! conn :file params))
 
-          (files/persist-pointers! conn file-id')
+          (feat.fdata/persist-pointers! system file-id')
 
           (when overwrite?
-            (db/delete! conn :file-thumbnail {:file-id file-id'})))))))
+            (db/delete! conn :file-thumbnail {:file-id file-id'}))
+
+          file-id')))))
 
 (defmethod read-section :v1/rels
   [{:keys [::db/conn ::input ::timestamp]}]
@@ -712,10 +836,10 @@
 
         (if (contains? ids library-file-id)
           (do
-            (l/debug :hint "create file library link"
-                     :file-id (:file-id rel)
-                     :lib-id (:library-file-id rel)
-                     ::l/sync? true)
+            (l/dbg :hint "create file library link"
+                   :file-id (:file-id rel)
+                   :lib-id (:library-file-id rel)
+                   ::l/sync? true)
             (db/insert! conn :file-library-rel rel))
 
           (l/warn :hint "ignoring file library link"
@@ -724,9 +848,10 @@
                   ::l/sync? true))))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::sto/storage ::db/conn ::input ::overwrite?]}]
+  [{:keys [::sto/storage ::db/conn ::input ::overwrite? ::timestamp]}]
   (let [storage (media/configure-assets-storage storage)
-        ids     (read-obj! input)]
+        ids     (read-obj! input)
+        thumb?  (into #{} (map :media-id) (:thumbnails @*state*))]
 
     (doseq [expected-storage-id ids]
       (let [id    (read-uuid! input)
@@ -737,43 +862,62 @@
                     :code :inconsistent-penpot-file
                     :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
 
-        (l/debug :hint "readed storage object" :id id ::l/sync? true)
+        (l/dbg :hint "readed storage object" :id (str id) ::l/sync? true)
 
         (let [[size resource] (read-stream! input)
               hash            (sto/calculate-hash resource)
               content         (-> (sto/content resource size)
                                   (sto/wrap-with-hash hash))
+
               params          (-> mdata
-                                  (assoc ::sto/deduplicate? true)
                                   (assoc ::sto/content content)
-                                  (assoc ::sto/touched-at (dt/now))
-                                  (assoc :bucket "file-media-object"))
+                                  (assoc ::sto/deduplicate? true)
+                                  (assoc ::sto/touched-at timestamp))
+
+              params          (if (thumb? id)
+                                (assoc params :bucket "file-object-thumbnail")
+                                (assoc params :bucket "file-media-object"))
 
               sobject         (sto/put-object! storage params)]
 
-          (l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/sync? true)
+          (l/dbg :hint "persisted storage object"
+                 :old-id (str id)
+                 :new-id (str (:id sobject))
+                 :is-thumbnail (boolean (thumb? id))
+                 ::l/sync? true)
+
           (vswap! *state* update :index assoc id (:id sobject)))))
 
     (doseq [item (:media @*state*)]
-      (l/debug :hint "inserting file media object"
-               :id (:id item)
-               :file-id (:file-id item)
-               ::l/sync? true)
+      (l/dbg :hint "inserting file media object"
+             :id (str (:id item))
+             :file-id (str (:file-id item))
+             ::l/sync? true)
 
       (let [file-id (lookup-index (:file-id item))]
         (if (= file-id (:file-id item))
-          (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/sync? true)
+          (l/warn :hint "ignoring file media object" :file-id (str file-id) ::l/sync? true)
           (db/insert! conn :file-media-object
                       (-> item
                           (assoc :file-id file-id)
                           (d/update-when :media-id lookup-index)
                           (d/update-when :thumbnail-id lookup-index))
-                      {:on-conflict-do-nothing overwrite?}))))))
+                      {::db/on-conflict-do-nothing? overwrite?}))))
+
+    (doseq [item (:thumbnails @*state*)]
+      (let [item (update item :media-id lookup-index)]
+        (l/dbg :hint "inserting file object thumbnail"
+               :file-id (str (:file-id item))
+               :media-id (str (:media-id item))
+               :object-id (:object-id item)
+               ::l/sync? true)
+        (db/insert! conn :file-tagged-object-thumbnail item
+                    {::db/on-conflict-do-nothing? overwrite?})))))
 
 (defn- lookup-index
   [id]
   (let [val (get-in @*state* [:index id])]
-    (l/debug :fn "lookup-index" :id id :val val ::l/sync? true)
+    (l/trc :fn "lookup-index" :id id :val val ::l/sync? true)
     (when (and (not (::ignore-index-errors? *options*)) (not val))
       (ex/raise :type :validation
                 :code :incomplete-index
@@ -786,7 +930,7 @@
          index index]
     (if-let [id (first items)]
       (let [new-id (if (::overwrite? *options*) id (uuid/next))]
-        (l/debug :fn "update-index" :id id :new-id new-id ::l/sync? true)
+        (l/trc :fn "update-index" :id id :new-id new-id ::l/sync? true)
         (recur (rest items)
                (assoc index id new-id)))
       index)))
@@ -863,8 +1007,8 @@
         ab (volatile! false)
         cs (volatile! nil)]
     (try
-      (l/info :hint "start exportation" :export-id id)
-      (dm/with-open [output (io/output-stream output)]
+      (l/info :hint "start exportation" :export-id (str id))
+      (pu/with-open [output (io/output-stream output)]
         (binding [*position* (atom 0)]
           (write-export! (assoc cfg ::output output))))
 
@@ -879,7 +1023,7 @@
         (throw cause))
 
       (finally
-        (l/info :hint "exportation finished" :export-id id
+        (l/info :hint "exportation finished" :export-id (str id)
                 :elapsed (str (inst-ms (tp)) "ms")
                 :aborted @ab
                 :cause @cs)))))
@@ -887,7 +1031,7 @@
 (defn export-to-tmpfile!
   [cfg]
   (let [path (tmp/tempfile :prefix "penpot.export.")]
-    (dm/with-open [output (io/output-stream path)]
+    (pu/with-open [output (io/output-stream path)]
       (export! cfg output)
       path)))
 
@@ -896,10 +1040,10 @@
   (let [id (uuid/next)
         tp (dt/tpoint)
         cs (volatile! nil)]
-    (l/info :hint "import: started" :import-id id)
+    (l/info :hint "import: started" :id (str id))
     (try
       (binding [*position* (atom 0)]
-        (dm/with-open [input (io/input-stream input)]
+        (pu/with-open [input (io/input-stream input)]
           (read-import! (assoc cfg ::input input))))
 
       (catch Throwable cause
@@ -908,61 +1052,78 @@
 
       (finally
         (l/info :hint "import: terminated"
-                :import-id id
+                :id (str id)
                 :elapsed (dt/format-duration (tp))
-                :error? (some? @cs)
-                :cause @cs
-                )))))
+                :error? (some? @cs))))))
+
 
 ;; --- Command: export-binfile
 
-(s/def ::file-id ::us/uuid)
-(s/def ::include-libraries? ::us/boolean)
-(s/def ::embed-assets? ::us/boolean)
-
-(s/def ::export-binfile
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id ::include-libraries? ::embed-assets?]))
+(def ^:private
+  schema:export-binfile
+  (sm/define
+    [:map {:title "export-binfile"}
+     [:file-id ::sm/uuid]
+     [:include-libraries? :boolean]
+     [:embed-assets? :boolean]]))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
-   ::webhooks/event? true}
+   ::webhooks/event? true
+   ::sm/result schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries? embed-assets?] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (let [body (reify yrs/StreamableResponseBody
-               (-write-body-to-stream [_ _ output-stream]
-                 (-> cfg
-                     (assoc ::file-ids [file-id])
-                     (assoc ::embed-assets? embed-assets?)
-                     (assoc ::include-libraries? include-libraries?)
-                     (export! output-stream))))]
+  (fn [_]
+    {::rres/status 200
+     ::rres/headers {"content-type" "application/octet-stream"}
+     ::rres/body (reify rres/StreamableResponseBody
+                   (-write-body-to-stream [_ _ output-stream]
+                     (-> cfg
+                         (assoc ::file-ids [file-id])
+                         (assoc ::embed-assets? embed-assets?)
+                         (assoc ::include-libraries? include-libraries?)
+                         (export! output-stream))))}))
 
-    (fn [_]
-      {::yrs/status 200
-       ::yrs/body body
-       ::yrs/headers {"content-type" "application/octet-stream"}})))
 
-(s/def ::file ::media/upload)
-(s/def ::import-binfile
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::project-id ::file]))
+;; --- Command: import-binfile
+
+(def ^:private
+  schema:import-binfile
+  (sm/define
+    [:map {:title "import-binfile"}
+     [:project-id ::sm/uuid]
+     [:file ::media/upload]]))
+
+(declare ^:private import-binfile)
 
 (sv/defmethod ::import-binfile
   "Import a penpot file in a binary format."
   {::doc/added "1.15"
-   ::webhooks/event? true}
+   ::webhooks/event? true
+   ::sse/stream? true
+   ::sm/params schema:import-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id file] :as params}]
-  (db/with-atomic [conn pool]
-    (projects/check-read-permissions! conn profile-id project-id)
-    (let [ids (import! (assoc cfg
-                              ::input (:path file)
-                              ::project-id project-id
-                              ::ignore-index-errors? true))]
+  (projects/check-read-permissions! pool profile-id project-id)
+  (let [params (-> cfg
+                   (assoc ::input (:path file))
+                   (assoc ::project-id project-id)
+                   (assoc ::profile-id profile-id)
+                   (assoc ::ignore-index-errors? true))]
+    (with-meta
+      (sse/response #(import-binfile params))
+      {::audit/props {:file nil}})))
 
-      (db/update! conn :project
-                  {:modified-at (dt/now)}
-                  {:id project-id})
-
-      (rph/with-meta ids
-        {::audit/props {:file nil :file-ids ids}}))))
+(defn- import-binfile
+  [{:keys [::wrk/executor ::project-id] :as params}]
+  (db/tx-run! params
+              (fn [{:keys [::db/conn] :as params}]
+                ;; NOTE: the importation process performs some operations that
+                ;; are not very friendly with virtual threads, and for avoid
+                ;; unexpected blocking of other concurrent operations we
+                ;; dispatch that operation to a dedicated executor.
+                (let [result (p/thread-call executor (partial import! params))]
+                  (db/update! conn :project
+                              {:modified-at (dt/now)}
+                              {:id project-id})
+                  (deref result)))))

backend/src/app/rpc/commands/comments.clj

@@ -12,6 +12,7 @@
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
+   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -43,15 +44,17 @@
 
 (defn- get-file
   "A specialized version of get-file for comments module."
-  [conn file-id page-id]
-  (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
-    (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id]) (files/decode-row))]
+  [{:keys [::db/conn] :as cfg} file-id page-id]
+  (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id])
+                                           (files/decode-row))]
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
       (-> file
           (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
-          (assoc :page-id page-id))
-      (ex/raise :type :not-found
-                :code :object-not-found
-                :hint "file not found"))))
+          (assoc :page-id page-id)))
+
+    (ex/raise :type :not-found
+              :code :object-not-found
+              :hint "file not found")))
 
 (defn- get-comment-thread
   [conn thread-id & {:as opts}]
@@ -288,38 +291,37 @@
 (sv/defmethod ::create-comment-thread
   {::doc/added "1.15"
    ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg}
-   {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
+  [cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (files/check-comment-permissions! conn profile-id file-id share-id)
+                (let [{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
 
-  (db/with-atomic [conn pool]
-    (let [{:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
-      (files/check-comment-permissions! conn profile-id file-id share-id)
+                  (run! (partial quotes/check-quote! conn)
+                        (list {::quotes/id ::quotes/comment-threads-per-file
+                               ::quotes/profile-id profile-id
+                               ::quotes/team-id team-id
+                               ::quotes/project-id project-id
+                               ::quotes/file-id file-id}
+                              {::quotes/id ::quotes/comments-per-file
+                               ::quotes/profile-id profile-id
+                               ::quotes/team-id team-id
+                               ::quotes/project-id project-id
+                               ::quotes/file-id file-id}))
 
-      (run! (partial quotes/check-quote! conn)
-            (list {::quotes/id ::quotes/comment-threads-per-file
-                   ::quotes/profile-id profile-id
-                   ::quotes/team-id team-id
-                   ::quotes/project-id project-id
-                   ::quotes/file-id file-id}
-                  {::quotes/id ::quotes/comments-per-file
-                   ::quotes/profile-id profile-id
-                   ::quotes/team-id team-id
-                   ::quotes/project-id project-id
-                   ::quotes/file-id file-id}))
-
-      (rtry/with-retry {::rtry/when rtry/conflict-exception?
-                        ::rtry/max-retries 3
-                        ::rtry/label "create-comment-thread"
-                        ::db/conn conn}
-        (create-comment-thread conn
-                               {:created-at request-at
-                                :profile-id profile-id
-                                :file-id file-id
-                                :page-id page-id
-                                :page-name page-name
-                                :position position
-                                :content content
-                                :frame-id frame-id})))))
+                  (rtry/with-retry {::rtry/when rtry/conflict-exception?
+                                    ::rtry/max-retries 3
+                                    ::rtry/label "create-comment-thread"
+                                    ::db/conn conn}
+                    (create-comment-thread conn
+                                           {:created-at request-at
+                                            :profile-id profile-id
+                                            :file-id file-id
+                                            :page-id page-id
+                                            :page-name page-name
+                                            :position position
+                                            :content content
+                                            :frame-id frame-id}))))))
 
 
 (defn- create-comment-thread
@@ -402,8 +404,7 @@
 
 ;; --- COMMAND: Add Comment
 
-(declare get-comment-thread)
-(declare create-comment)
+(declare ^:private get-comment-thread)
 
 (s/def ::create-comment
   (s/keys :req [::rpc/profile-id]
@@ -413,49 +414,51 @@
 (sv/defmethod ::create-comment
   {::doc/added "1.15"
    ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content] :as params}]
-  (db/with-atomic [conn pool]
-    (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
-          {:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
+  [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
+                      {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
 
-      (files/check-comment-permissions! conn profile-id (:id file) share-id)
-      (quotes/check-quote! conn
-                           {::quotes/id ::quotes/comments-per-file
-                            ::quotes/profile-id profile-id
-                            ::quotes/team-id team-id
-                            ::quotes/project-id project-id
-                            ::quotes/file-id (:id file)})
+                  (files/check-comment-permissions! conn profile-id (:id file) share-id)
+                  (quotes/check-quote! conn
+                                       {::quotes/id ::quotes/comments-per-file
+                                        ::quotes/profile-id profile-id
+                                        ::quotes/team-id team-id
+                                        ::quotes/project-id project-id
+                                        ::quotes/file-id (:id file)})
 
-      ;; Update the page-name cached attribute on comment thread table.
-      (when (not= page-name (:page-name thread))
-        (db/update! conn :comment-thread
-                    {:page-name page-name}
-                    {:id thread-id}))
+                  ;; Update the page-name cached attribute on comment thread table.
+                  (when (not= page-name (:page-name thread))
+                    (db/update! conn :comment-thread
+                                {:page-name page-name}
+                                {:id thread-id}))
 
-      (let [comment (db/insert! conn :comment
-                                {:id (uuid/next)
-                                 :created-at request-at
-                                 :modified-at request-at
-                                 :thread-id thread-id
-                                 :owner-id profile-id
-                                 :content content})
-            props    {:file-id file-id
-                      :share-id nil}]
+                  (let [comment (db/insert! conn :comment
+                                            {:id (uuid/next)
+                                             :created-at request-at
+                                             :modified-at request-at
+                                             :thread-id thread-id
+                                             :owner-id profile-id
+                                             :content content})
+                        props    {:file-id file-id
+                                  :share-id nil}]
 
-        ;; Update thread modified-at attribute and assoc the current
-        ;; profile to the participant set.
-        (db/update! conn :comment-thread
-                    {:modified-at request-at
-                     :participants (-> (:participants thread #{})
-                                       (conj profile-id)
-                                       (db/tjson))}
-                    {:id thread-id})
+                    ;; Update thread modified-at attribute and assoc the current
+                    ;; profile to the participant set.
+                    (db/update! conn :comment-thread
+                                {:modified-at request-at
+                                 :participants (-> (:participants thread #{})
+                                                   (conj profile-id)
+                                                   (db/tjson))}
+                                {:id thread-id})
 
-        ;; Update the current profile status in relation to the
-        ;; current thread.
-        (upsert-comment-thread-status! conn profile-id thread-id request-at)
+                    ;; Update the current profile status in relation to the
+                    ;; current thread.
+                    (upsert-comment-thread-status! conn profile-id thread-id request-at)
+
+                    (vary-meta comment assoc ::audit/props props))))))
 
-        (vary-meta comment assoc ::audit/props props)))))
 
 ;; --- COMMAND: Update Comment
 
@@ -466,29 +469,31 @@
 
 (sv/defmethod ::update-comment
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content] :as params}]
-  (db/with-atomic [conn pool]
-    (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true)
-          {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
+  [cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}]
 
-      (files/check-comment-permissions! conn profile-id file-id share-id)
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true)
+                      {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
 
-      ;; Don't allow edit comments to not owners
-      (when-not (= owner-id profile-id)
-        (ex/raise :type :validation
-                  :code :not-allowed))
+                  (files/check-comment-permissions! conn profile-id file-id share-id)
 
-      (let [{:keys [page-name] :as file} (get-file conn file-id page-id)]
-        (db/update! conn :comment
-                    {:content content
-                     :modified-at request-at}
-                    {:id id})
+                  ;; Don't allow edit comments to not owners
+                  (when-not (= owner-id profile-id)
+                    (ex/raise :type :validation
+                              :code :not-allowed))
 
-        (db/update! conn :comment-thread
-                    {:modified-at request-at
-                     :page-name page-name}
-                    {:id thread-id})
-        nil))))
+                  (let [{:keys [page-name] :as file} (get-file cfg file-id page-id)]
+                    (db/update! conn :comment
+                                {:content content
+                                 :modified-at request-at}
+                                {:id id})
+
+                    (db/update! conn :comment-thread
+                                {:modified-at request-at
+                                 :page-name page-name}
+                                {:id thread-id})
+                    nil)))))
 
 ;; --- COMMAND: Delete Comment Thread
 
@@ -499,7 +504,7 @@
 
 (sv/defmethod ::delete-comment-thread
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}]
   (db/with-atomic [conn pool]
     (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
@@ -539,12 +544,12 @@
 
 (sv/defmethod ::update-comment-thread-position
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id position frame-id share-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
   (db/with-atomic [conn pool]
     (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
       (db/update! conn :comment-thread
-                  {:modified-at (::rpc/request-at params)
+                  {:modified-at request-at
                    :position (db/pgpoint position)
                    :frame-id frame-id}
                   {:id (:id thread)})
@@ -559,12 +564,12 @@
 
 (sv/defmethod ::update-comment-thread-frame
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id frame-id share-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
   (db/with-atomic [conn pool]
     (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
       (db/update! conn :comment-thread
-                  {:modified-at (::rpc/request-at params)
+                  {:modified-at request-at
                    :frame-id frame-id}
                   {:id id})
       nil)))

backend/src/app/rpc/commands/files_create.clj

@@ -7,15 +7,20 @@
 (ns app.rpc.commands.files-create
   (:require
    [app.common.data :as d]
-   [app.common.files.features :as ffeat]
+   [app.common.data.macros :as dm]
+   [app.common.features :as cfeat]
+   [app.common.schema :as sm]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
+   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.permissions :as perms]
    [app.rpc.quotes :as quotes]
@@ -24,7 +29,7 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.spec.alpha :as s]))
+   [clojure.set :as set]))
 
 (defn create-file-role!
   [conn {:keys [file-id profile-id role]}]
@@ -34,27 +39,34 @@
          (db/insert! conn :file-profile-rel))))
 
 (defn create-file
-  [conn {:keys [id name project-id is-shared revn
-                modified-at deleted-at create-page
-                ignore-sync-until features]
-         :or {is-shared false revn 0 create-page true}
-         :as params}]
+  [{:keys [::db/conn] :as cfg}
+   {:keys [id name project-id is-shared revn
+           modified-at deleted-at create-page
+           ignore-sync-until features]
+    :or {is-shared false revn 0 create-page true}
+    :as params}]
+
+  (dm/assert!
+   "expected a valid connection"
+   (db/connection? conn))
 
   (let [id       (or id (uuid/next))
-        features (->> features
-                      (into (files/get-default-features))
-                      (files/check-features-compatibility!))
 
-        pointers (atom {})
+        pointers (pmap/create-tracked)
+        pmap?    (contains? features "fdata/pointer-map")
+        omap?    (contains? features "fdata/objects-map")
+
         data     (binding [pmap/*tracked* pointers
-                           ffeat/*current* features
-                           ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity)
-                           ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)]
+                           cfeat/*current* features
+                           cfeat/*wrap-with-objects-map-fn* (if omap? omap/wrap identity)
+                           cfeat/*wrap-with-pointer-map-fn* (if pmap? pmap/wrap identity)]
                    (if create-page
                      (ctf/make-file-data id)
                      (ctf/make-file-data id nil)))
 
-        features (db/create-array conn "text" features)
+        features (->> (set/difference features cfeat/frontend-only-features)
+                      (db/create-array conn "text"))
+
         file     (db/insert! conn :file
                              (d/without-nils
                               {:id id
@@ -69,40 +81,69 @@
                                :deleted-at deleted-at}))]
 
     (binding [pmap/*tracked* pointers]
-      (files/persist-pointers! conn id))
+      (feat.fdata/persist-pointers! cfg id))
 
     (->> (assoc params :file-id id :role :owner)
          (create-file-role! conn))
 
     (db/update! conn :project
-      {:modified-at (dt/now)}
-      {:id project-id})
+                {:modified-at (dt/now)}
+                {:id project-id})
 
     (files/decode-row file)))
 
-(s/def ::create-file
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::files/name
-                   ::files/project-id]
-          :opt-un [::files/id
-                   ::files/is-shared
-                   ::files/features]))
+(def ^:private schema:create-file
+  [:map {:title "create-file"}
+   [:name :string]
+   [:project-id ::sm/uuid]
+   [:id {:optional true} ::sm/uuid]
+   [:is-shared {:optional true} :boolean]
+   [:features {:optional true} ::cfeat/features]])
 
 (sv/defmethod ::create-file
   {::doc/added "1.17"
    ::doc/module :files
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
-  (db/with-atomic [conn pool]
-    (projects/check-edition-permissions! conn profile-id project-id)
-    (let [team-id (files/get-team-id conn project-id)
-          params  (assoc params :profile-id profile-id)]
+   ::webhooks/event? true
+   ::sm/params schema:create-file}
+  [cfg {:keys [::rpc/profile-id project-id] :as params}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (projects/check-edition-permissions! conn profile-id project-id)
+                (let [team     (teams/get-team conn
+                                               :profile-id profile-id
+                                               :project-id project-id)
+                      team-id  (:id team)
 
-      (run! (partial quotes/check-quote! conn)
-            (list {::quotes/id ::quotes/files-per-project
-                   ::quotes/team-id team-id
-                   ::quotes/profile-id profile-id
-                   ::quotes/project-id project-id}))
+                      ;; When we create files, we only need to respect the team
+                      ;; features, because some features can be enabled
+                      ;; globally, but the team is still not migrated properly.
+                      features (-> (cfeat/get-team-enabled-features cf/flags team)
+                                   (cfeat/check-client-features! (:features params)))
 
-      (-> (create-file conn params)
-          (vary-meta assoc ::audit/props {:team-id team-id})))))
+                      ;; We also include all no migration features declared by
+                      ;; client; that enables the ability to enable a runtime
+                      ;; feature on frontend and make it permanent on file
+                      features (-> (:features params #{})
+                                   (set/intersection cfeat/no-migration-features)
+                                   (set/union features))
+
+                      params   (-> params
+                                   (assoc :profile-id profile-id)
+                                   (assoc :features features))]
+
+                  (run! (partial quotes/check-quote! conn)
+                        (list {::quotes/id ::quotes/files-per-project
+                               ::quotes/team-id team-id
+                               ::quotes/profile-id profile-id
+                               ::quotes/project-id project-id}))
+
+                  ;; When newly computed features does not match exactly with
+                  ;; the features defined on team row, we update it.
+                  (when (not= features (:features team))
+                    (let [features (db/create-array conn "text" features)]
+                      (db/update! conn :team
+                                  {:features features}
+                                  {:id team-id})))
+
+                  (-> (create-file cfg params)
+                      (vary-meta assoc ::audit/props {:team-id team-id}))))))

backend/src/app/rpc/commands/files_snapshot.clj

@@ -0,0 +1,135 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files-snapshot
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.main :as-alias main]
+   [app.media :as media]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.profile :as profile]
+   [app.rpc.doc :as-alias doc]
+   [app.storage :as sto]
+   [app.util.services :as sv]
+   [app.util.time :as dt]))
+
+(defn check-authorized!
+  [{:keys [::db/pool]} profile-id]
+  (when-not (or (= "devenv" (cf/get :host))
+                (let [profile (ex/ignoring (profile/get-profile pool profile-id))
+                      admins  (or (cf/get :admins) #{})]
+                  (contains? admins (:email profile))))
+    (ex/raise :type :authentication
+              :code :authentication-required
+              :hint "only admins allowed")))
+
+(defn get-file-snapshots
+  [{:keys [::db/conn]} {:keys [file-id limit start-at]
+                        :or {limit Long/MAX_VALUE}}]
+  (let [query    (str "select id, label, revn, created_at "
+                      "  from file_change "
+                      " where file_id = ? "
+                      "   and created_at < ? "
+                      "   and data is not null "
+                      " order by created_at desc "
+                      " limit ?")
+        start-at (or start-at (dt/now))
+        limit    (min limit 20)]
+
+    (->> (db/exec! conn [query file-id start-at limit])
+         (mapv (fn [row]
+                 (update row :created-at dt/format-instant :rfc1123))))))
+
+(def ^:private schema:get-file-snapshots
+  [:map [:file-id ::sm/uuid]])
+
+(sv/defmethod ::get-file-snapshots
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:get-file-snapshots}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/run! cfg #(get-file-snapshots % params)))
+
+(defn restore-file-snapshot!
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id id]}]
+  (let [storage  (media/configure-assets-storage storage conn)
+        params   {:id id :file-id file-id}
+        options  {:columns [:id :data :revn]}
+        snapshot (db/get* conn :file-change params options)]
+
+    (when (and (some? snapshot)
+               (some? (:data snapshot)))
+
+      (l/debug :hint "snapshot found"
+               :snapshot-id (:id snapshot)
+               :file-id file-id)
+
+      (db/update! conn :file
+                  {:data (:data snapshot)}
+                  {:id file-id})
+
+      ;; clean object thumbnails
+      (let [sql (str "delete from file_object_thumbnail "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/del-object! storage media-id)))
+
+      ;; clean object thumbnails
+      (let [sql (str "delete from file_thumbnail "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/del-object! storage media-id)))
+
+      {:id (:id snapshot)})))
+
+(def ^:private schema:restore-file-snapshot
+  [:map
+   [:file-id ::sm/uuid]
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::restore-file-snapshot
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:restore-file-snapshot}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/tx-run! cfg #(restore-file-snapshot! % params)))
+
+(defn take-file-snapshot!
+  [{:keys [::db/conn]} {:keys [file-id label]}]
+  (when-let [file (db/get* conn :file {:id file-id})]
+    (let [id    (uuid/next)
+          label (or label (str "Snapshot at " (dt/format-instant (dt/now) :rfc1123)))]
+      (l/debug :hint "persisting file snapshot" :file-id file-id :label label)
+      (db/insert! conn :file-change
+                  {:id id
+                   :revn (:revn file)
+                   :data (:data file)
+                   :features (:features file)
+                   :file-id (:id file)
+                   :label label})
+      {:id id})))
+
+(def ^:private schema:take-file-snapshot
+  [:map [:file-id ::sm/uuid]])
+
+(sv/defmethod ::take-file-snapshot
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:take-file-snapshot}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/tx-run! cfg #(take-file-snapshot! % params)))
+

backend/src/app/rpc/commands/files_temp.clj

@@ -7,21 +7,26 @@
 (ns app.rpc.commands.files-temp
   (:require
    [app.common.exceptions :as ex]
-   [app.common.pages :as cp]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as fch]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-create :refer [create-file]]
+   [app.rpc.commands.files-create :as files.create]
    [app.rpc.commands.files-update :as-alias files.update]
    [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.util.blob :as blob]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [clojure.set :as set]
    [clojure.spec.alpha :as s]))
 
+
 ;; --- MUTATION COMMAND: create-temp-file
 
 (s/def ::create-page ::us/boolean)
@@ -38,25 +43,35 @@
 (sv/defmethod ::create-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
-  (db/with-atomic [conn pool]
-    (projects/check-edition-permissions! conn profile-id project-id)
-    (create-file conn (assoc params :profile-id profile-id :deleted-at (dt/in-future {:days 1})))))
+  [cfg {:keys [::rpc/profile-id project-id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (projects/check-edition-permissions! conn profile-id project-id)
+                    (let [team     (teams/get-team conn
+                                                   :profile-id profile-id
+                                                   :project-id project-id)
+
+                          ;; When we create files, we only need to respect the team
+                          ;; features, because some features can be enabled
+                          ;; globally, but the team is still not migrated properly.
+                          features (-> (cfeat/get-team-enabled-features cf/flags team)
+                                       (cfeat/check-client-features! (:features params)))
+
+                          ;; We also include all no migration features declared by
+                          ;; client; that enables the ability to enable a runtime
+                          ;; feature on frontend and make it permanent on file
+                          features (-> (:features params #{})
+                                       (set/intersection cfeat/no-migration-features)
+                                       (set/union features))
+
+                          params   (-> params
+                                       (assoc :profile-id profile-id)
+                                       (assoc :deleted-at (dt/in-future {:days 1}))
+                                       (assoc :features features))]
+
+                      (files.create/create-file cfg params)))))
 
 ;; --- MUTATION COMMAND: update-temp-file
 
-(defn update-temp-file
-  [conn {:keys [profile-id session-id id revn changes] :as params}]
-  (db/insert! conn :file-change
-              {:id (uuid/next)
-               :session-id session-id
-               :profile-id profile-id
-               :created-at (dt/now)
-               :file-id id
-               :revn revn
-               :data nil
-               :changes (blob/encode changes)}))
-
 (s/def ::update-temp-file
   (s/keys :req [::rpc/profile-id]
           :req-un [::files.update/changes
@@ -67,10 +82,18 @@
 (sv/defmethod ::update-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (update-temp-file conn (assoc params :profile-id profile-id))
-    nil))
+  [cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (db/insert! conn :file-change
+                                {:id (uuid/next)
+                                 :session-id session-id
+                                 :profile-id profile-id
+                                 :created-at (dt/now)
+                                 :file-id id
+                                 :revn revn
+                                 :data nil
+                                 :changes (blob/encode changes)})
+                    nil)))
 
 ;; --- MUTATION COMMAND: persist-temp-file
 
@@ -90,7 +113,7 @@
     (let [data
           (->> revs
                (mapcat #(->> % :changes blob/decode))
-               (cp/process-changes (blob/decode (:data file))))]
+               (fch/process-changes (blob/decode (:data file))))]
       (db/update! conn :file
                   {:deleted-at nil
                    :revn revn
@@ -105,7 +128,7 @@
 (sv/defmethod ::persist-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
-    (persist-temp-file conn params)))
+  [cfg {:keys [::rpc/profile-id id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (persist-temp-file conn params))))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -8,22 +8,24 @@
   (:require
    [app.common.data :as d]
    [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [app.common.files.helpers :as cfh]
    [app.common.geom.shapes :as gsh]
-   [app.common.pages.helpers :as cph]
    [app.common.schema :as sm]
-   [app.common.spec :as us]
+   [app.common.thumbnails :as thc]
    [app.common.types.shape-tree :as ctt]
+   [app.config :as cf]
    [app.db :as db]
-   [app.db.sql :as sql]
+   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
    [app.rpc :as-alias rpc]
+   [app.rpc.climit :as-alias climit]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.helpers :as rph]
    [app.storage :as sto]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
@@ -38,88 +40,60 @@
 
 ;; --- COMMAND QUERY: get-file-object-thumbnails
 
+(defn- get-object-thumbnails-by-tag
+  [conn file-id tag]
+  (let [sql (str/concat
+             "select object_id, media_id, tag "
+             "  from file_tagged_object_thumbnail"
+             " where file_id=? and tag=?")
+        res (db/exec! conn [sql file-id tag])]
+    (->> res
+         (d/index-by :object-id (fn [row]
+                                  (files/resolve-public-uri (:media-id row))))
+         (d/without-nils))))
+
 (defn- get-object-thumbnails
   ([conn file-id]
    (let [sql (str/concat
-              "select object_id, data, media_id "
-              "  from file_object_thumbnail"
+              "select object_id, media_id, tag "
+              "  from file_tagged_object_thumbnail"
               " where file_id=?")
          res (db/exec! conn [sql file-id])]
      (->> res
           (d/index-by :object-id (fn [row]
-                                   (or (some-> row :media-id files/resolve-public-uri)
-                                       (:data row))))
+                                   (files/resolve-public-uri (:media-id row))))
           (d/without-nils))))
 
   ([conn file-id object-ids]
    (let [sql (str/concat
-              "select object_id, data, media_id "
-              "  from file_object_thumbnail"
+              "select object_id, media_id, tag "
+              "  from file_tagged_object_thumbnail"
               " where file_id=? and object_id = ANY(?)")
          ids (db/create-array conn "text" (seq object-ids))
          res (db/exec! conn [sql file-id ids])]
-     (d/index-by :object-id
-                 (fn [row]
-                   (or (some-> row :media-id files/resolve-public-uri)
-                       (:data row)))
-                 res))))
+
+     (->> res
+          (d/index-by :object-id (fn [row]
+                                   (files/resolve-public-uri (:media-id row))))
+          (d/without-nils)))))
 
 (sv/defmethod ::get-file-object-thumbnails
   "Retrieve a file object thumbnails."
   {::doc/added "1.17"
    ::doc/module :files
    ::sm/params [:map {:title "get-file-object-thumbnails"}
-                [:file-id ::sm/uuid]]
+                [:file-id ::sm/uuid]
+                [:tag {:optional true} :string]]
    ::sm/result [:map-of :string :string]
    ::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
    ::cond/reuse-key? true
    ::cond/key-fn files/get-file-etag}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id tag] :as params}]
   (dm/with-open [conn (db/open pool)]
     (files/check-read-permissions! conn profile-id file-id)
-    (get-object-thumbnails conn file-id)))
-
-;; --- COMMAND QUERY: get-file-thumbnail
-
-(defn get-file-thumbnail
-  [conn file-id revn]
-  (let [sql (sql/select :file-thumbnail
-                        (cond-> {:file-id file-id}
-                          revn (assoc :revn revn))
-                        {:limit 1
-                         :order-by [[:revn :desc]]})
-        row (db/exec-one! conn sql)]
-
-    (when-not row
-      (ex/raise :type :not-found
-                :code :file-thumbnail-not-found))
-
-    (when-not (:data row)
-      (ex/raise :type :not-found
-                :code :file-thumbnail-not-found))
-
-    {:data (:data row)
-     :props (some-> (:props row) db/decode-transit-pgobject)
-     :revn (:revn row)
-     :file-id (:file-id row)}))
-
-(s/def ::revn ::us/integer)
-(s/def ::file-id ::us/uuid)
-
-(s/def ::get-file-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id]
-          :opt-un [::revn]))
-
-(sv/defmethod ::get-file-thumbnail
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::doc/deprecated "1.19"}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
-  (dm/with-open [conn (db/open pool)]
-    (files/check-read-permissions! conn profile-id file-id)
-    (-> (get-file-thumbnail conn file-id revn)
-        (rph/with-http-cache long-cache-duration))))
+    (if tag
+      (get-object-thumbnails-by-tag conn file-id tag)
+      (get-object-thumbnails conn file-id))))
 
 ;; --- COMMAND QUERY: get-file-data-for-thumbnail
 
@@ -127,55 +101,55 @@
 ;; loading all pages into memory for find the frame set for thumbnail.
 
 (defn get-file-data-for-thumbnail
-  [conn {:keys [data id] :as file}]
+  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
   (letfn [;; function responsible on finding the frame marked to be
           ;; used as thumbnail; the returned frame always have
           ;; the :page-id set to the page that it belongs.
 
-          (get-thumbnail-frame [data]
+          (get-thumbnail-frame [file]
             ;; NOTE: this is a hack for avoid perform blocking
             ;; operation inside the for loop, clojure lazy-seq uses
             ;; synchronized blocks that does not plays well with
-            ;; virtual threads, so we need to perform the load
-            ;; operation first. This operation forces all pointer maps
-            ;; load into the memory.
-            (->> (-> data :pages-index vals)
-                 (filter pmap/pointer-map?)
-                 (run! pmap/load!))
-
-            ;; Then proceed to find the frame set for thumbnail
-
-            (d/seek :use-for-thumbnail?
-                    (for [page  (-> data :pages-index vals)
-                          frame (-> page :objects ctt/get-frames)]
-                      (assoc frame :page-id (:id page)))))
+            ;; virtual threads where all rpc methods calls are
+            ;; dispatched, so we need to perform the load operation
+            ;; first. This operation forces all pointer maps load into
+            ;; the memory.
+            ;;
+            ;; FIXME: this is no longer true with clojure>=1.12
+            (let [{:keys [data]} (update file :data feat.fdata/process-pointers pmap/load!)]
+              ;; Then proceed to find the frame set for thumbnail
+              (d/seek #(or (:use-for-thumbnail %)
+                           (:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21)
+                      (for [page  (-> data :pages-index vals)
+                            frame (-> page :objects ctt/get-frames)]
+                        (assoc frame :page-id (:id page))))))
 
           ;; function responsible to filter objects data structure of
           ;; all unneeded shapes if a concrete frame is provided. If no
           ;; frame, the objects is returned untouched.
           (filter-objects [objects frame-id]
-            (d/index-by :id (cph/get-children-with-self objects frame-id)))
+            (d/index-by :id (cfh/get-children-with-self objects frame-id)))
 
           ;; function responsible of assoc available thumbnails
           ;; to frames and remove all children shapes from objects if
           ;; thumbnails is available
           (assoc-thumbnails [objects page-id thumbnails]
             (loop [objects objects
-                   frames  (filter cph/frame-shape? (vals objects))]
+                   frames  (filter cfh/frame-shape? (vals objects))]
 
               (if-let [frame  (-> frames first)]
-                (let [frame-id (:id frame)
-                      object-id (str page-id frame-id)
-                      frame (if-let [thumb (get thumbnails object-id)]
-                              (assoc frame :thumbnail thumb :shapes [])
-                              (dissoc frame :thumbnail))
+                (let [frame-id  (:id frame)
+                      object-id (thc/fmt-object-id (:id file) page-id frame-id "frame")
+                      frame     (if-let [thumb (get thumbnails object-id)]
+                                  (assoc frame :thumbnail thumb :shapes [])
+                                  (dissoc frame :thumbnail))
 
                       children-ids
-                      (cph/get-children-ids objects frame-id)
+                      (cfh/get-children-ids objects frame-id)
 
                       bounds
                       (when (:show-content frame)
-                        (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
+                        (gsh/shapes->rect (cons frame (map (d/getf objects) children-ids))))
 
                       frame
                       (cond-> frame
@@ -192,8 +166,8 @@
 
                 objects)))]
 
-    (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
-      (let [frame     (get-thumbnail-frame data)
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+      (let [frame     (get-thumbnail-frame file)
             frame-id  (:id frame)
             page-id   (or (:page-id frame)
                           (-> data :pages first))
@@ -202,7 +176,7 @@
             page      (cond-> page (pmap/pointer-map? page) deref)
             frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
 
-            obj-ids   (map #(str page-id %) frame-ids)
+            obj-ids   (map #(thc/fmt-object-id (:id file) page-id % "frame") frame-ids)
             thumbs    (get-object-thumbnails conn id obj-ids)]
 
         (cond-> page
@@ -217,81 +191,61 @@
           :always
           (update :objects assoc-thumbnails page-id thumbs))))))
 
+(def ^:private
+  schema:get-file-data-for-thumbnail
+  (sm/define
+    [:map {:title "get-file-data-for-thumbnail"}
+     [:file-id ::sm/uuid]
+     [:features {:optional true} ::cfeat/features]]))
+
+(def ^:private
+  schema:partial-file
+  (sm/define
+    [:map {:title "PartialFile"}
+     [:id ::sm/uuid]
+     [:revn {:min 0} :int]
+     [:page :any]]))
+
 (sv/defmethod ::get-file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
   mainly for render thumbnails on dashboard."
-
   {::doc/added "1.17"
    ::doc/module :files
-   ::sm/params [:map {:title "get-file-data-for-thumbnail"}
-                [:file-id ::sm/uuid]
-                [:features {:optional true} ::files/features]]
-   ::sm/result [:map {:title "PartialFile"}
-                [:id ::sm/uuid]
-                [:revn {:min 0} :int]
-                [:page :any]]}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
-  (dm/with-open [conn (db/open pool)]
-    (files/check-read-permissions! conn profile-id file-id)
-    ;; NOTE: we force here the "storage/pointer-map" feature, because
-    ;; it used internally only and is independent if user supports it
-    ;; or not.
-    (let [feat (into #{"storage/pointer-map"} features)
-          file (files/get-file conn file-id feat)]
-      {:file-id file-id
-       :revn (:revn file)
-       :page (get-file-data-for-thumbnail conn file)})))
+   ::sm/params schema:get-file-data-for-thumbnail
+   ::sm/result schema:partial-file}
+  [cfg {:keys [::rpc/profile-id file-id] :as params}]
+  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                 (files/check-read-permissions! conn profile-id file-id)
+
+                 (let [team     (teams/get-team conn
+                                                :profile-id profile-id
+                                                :file-id file-id)
+
+                       file     (files/get-file cfg file-id)]
+
+                   (-> (cfeat/get-team-enabled-features cf/flags team)
+                       (cfeat/check-client-features! (:features params))
+                       (cfeat/check-file-features! (:features file) (:features params)))
+
+                   {:file-id file-id
+                    :revn (:revn file)
+                    :page (get-file-data-for-thumbnail cfg file)}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; --- MUTATION COMMAND: upsert-file-object-thumbnail
-
-(def sql:upsert-object-thumbnail
-  "insert into file_object_thumbnail(file_id, object_id, data)
-   values (?, ?, ?)
-       on conflict(file_id, object_id) do
-          update set data = ?;")
-
-(defn upsert-file-object-thumbnail!
-  [conn {:keys [file-id object-id data]}]
-  (if data
-    (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
-    (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
-
-(s/def ::data (s/nilable ::us/string))
-(s/def ::object-id ::us/string)
-
-(s/def ::upsert-file-object-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id ::object-id]
-          :opt-un [::data]))
-
-(sv/defmethod ::upsert-file-object-thumbnail
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::doc/deprecated "1.19"
-   ::audit/skip true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-
-    (when-not (db/read-only? conn)
-      (upsert-file-object-thumbnail! conn params)
-      nil)))
-
-
 ;; --- MUTATION COMMAND: create-file-object-thumbnail
 
 (def ^:private sql:create-object-thumbnail
-  "insert into file_object_thumbnail(file_id, object_id, media_id)
-   values (?, ?, ?)
-       on conflict(file_id, object_id) do
-          update set media_id = ?;")
+  "insert into file_tagged_object_thumbnail(file_id, object_id, media_id, tag)
+   values (?, ?, ?, ?)
+       on conflict(file_id, tag, object_id) do
+          update set media_id = ?
+   returning *;")
 
 (defn- create-file-object-thumbnail!
-  [{:keys [::db/conn ::sto/storage]} file-id object-id media]
+  [{:keys [::db/conn ::sto/storage]} file-id object-id media tag]
 
   (let [path  (:path media)
         mtype (:mtype media)
@@ -300,27 +254,30 @@
                   (sto/wrap-with-hash hash))
         media (sto/put-object! storage
                                {::sto/content data
-                                ::sto/deduplicate? false
+                                ::sto/deduplicate? true
+                                ::sto/touched-at (dt/now)
                                 :content-type mtype
                                 :bucket "file-object-thumbnail"})]
 
     (db/exec-one! conn [sql:create-object-thumbnail file-id object-id
-                        (:id media) (:id media)])))
-
+                        (:id media) tag (:id media)])))
 
 (def schema:create-file-object-thumbnail
   [:map {:title "create-file-object-thumbnail"}
    [:file-id ::sm/uuid]
    [:object-id :string]
-   [:media ::media/upload]])
+   [:media ::media/upload]
+   [:tag {:optional true} :string]])
 
 (sv/defmethod ::create-file-object-thumbnail
-  {:doc/added "1.19"
+  {::doc/added "1.19"
    ::doc/module :files
+   ::climit/id :file-thumbnail-ops
+   ::climit/key-fn ::rpc/profile-id
    ::audit/skip true
    ::sm/params schema:create-file-object-thumbnail}
 
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media tag]}]
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id file-id)
     (media/validate-media-type! media)
@@ -330,21 +287,19 @@
       (-> cfg
           (update ::sto/storage media/configure-assets-storage)
           (assoc ::db/conn conn)
-          (create-file-object-thumbnail! file-id object-id media))
-      nil)))
+          (create-file-object-thumbnail! file-id object-id media (or tag "frame"))))))
 
 ;; --- MUTATION COMMAND: delete-file-object-thumbnail
 
 (defn- delete-file-object-thumbnail!
   [{:keys [::db/conn ::sto/storage]} file-id object-id]
-  (when-let [{:keys [media-id]} (db/get* conn :file-object-thumbnail
+  (when-let [{:keys [media-id]} (db/get* conn :file-tagged-object-thumbnail
                                          {:file-id file-id
                                           :object-id object-id}
                                          {::db/for-update? true})]
-    (when media-id
-      (sto/del-object! storage media-id))
 
-    (db/delete! conn :file-object-thumbnail
+    (sto/touch-object! storage media-id)
+    (db/delete! conn :file-tagged-object-thumbnail
                 {:file-id file-id
                  :object-id object-id})
     nil))
@@ -354,8 +309,10 @@
           :req-un [::file-id ::object-id]))
 
 (sv/defmethod ::delete-file-object-thumbnail
-  {:doc/added "1.19"
+  {::doc/added "1.19"
    ::doc/module :files
+   ::climit/id :file-thumbnail-ops
+   ::climit/key-fn ::rpc/profile-id
    ::audit/skip true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}]
 
@@ -369,41 +326,6 @@
           (delete-file-object-thumbnail! file-id object-id))
       nil)))
 
-;; --- MUTATION COMMAND: upsert-file-thumbnail
-
-(def ^:private sql:upsert-file-thumbnail
-  "insert into file_thumbnail (file_id, revn, data, props)
-   values (?, ?, ?, ?::jsonb)
-       on conflict(file_id, revn) do
-          update set data = ?, props=?, updated_at=now();")
-
-(defn- upsert-file-thumbnail!
-  [conn {:keys [file-id revn data props]}]
-  (let [props (db/tjson (or props {}))]
-    (db/exec-one! conn [sql:upsert-file-thumbnail
-                        file-id revn data props data props])))
-
-(s/def ::revn ::us/integer)
-(s/def ::props map?)
-
-(s/def ::upsert-file-thumbnail
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id ::revn ::props ::data]))
-
-(sv/defmethod ::upsert-file-thumbnail
-  "Creates or updates the file thumbnail. Mainly used for paint the
-  grid thumbnails."
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::doc/deprecated "1.19"
-   ::audit/skip true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (when-not (db/read-only? conn)
-      (upsert-file-thumbnail! conn params)
-      nil)))
-
 ;; --- MUTATION COMMAND: create-file-thumbnail
 
 (def ^:private sql:create-file-thumbnail
@@ -439,11 +361,12 @@
   {::doc/added "1.19"
    ::doc/module :files
    ::audit/skip true
+   ::climit/id :file-thumbnail-ops
+   ::climit/key-fn ::rpc/profile-id
    ::sm/params [:map {:title "create-file-thumbnail"}
                 [:file-id ::sm/uuid]
                 [:revn :int]
-                [:media ::media/upload]]
-   }
+                [:media ::media/upload]]}
 
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (db/with-atomic [conn pool]

backend/src/app/rpc/commands/files_update.clj

@@ -6,19 +6,19 @@
 
 (ns app.rpc.commands.files-update
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.files.features :as ffeat]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as cpc]
+   [app.common.files.migrations :as fmg]
+   [app.common.files.validate :as val]
    [app.common.logging :as l]
-   [app.common.pages :as cp]
-   [app.common.pages.changes :as cpc]
-   [app.common.pages.migrations :as pmg]
    [app.common.schema :as sm]
-   [app.common.schema.generators :as smg]
-   [app.common.spec :as us]
-   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.features.fdata :as feat.fdata]
+   [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
    [app.metrics :as mtx]
@@ -26,6 +26,7 @@
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as climit]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
    [app.util.blob :as blob]
@@ -33,69 +34,37 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.spec.alpha :as s]))
-
-;; --- SPECS
-
-(s/def ::changes
-  (s/coll-of map? :kind vector?))
-
-(s/def ::hint-origin ::us/keyword)
-(s/def ::hint-events
-  (s/every ::us/keyword :kind vector?))
-
-(s/def ::change-with-metadata
-  (s/keys :req-un [::changes]
-          :opt-un [::hint-origin
-                   ::hint-events]))
-
-(s/def ::changes-with-metadata
-  (s/every ::change-with-metadata :kind vector?))
-
-(s/def ::session-id ::us/uuid)
-(s/def ::revn ::us/integer)
-(s/def ::update-file
-  (s/and
-   (s/keys :req [::rpc/profile-id]
-           :req-un [::files/id ::session-id ::revn]
-           :opt-un [::changes ::changes-with-metadata ::features])
-   (fn [o]
-     (or (contains? o :changes)
-         (contains? o :changes-with-metadata)))))
-
+   [app.worker :as-alias wrk]
+   [clojure.set :as set]))
 
 ;; --- SCHEMA
 
-(sm/def! ::changes
-  [:vector ::cpc/change])
+(def ^:private
+  schema:update-file
+  (sm/define
+    [:map {:title "update-file"}
+     [:id ::sm/uuid]
+     [:session-id ::sm/uuid]
+     [:revn {:min 0} :int]
+     [:features {:optional true} ::cfeat/features]
+     [:changes {:optional true} [:vector ::cpc/change]]
+     [:changes-with-metadata {:optional true}
+      [:vector [:map
+                [:changes [:vector ::cpc/change]]
+                [:hint-origin {:optional true} :keyword]
+                [:hint-events {:optional true} [:vector :string]]]]]
+     [:skip-validate {:optional true} :boolean]]))
 
-(sm/def! ::change-with-metadata
-  [:map {:title "ChangeWithMetadata"}
-   [:changes ::changes]
-   [:hint-origin {:optional true} :keyword]
-   [:hint-events {:optional true} [:vector :string]]])
-
-(sm/def! ::update-file-params
-  [:map {:title "UpdateFileParams"}
-   [:id ::sm/uuid]
-   [:session-id ::sm/uuid]
-   [:revn {:min 0} :int]
-   [:features {:optional true
-               :gen/max 3
-               :gen/gen (smg/subseq files/supported-features)}
-    ::sm/set-of-strings]
-   [:changes {:optional true} ::changes]
-   [:changes-with-metadata {:optional true}
-    [:vector ::change-with-metadata]]])
-
-(sm/def! ::update-file-result
-  [:vector {:title "UpdateFileResults"}
-   [:map {:title "UpdateFileResult"}
-    [:changes ::changes]
-    [:file-id ::sm/uuid]
-    [:id ::sm/uuid]
-    [:revn {:min 0} :int]
-    [:session-id ::sm/uuid]]])
+(def ^:private
+  schema:update-file-result
+  (sm/define
+    [:vector {:title "update-file-result"}
+     [:map
+      [:changes [:vector ::cpc/change]]
+      [:file-id ::sm/uuid]
+      [:id ::sm/uuid]
+      [:revn {:min 0} :int]
+      [:session-id ::sm/uuid]]]))
 
 ;; --- HELPERS
 
@@ -105,7 +74,7 @@
 (def ^:private library-change-types
   #{:add-color :mod-color :del-color
     :add-media :mod-media :del-media
-    :add-component :mod-component :del-component
+    :add-component :mod-component :del-component :restore-component
     :add-typography :mod-typography :del-typography})
 
 (def ^:private file-change-types
@@ -137,24 +106,25 @@
 
 (defn- wrap-with-pointer-map-context
   [f]
-  (fn [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-    (binding [pmap/*tracked* (atom {})
-              pmap/*load-fn* (partial files/load-pointer conn id)
-              ffeat/*wrap-with-pointer-map-fn* pmap/wrap]
+  (fn [cfg {:keys [id] :as file}]
+    (binding [pmap/*tracked* (pmap/create-tracked)
+              pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+              cfeat/*wrap-with-pointer-map-fn* pmap/wrap]
       (let [result (f cfg file)]
-        (files/persist-pointers! conn id)
+        (feat.fdata/persist-pointers! cfg id)
         result))))
 
 (defn- wrap-with-objects-map-context
   [f]
   (fn [cfg file]
-    (binding [ffeat/*wrap-with-objects-map-fn* omap/wrap]
+    (binding [cfeat/*wrap-with-objects-map-fn* omap/wrap]
       (f cfg file))))
 
 (declare get-lagged-changes)
 (declare send-notifications!)
 (declare update-file)
 (declare update-file*)
+(declare update-file-data)
 (declare take-snapshot?)
 
 ;; If features are specified from params and the final feature
@@ -162,73 +132,94 @@
 ;; database.
 
 (sv/defmethod ::update-file
-  {::climit/id :update-file-by-id
-   ::climit/key-fn :id
+  {::climit/id :update-file/by-profile
+   ::climit/key-fn ::rpc/profile-id
    ::webhooks/event? true
    ::webhooks/batch-timeout (dt/duration "2m")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
 
-   ::sm/params ::update-file-params
-   ::sm/result ::update-file-result
-
+   ::sm/params schema:update-file
+   ::sm/result schema:update-file-result
    ::doc/module :files
    ::doc/added "1.17"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id id)
-    (db/xact-lock! conn id)
-    (let [cfg    (assoc cfg ::db/conn conn)
-          params (assoc params :profile-id profile-id)
-          tpoint (dt/tpoint)]
-      (-> (update-file cfg params)
-          (rph/with-defer #(let [elapsed (tpoint)]
-                             (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))
+  [cfg {:keys [::rpc/profile-id id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (db/xact-lock! conn id)
+
+                    (let [file     (get-file conn id)
+                          team     (teams/get-team conn
+                                                   :profile-id profile-id
+                                                   :team-id (:team-id file))
+
+                          features (-> (cfeat/get-team-enabled-features cf/flags team)
+                                       (cfeat/check-client-features! (:features params))
+                                       (cfeat/check-file-features! (:features file) (:features params)))
+
+                          params   (assoc params
+                                          :profile-id profile-id
+                                          :features features
+                                          :team team
+                                          :file file)
+
+                          tpoint   (dt/tpoint)]
+
+                      ;; When newly computed features does not match exactly with
+                      ;; the features defined on team row, we update it.
+                      (when (not= features (:features team))
+                        (let [features (db/create-array conn "text" features)]
+                          (db/update! conn :team
+                                      {:features features}
+                                      {:id (:id team)})))
+
+                      (binding [l/*context* (some-> (meta params)
+                                                    (get :app.http/request)
+                                                    (errors/request->context))]
+                        (-> (update-file cfg params)
+                            (rph/with-defer #(let [elapsed (tpoint)]
+                                               (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
 
 (defn update-file
-  [{:keys [::db/conn ::mtx/metrics] :as cfg} {:keys [profile-id id changes changes-with-metadata] :as params}]
-  (let [file     (get-file conn id)
-        features (->> (concat (:features file)
-                              (:features params))
-                      (into (files/get-default-features))
-                      (files/check-features-compatibility!))]
+  [{:keys [::db/conn ::mtx/metrics] :as cfg}
+   {:keys [id file features changes changes-with-metadata] :as params}]
+  (binding [cfeat/*current*  features
+            cfeat/*previous* (:features file)]
+    (let [update-fn (cond-> update-file*
+                      (contains? features "fdata/pointer-map")
+                      (wrap-with-pointer-map-context)
 
-    (files/check-edition-permissions! conn profile-id (:id file))
+                      (contains? features "fdata/objects-map")
+                      (wrap-with-objects-map-context))
 
-    (binding [ffeat/*current*  features
-              ffeat/*previous* (:features file)]
+          changes   (if changes-with-metadata
+                      (->> changes-with-metadata (mapcat :changes) vec)
+                      (vec changes))
 
-      (let [update-fn (cond-> update-file*
-                        (contains? features "storage/pointer-map")
-                        (wrap-with-pointer-map-context)
+          features  (-> features
+                        (set/difference cfeat/frontend-only-features)
+                        (set/union (:features file)))]
 
-                        (contains? features "storage/objects-map")
-                        (wrap-with-objects-map-context))
+      (when (> (:revn params)
+               (:revn file))
+        (ex/raise :type :validation
+                  :code :revn-conflict
+                  :hint "The incoming revision number is greater that stored version."
+                  :context {:incoming-revn (:revn params)
+                            :stored-revn (:revn file)}))
 
-            file      (assoc file :features features)
-            changes   (if changes-with-metadata
-                        (->> changes-with-metadata (mapcat :changes) vec)
-                        (vec changes))
+      (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
 
-            params    (-> params
-                          (assoc :file file)
-                          (assoc :changes changes)
-                          (assoc ::created-at (dt/now)))]
+      (when (not= features (:features file))
+        (let [features (db/create-array conn "text" features)]
+          (db/update! conn :file
+                      {:features features}
+                      {:id id})))
 
-        (when (> (:revn params)
-                 (:revn file))
-          (ex/raise :type :validation
-                    :code :revn-conflict
-                    :hint "The incoming revision number is greater that stored version."
-                    :context {:incoming-revn (:revn params)
-                              :stored-revn (:revn file)}))
-
-        (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
-
-        (when (not= features (:features file))
-          (let [features (db/create-array conn "text" features)]
-            (db/update! conn :file
-                        {:features features}
-                        {:id id})))
+      (let [file   (assoc file :features features)
+            params (-> params
+                       (assoc :file file)
+                       (assoc :changes changes)
+                       (assoc ::created-at (dt/now)))]
 
         (-> (update-fn cfg params)
             (vary-meta assoc ::audit/replace-props
@@ -238,33 +229,16 @@
                         :project-id (:project-id file)
                         :team-id    (:team-id file)}))))))
 
-(defn- update-file-data
-  [file changes]
-  (-> file
-      (update :revn inc)
-      (update :data (fn [data]
-                      (cond-> data
-                        :always
-                        (-> (blob/decode)
-                            (assoc :id (:id file))
-                            (pmg/migrate-data))
-
-                        (and (contains? ffeat/*current* "components/v2")
-                             (not (contains? ffeat/*previous* "components/v2")))
-                        (ctf/migrate-to-components-v2)
-
-                        :always
-                        (-> (cp/process-changes changes)
-                            (blob/encode)))))))
-
-
 (defn- update-file*
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file changes session-id ::created-at] :as params}]
+  [{:keys [::db/conn ::wrk/executor] :as cfg}
+   {:keys [profile-id file changes session-id ::created-at skip-validate] :as params}]
   (let [;; Process the file data in the CLIMIT context; scheduling it
         ;; to be executed on a separated executor for avoid to do the
         ;; CPU intensive operation on vthread.
-        file (-> (climit/configure cfg :update-file)
-                 (climit/submit! (partial update-file-data file changes)))]
+
+        update-fdata-fn (partial update-file-data cfg file changes skip-validate)
+        file            (-> (climit/configure cfg :update-file/global)
+                            (climit/run! update-fdata-fn executor))]
 
     (db/insert! conn :file-change
                 {:id (uuid/next)
@@ -297,6 +271,76 @@
       ;; Retrieve and return lagged data
       (get-lagged-changes conn params))))
 
+(defn- soft-validate-file-schema!
+  [file]
+  (try
+    (val/validate-file-schema! file)
+    (catch Throwable cause
+      (l/error :hint "file schema validation error" :cause cause)))
+
+  file)
+
+(defn- soft-validate-file!
+  [file libs]
+  (try
+    (val/validate-file! file libs)
+    (catch Throwable cause
+      (l/error :hint "file validation error"
+               :cause cause)))
+  file)
+
+(defn- update-file-data
+  [{:keys [::db/conn] :as cfg} file changes skip-validate]
+  (let [file (update file :data (fn [data]
+                                  (-> data
+                                      (blob/decode)
+                                      (assoc :id (:id file))
+                                      (fmg/migrate-data)
+                                      (d/without-nils))))
+
+        ;; WARNING: this ruins performance; maybe we need to find
+        ;; some other way to do general validation
+        libs (when (and (contains? cf/flags :file-validation)
+                        (not skip-validate))
+               (->> (files/get-file-libraries conn (:id file))
+                    (into [file] (map (fn [{:keys [id]}]
+                                        (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+                                                  pmap/*tracked* nil]
+                                          (-> (files/get-file cfg id :migrate? false)
+                                              (feat.fdata/process-pointers deref) ; ensure all pointers resolved
+                                              (fmg/migrate-file))))))
+                    (d/index-by :id)))]
+
+    (-> (files/check-version! file)
+        (update :revn inc)
+        (update :data cpc/process-changes changes)
+
+        ;; If `libs` is defined, then full validation is performed
+        (cond-> (contains? cf/flags :soft-file-validation)
+          (soft-validate-file! libs))
+
+        (cond-> (contains? cf/flags :soft-file-schema-validation)
+          (soft-validate-file-schema!))
+
+        (cond-> (and (contains? cf/flags :file-validation)
+                     (not skip-validate))
+          (val/validate-file! libs))
+
+        (cond-> (and (contains? cf/flags :file-schema-validation)
+                     (not skip-validate))
+          (val/validate-file-schema!))
+
+        (cond-> (and (contains? cfeat/*current* "fdata/objects-map")
+                     (not (contains? cfeat/*previous* "fdata/objects-map")))
+          (feat.fdata/enable-objects-map))
+
+        (cond-> (and (contains? cfeat/*current* "fdata/pointer-map")
+                     (not (contains? cfeat/*previous* "fdata/pointer-map")))
+          (feat.fdata/enable-pointer-map))
+
+        (update :data blob/encode))))
+
+
 (defn- take-snapshot?
   "Defines the rule when file `data` snapshot should be saved."
   [{:keys [revn modified-at] :as file}]
@@ -324,7 +368,7 @@
        (vec)))
 
 (defn- send-notifications!
-  [{:keys [::db/conn] :as cfg} {:keys [file changes session-id] :as params}]
+  [cfg {:keys [file team changes session-id] :as params}]
   (let [lchanges (filter library-change? changes)
         msgbus   (::mbus/msgbus cfg)]
 
@@ -338,14 +382,12 @@
                          :changes changes})
 
     (when (and (:is-shared file) (seq lchanges))
-      (let [team-id (or (:team-id file)
-                        (files/get-team-id conn (:project-id file)))]
-        (mbus/pub! msgbus
-                   :topic team-id
-                   :message {:type :library-change
-                             :profile-id (:profile-id params)
-                             :file-id (:id file)
-                             :session-id session-id
-                             :revn (:revn file)
-                             :modified-at (dt/now)
-                             :changes lchanges})))))
+      (mbus/pub! msgbus
+                 :topic (:id team)
+                 :message {:type :library-change
+                           :profile-id (:profile-id params)
+                           :file-id (:id file)
+                           :session-id session-id
+                           :revn (:revn file)
+                           :modified-at (dt/now)
+                           :changes lchanges}))))

backend/src/app/rpc/commands/fonts.clj

@@ -25,6 +25,7 @@
    [app.storage :as sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]))
 
 (def valid-weight #{100 200 300 400 500 600 700 800 900 950})
@@ -80,8 +81,8 @@
             perms   (files/get-permissions conn profile-id file-id share-id)]
         (files/check-read-permissions! perms)
         (db/query conn :team-font-variant
-          {:team-id (:team-id project)
-           :deleted-at nil})))))
+                  {:team-id (:team-id project)
+                   :deleted-at nil})))))
 
 
 (declare create-font-variant)
@@ -156,11 +157,11 @@
                          :woff1-file-id (:id woff1)
                          :woff2-file-id (:id woff2)
                          :otf-file-id (:id otf)
-                         :ttf-file-id (:id ttf)}))
-          ]
+                         :ttf-file-id (:id ttf)}))]
 
-    (let [data   (-> (climit/configure cfg :process-font)
-                     (climit/submit! (partial generate-missing! data)))
+    (let [data   (-> (climit/configure cfg :process-font/global)
+                     (climit/run! (partial generate-missing! data)
+                                  (::wrk/executor cfg)))
           assets (persist-fonts-files! data)
           result (insert-font-variant! assets)]
       (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))

backend/src/app/rpc/commands/ldap.clj

@@ -78,13 +78,13 @@
                             ::audit/profile-id (:id profile)}))))))
 
 (defn- login-or-register
-  [{:keys [::db/pool] :as cfg} info]
-  (db/with-atomic [conn pool]
-    (or (some->> (:email info)
-                 (profile/get-profile-by-email conn)
-                 (profile/decode-row))
-        (->> (assoc info :is-active true :is-demo false)
-             (auth/create-profile! conn)
-             (auth/create-profile-rels! conn)
-             (profile/strip-private-attrs)))))
-
+  [cfg info]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (or (some->> (:email info)
+                             (profile/get-profile-by-email conn)
+                             (profile/decode-row))
+                    (->> (assoc info :is-active true :is-demo false)
+                         (auth/create-profile! conn)
+                         (auth/create-profile-rels! conn)
+                         (profile/strip-private-attrs))))))

backend/src/app/rpc/commands/management.clj

@@ -9,17 +9,19 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.pages.migrations :as pmg]
+   [app.common.features :as cfeat]
+   [app.common.files.migrations :as pmg]
    [app.common.schema :as sm]
-   [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
+   [app.features.fdata :as feat.fdata]
+   [app.http.sse :as sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.binfile :as binfile]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as proj]
-   [app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
@@ -27,32 +29,29 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.spec.alpha :as s]
+   [app.worker :as-alias wrk]
    [clojure.walk :as walk]
+   [promesa.core :as p]
    [promesa.exec :as px]))
 
 ;; --- COMMAND: Duplicate File
 
 (declare duplicate-file)
 
-(s/def ::id ::us/uuid)
-(s/def ::project-id ::us/uuid)
-(s/def ::file-id ::us/uuid)
-(s/def ::team-id ::us/uuid)
-(s/def ::name ::us/string)
-
-(s/def ::duplicate-file
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::file-id]
-          :opt-un [::name]))
+(def ^:private
+  schema:duplicate-file
+  (sm/define
+    [:map {:title "duplicate-file"}
+     [:file-id ::sm/uuid]
+     [:name {:optional true} :string]]))
 
 (sv/defmethod ::duplicate-file
   "Duplicate a single file in the same team."
   {::doc/added "1.16"
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (duplicate-file conn (assoc params :profile-id profile-id))))
+   ::webhooks/event? true
+   ::sm/params schema:duplicate-file}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg duplicate-file (assoc params :profile-id profile-id)))
 
 (defn- remap-id
   [item index key]
@@ -61,7 +60,7 @@
     (assoc key (get index (get item key) (get item key)))))
 
 (defn- process-file
-  [conn {:keys [id] :as file} index]
+  [cfg index {:keys [id] :as file}]
   (letfn [(process-form [form]
             (cond-> form
               ;; Relink library items
@@ -104,35 +103,38 @@
                                  (dissoc k))
                              res)))
                        media
-                       media))]
-    (-> file
-        (update :id #(get index %))
-        (update :data
-                (fn [data]
-                  (binding [pmap/*load-fn* (partial files/load-pointer conn id)
-                            pmap/*tracked* (atom {})]
-                    (let [file-id (get index id)
-                          data    (-> data
-                                      (blob/decode)
-                                      (assoc :id file-id)
-                                      (pmg/migrate-data)
-                                      (update :pages-index relink-shapes)
-                                      (update :components relink-shapes)
-                                      (update :media relink-media)
-                                      (d/without-nils)
-                                      (files/process-pointers pmap/clone)
-                                      (blob/encode))]
-                      (files/persist-pointers! conn file-id)
-                      data)))))))
+                       media))
 
-(def sql:retrieve-used-libraries
+          (update-fdata [fdata new-id]
+            (-> fdata
+                (assoc :id new-id)
+                (pmg/migrate-data)
+                (update :pages-index relink-shapes)
+                (update :components relink-shapes)
+                (update :media relink-media)
+                (d/without-nils)
+                (feat.fdata/process-pointers pmap/clone)))]
+
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+              pmap/*tracked* (pmap/create-tracked)
+              cfeat/*new*    (atom #{})]
+      (let [new-id (get index id)
+            file   (-> file
+                       (assoc :id new-id)
+                       (update :data update-fdata new-id)
+                       (update :features into (deref cfeat/*new*))
+                       (update :features cfeat/migrate-legacy-features))]
+        (feat.fdata/persist-pointers! cfg new-id)
+        file))))
+
+(def sql:get-used-libraries
   "select flr.*
      from file_library_rel as flr
     inner join file as l on (flr.library_file_id = l.id)
     where flr.file_id = ?
       and l.deleted_at is null")
 
-(def sql:retrieve-used-media-objects
+(def sql:get-used-media-objects
   "select fmo.*
      from file_media_object as fmo
     inner join storage_object as so on (fmo.media_id = so.id)
@@ -140,9 +142,9 @@
       and so.deleted_at is null")
 
 (defn duplicate-file*
-  [conn {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
-  (let [flibs    (or flibs (db/exec! conn [sql:retrieve-used-libraries (:id file)]))
-        fmeds    (or fmeds (db/exec! conn [sql:retrieve-used-media-objects (:id file)]))
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
+  (let [flibs    (or flibs (db/exec! conn [sql:get-used-libraries (:id file)]))
+        fmeds    (or fmeds (db/exec! conn [sql:get-used-media-objects (:id file)]))
 
         ;; memo uniform creation/modification date
         now      (dt/now)
@@ -183,9 +185,13 @@
                     (assoc :modified-at now)
                     (assoc :ignore-sync-until ignore))
 
-        file    (process-file conn file index)]
+        file    (process-file cfg index file)]
+
+    (db/insert! conn :file
+                (-> file
+                    (update :features #(db/create-array conn "text" %))
+                    (update :data blob/encode)))
 
-    (db/insert! conn :file file)
     (db/insert! conn :file-profile-rel
                 {:file-id (:id file)
                  :profile-id profile-id
@@ -202,35 +208,36 @@
     file))
 
 (defn duplicate-file
-  [conn {:keys [profile-id file-id] :as params}]
-  (let [file   (db/get-by-id conn :file file-id)
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id] :as params}]
+  (let [;; We don't touch the original file on duplication
+        file   (files/get-file cfg file-id :migrate? false)
         index  {file-id (uuid/next)}
         params (assoc params :index index :file file)]
     (proj/check-edition-permissions! conn profile-id (:project-id file))
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-    (-> (duplicate-file* conn params {:reset-shared-flag true})
-        (update :data blob/decode)
-        (update :features db/decode-pgarray #{}))))
+    (duplicate-file* cfg params {:reset-shared-flag true})))
 
 ;; --- COMMAND: Duplicate Project
 
 (declare duplicate-project)
 
-(s/def ::duplicate-project
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::project-id]
-          :opt-un [::name]))
+(def ^:private
+  schema:duplicate-project
+  (sm/define
+    [:map {:title "duplicate-project"}
+     [:project-id ::sm/uuid]
+     [:name {:optional true} :string]]))
 
 (sv/defmethod ::duplicate-project
   "Duplicate an entire project with all the files"
   {::doc/added "1.16"
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} params]
-  (db/with-atomic [conn pool]
-    (duplicate-project conn (assoc params :profile-id (::rpc/profile-id params)))))
+   ::webhooks/event? true
+   ::sm/params schema:duplicate-project}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg duplicate-project (assoc params :profile-id profile-id)))
 
 (defn duplicate-project
-  [conn {:keys [profile-id project-id name] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id project-id name] :as params}]
 
   ;; Defer all constraints
   (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
@@ -239,9 +246,9 @@
                     (assoc :is-pinned false))
 
         files   (db/query conn :file
-                  {:project-id (:id project)
-                   :deleted-at nil}
-                  {:columns [:id]})
+                          {:project-id (:id project)
+                           :deleted-at nil}
+                          {:columns [:id]})
 
         project (cond-> project
                   (string? name)
@@ -255,8 +262,8 @@
 
     ;; create the duplicated project and assign the current profile as
     ;; a project owner
-    (create-project conn project)
-    (create-project-role conn profile-id (:id project) :owner)
+    (teams/create-project conn project)
+    (teams/create-project-role conn profile-id (:id project) :owner)
 
     ;; duplicate all files
     (let [index  (reduce #(assoc %1 (:id %2) (uuid/next)) {} files)
@@ -265,17 +272,17 @@
                      (assoc :project-id (:id project))
                      (assoc :index index))]
       (doseq [{:keys [id]} files]
-        (let [file   (db/get-by-id conn :file id)
+        (let [file   (files/get-file cfg id :migrate? false)
               params (assoc params :file file)
               opts   {:reset-shared-flag false}]
-          (duplicate-file* conn params opts))))
+          (duplicate-file* cfg params opts))))
 
     ;; return the created project
     project))
 
 ;; --- COMMAND: Move file
 
-(def sql:retrieve-files
+(def sql:get-files
   "select id, project_id from file where id = ANY(?)")
 
 (def sql:move-files
@@ -297,14 +304,19 @@
       and rel.library_file_id = br.library_file_id")
 
 (defn move-files
-  [conn {:keys [profile-id ids project-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id ids project-id] :as params}]
 
   (let [fids    (db/create-array conn "uuid" ids)
-        files   (db/exec! conn [sql:retrieve-files fids])
+        files   (db/exec! conn [sql:get-files fids])
         source  (into #{} (map :project-id) files)
         pids    (->> (conj source project-id)
                      (db/create-array conn "uuid"))]
 
+    (when (contains? source project-id)
+      (ex/raise :type :validation
+                :code :cant-move-to-same-project
+                :hint "Unable to move a file to the same project"))
+
     ;; Check if we have permissions on the destination project
     (proj/check-edition-permissions! conn profile-id project-id)
 
@@ -312,10 +324,10 @@
     (doseq [project-id source]
       (proj/check-edition-permissions! conn profile-id project-id))
 
-    (when (contains? source project-id)
-      (ex/raise :type :validation
-                :code :cant-move-to-same-project
-                :hint "Unable to move a file to the same project"))
+    ;; Check the team compatibility
+    (let [orig-team (teams/get-team conn :profile-id profile-id :project-id (first source))
+          dest-team (teams/get-team conn :profile-id profile-id :project-id project-id)]
+      (cfeat/check-teams-compatibility! orig-team dest-team))
 
     ;; move all files to the project
     (db/exec-one! conn [sql:move-files project-id fids])
@@ -337,36 +349,43 @@
 
     nil))
 
-(s/def ::ids (s/every ::us/uuid :kind set?))
-(s/def ::move-files
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::ids ::project-id]))
+(def ^:private
+  schema:move-files
+  (sm/define
+    [:map {:title "move-files"}
+     [:ids ::sm/set-of-uuid]
+     [:project-id ::sm/uuid]]))
 
 (sv/defmethod ::move-files
   "Move a set of files from one project to other."
   {::doc/added "1.16"
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (move-files conn (assoc params :profile-id profile-id))))
+   ::webhooks/event? true
+   ::sm/params schema:move-files}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg #(move-files % (assoc params :profile-id profile-id))))
 
 ;; --- COMMAND: Move project
 
 (defn move-project
-  [conn {:keys [profile-id team-id project-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id team-id project-id] :as params}]
   (let [project (db/get-by-id conn :project project-id {:columns [:id :team-id]})
         pids    (->> (db/query conn :project {:team-id (:team-id project)} {:columns [:id]})
                      (map :id)
                      (db/create-array conn "uuid"))]
 
-    (teams/check-edition-permissions! conn profile-id (:team-id project))
-    (teams/check-edition-permissions! conn profile-id team-id)
-
     (when (= team-id (:team-id project))
       (ex/raise :type :validation
                 :code :cant-move-to-same-team
                 :hint "Unable to move a project to same team"))
 
+    (teams/check-edition-permissions! conn profile-id (:team-id project))
+    (teams/check-edition-permissions! conn profile-id team-id)
+
+    ;; Check the teams compatibility
+    (let [orig-team (teams/get-team conn :profile-id profile-id :team-id (:team-id project))
+          dest-team (teams/get-team conn :profile-id profile-id :team-id team-id)]
+      (cfeat/check-teams-compatibility! orig-team dest-team))
+
     ;; move project to the destination team
     (db/update! conn :project
                 {:team-id team-id}
@@ -377,67 +396,73 @@
 
     nil))
 
-(s/def ::move-project
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::team-id ::project-id]))
+(def ^:private
+  schema:move-project
+  (sm/define
+    [:map {:title "move-project"}
+     [:team-id ::sm/uuid]
+     [:project-id ::sm/uuid]]))
 
 (sv/defmethod ::move-project
-  "Move projects between teams."
+  "Move projects between teams"
   {::doc/added "1.16"
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (move-project conn (assoc params :profile-id profile-id))))
+   ::webhooks/event? true
+   ::sm/params schema:move-project}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg #(move-project % (assoc params :profile-id profile-id))))
 
 ;; --- COMMAND: Clone Template
 
-(defn- clone-template!
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id template-id project-id]}]
-  (let [template (tmpl/get-template-stream cfg template-id)
-        project  (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
+(def ^:private
+  schema:clone-template
+  (sm/define
+    [:map {:title "clone-template"}
+     [:project-id ::sm/uuid]
+     [:template-id ::sm/word-string]]))
+
+(declare ^:private clone-template)
+
+(sv/defmethod ::clone-template
+  "Clone into the specified project the template by its id."
+  {::doc/added "1.16"
+   ::sse/stream? true
+   ::webhooks/event? true
+   ::sm/params schema:clone-template}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id template-id] :as params}]
+  (let [project   (db/get-by-id pool :project project-id {:columns [:id :team-id]})
+        _         (teams/check-edition-permissions! pool profile-id (:team-id project))
+        template  (tmpl/get-template-stream cfg template-id)
+        params    (-> cfg
+                      (assoc ::binfile/input template)
+                      (assoc ::binfile/project-id (:id project))
+                      (assoc ::binfile/profile-id profile-id)
+                      (assoc ::binfile/ignore-index-errors? true)
+                      (assoc ::binfile/migrate? true))]
 
     (when-not template
       (ex/raise :type :not-found
                 :code :template-not-found
                 :hint "template not found"))
 
-    (teams/check-edition-permissions! conn profile-id (:team-id project))
+    (sse/response #(clone-template params))))
 
-    (-> cfg
-        ;; FIXME: maybe reuse the conn instead of creating more
-        ;; connections in the import process?
-        (dissoc ::db/conn)
-        (assoc ::binfile/input template)
-        (assoc ::binfile/project-id (:id project))
-        (assoc ::binfile/ignore-index-errors? true)
-        (assoc ::binfile/migrate? true)
-        (binfile/import!))))
+(defn- clone-template
+  [{:keys [::wrk/executor ::binfile/project-id] :as params}]
+  (db/tx-run! params
+              (fn [{:keys [::db/conn] :as params}]
+                ;; NOTE: the importation process performs some operations that
+                ;; are not very friendly with virtual threads, and for avoid
+                ;; unexpected blocking of other concurrent operations we
+                ;; dispatch that operation to a dedicated executor.
+                (let [result (p/thread-call executor (partial binfile/import! params))]
+                  (db/update! conn :project
+                              {:modified-at (dt/now)}
+                              {:id project-id})
 
-(def schema:clone-template
-  [:map {:title "clone-template"}
-   [:project-id ::sm/uuid]
-   [:template-id ::sm/word-string]])
-
-(sv/defmethod ::clone-template
-  "Clone into the specified project the template by its id."
-  {::doc/added "1.16"
-   ::webhooks/event? true
-   ::sm/params schema:clone-template}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (-> (assoc cfg ::db/conn conn)
-        (clone-template! (assoc params :profile-id profile-id)))))
+                  (deref result)))))
 
 ;; --- COMMAND: Get list of builtin templates
 
-(s/def ::retrieve-list-of-builtin-templates any?)
-
-(sv/defmethod ::retrieve-list-of-builtin-templates
-  {::doc/added "1.10"
-   ::doc/deprecated "1.19"}
-  [cfg _params]
-  (mapv #(select-keys % [:id :name]) (::setup/templates cfg)))
-
 (sv/defmethod ::get-builtin-templates
   {::doc/added "1.19"}
   [cfg _params]

backend/src/app/rpc/commands/media.clj

@@ -23,6 +23,7 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.services :as sv]
+   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.io :as io]))
@@ -60,7 +61,7 @@
     (files/check-edition-permissions! pool profile-id file-id)
     (media/validate-media-type! content)
     (media/validate-media-size! content)
-    (let [object (create-file-media-object cfg params)
+    (let [object (db/run! cfg #(create-file-media-object % params))
           props  {:name (:name params)
                   :file-id file-id
                   :is-local (:is-local params)
@@ -142,17 +143,17 @@
       (assoc ::image (process-main-image info)))))
 
 (defn create-file-media-object
-  [{:keys [::sto/storage ::db/pool] :as cfg}
+  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
    {:keys [id file-id is-local name content]}]
 
-  (let [result (-> (climit/configure cfg :process-image)
-                   (climit/submit! (partial process-image content)))
+  (let [result (-> (climit/configure cfg :process-image/global)
+                   (climit/run! (partial process-image content) executor))
 
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]
 
-    (db/exec-one! pool [sql:create-file-media-object
+    (db/exec-one! conn [sql:create-file-media-object
                         (or id (uuid/next))
                         file-id is-local name
                         (:id image)
@@ -176,9 +177,9 @@
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
     (files/check-edition-permissions! pool profile-id file-id)
-    (create-file-media-object-from-url cfg params)))
+    (db/run! cfg #(create-file-media-object-from-url % params))))
 
-(defn- download-image
+(defn download-image
   [{:keys [::http/client]} uri]
   (letfn [(parse-and-validate [{:keys [headers] :as response}]
             (let [size     (some-> (get headers "content-length") d/parse-integer)
@@ -209,7 +210,6 @@
                                                  {:method :get :uri uri}
                                                  {:response-type :input-stream :sync? true})
           {:keys [size mtype]} (parse-and-validate response)
-
           path    (tmp/tempfile :prefix "penpot.media.download.")
           written (io/write-to-file! body path :size size)]
 
@@ -223,7 +223,6 @@
        :path path
        :mtype mtype})))
 
-
 (defn- create-file-media-object-from-url
   [cfg {:keys [url name] :as params}]
   (let [content (download-image cfg url)

backend/src/app/rpc/commands/profile.clj

@@ -8,7 +8,6 @@
   (:require
    [app.auth :as auth]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.common.uuid :as uuid]
@@ -27,6 +26,7 @@
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [app.worker :as-alias wrk]
    [cuerdas.core :as str]))
 
 (declare check-profile-existence!)
@@ -37,30 +37,30 @@
 (declare strip-private-attrs)
 (declare verify-password)
 
-(def schema:profile
-  [:map {:title "Profile"}
-   [:id ::sm/uuid]
-   [:fullname [::sm/word-string {:max 250}]]
-   [:email ::sm/email]
-   [:is-active {:optional true} :boolean]
-   [:is-blocked {:optional true} :boolean]
-   [:is-demo {:optional true} :boolean]
-   [:is-muted {:optional true} :boolean]
-   [:created-at {:optional true} ::sm/inst]
-   [:modified-at {:optional true} ::sm/inst]
-   [:default-project-id {:optional true} ::sm/uuid]
-   [:default-team-id {:optional true} ::sm/uuid]
-   [:props {:optional true}
-    [:map-of {:title "ProfileProps"} :keyword :any]]])
-
-(def profile?
-  (sm/pred-fn schema:profile))
+(def ^:private
+  schema:profile
+  (sm/define
+    [:map {:title "Profile"}
+     [:id ::sm/uuid]
+     [:fullname [::sm/word-string {:max 250}]]
+     [:email ::sm/email]
+     [:is-active {:optional true} :boolean]
+     [:is-blocked {:optional true} :boolean]
+     [:is-demo {:optional true} :boolean]
+     [:is-muted {:optional true} :boolean]
+     [:created-at {:optional true} ::sm/inst]
+     [:modified-at {:optional true} ::sm/inst]
+     [:default-project-id {:optional true} ::sm/uuid]
+     [:default-team-id {:optional true} ::sm/uuid]
+     [:props {:optional true}
+      [:map-of {:title "ProfileProps"} :keyword :any]]]))
 
 ;; --- QUERY: Get profile (own)
 
 (sv/defmethod ::get-profile
   {::rpc/auth false
    ::doc/added "1.18"
+   ::sm/params [:map]
    ::sm/result schema:profile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id]}]
   ;; We need to return the anonymous profile object in two cases, when
@@ -81,11 +81,13 @@
 
 ;; --- MUTATION: Update Profile (own)
 
-(def schema:update-profile
-  [:map {:title "update-profile"}
-   [:fullname [::sm/word-string {:max 250}]]
-   [:lang {:optional true} [:string {:max 5}]]
-   [:theme {:optional true} [:string {:max 250}]]])
+(def ^:private
+  schema:update-profile
+  (sm/define
+    [:map {:title "update-profile"}
+     [:fullname [::sm/word-string {:max 250}]]
+     [:lang {:optional true} [:string {:max 5}]]
+     [:theme {:optional true} [:string {:max 250}]]]))
 
 (sv/defmethod ::update-profile
   {::doc/added "1.0"
@@ -93,10 +95,6 @@
    ::sm/result schema:profile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id fullname lang theme] :as params}]
 
-  (dm/assert!
-   "expected valid profile data"
-   (profile? params))
-
   (db/with-atomic [conn pool]
     ;; NOTE: we need to retrieve the profile independently if we use
     ;; it or not for explicit locking and avoid concurrent updates of
@@ -108,8 +106,7 @@
           profile (-> profile
                       (assoc :fullname fullname)
                       (assoc :lang lang)
-                      (assoc :theme theme))
-          ]
+                      (assoc :theme theme))]
 
       (db/update! conn :profile
                   {:fullname fullname
@@ -130,11 +127,13 @@
 (declare update-profile-password!)
 (declare invalidate-profile-session!)
 
-(def schema:update-profile-password
-  [:map {:title "update-profile-password"}
-   [:password [::sm/word-string {:max 500}]]
-   ;; Social registered users don't have old-password
-   [:old-password {:optional true} [:maybe [::sm/word-string {:max 500}]]]])
+(def ^:private
+  schema:update-profile-password
+  (sm/define
+    [:map {:title "update-profile-password"}
+     [:password [::sm/word-string {:max 500}]]
+     ;; Social registered users don't have old-password
+     [:old-password {:optional true} [:maybe [::sm/word-string {:max 500}]]]]))
 
 (sv/defmethod ::update-profile-password
   {:doc/added "1.0"
@@ -184,9 +183,11 @@
 (declare upload-photo)
 (declare update-profile-photo)
 
-(def schema:update-profile-photo
-  [:map {:title "update-profile-photo"}
-   [:file ::media/upload]])
+(def ^:private
+  schema:update-profile-photo
+  (sm/define
+    [:map {:title "update-profile-photo"}
+     [:file ::media/upload]]))
 
 (sv/defmethod ::update-profile-photo
   {:doc/added "1.1"
@@ -237,9 +238,9 @@
      :content-type (:mtype thumb)}))
 
 (defn upload-photo
-  [{:keys [::sto/storage] :as cfg} {:keys [file]}]
-  (let [params (-> (climit/configure cfg :process-image)
-                   (climit/submit! (partial generate-thumbnail! file)))]
+  [{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file]}]
+  (let [params (-> (climit/configure cfg :process-image/global)
+                   (climit/run! (partial generate-thumbnail! file) executor))]
     (sto/put-object! storage params)))
 
 
@@ -248,9 +249,11 @@
 (declare ^:private request-email-change!)
 (declare ^:private change-email-immediately!)
 
-(def schema:request-email-change
-  [:map {:title "request-email-change"}
-   [:email ::sm/email]])
+(def ^:private
+  schema:request-email-change
+  (sm/define
+    [:map {:title "request-email-change"}
+     [:email ::sm/email]]))
 
 (sv/defmethod ::request-email-change
   {::doc/added "1.0"
@@ -315,9 +318,11 @@
 
 ;; --- MUTATION: Update Profile Props
 
-(def schema:update-profile-props
-  [:map {:title "update-profile-props"}
-   [:props [:map-of :keyword :any]]])
+(def ^:private
+  schema:update-profile-props
+  (sm/define
+    [:map {:title "update-profile-props"}
+     [:props [:map-of :keyword :any]]]))
 
 (sv/defmethod ::update-profile-props
   {::doc/added "1.0"
@@ -433,13 +438,15 @@
 (defn derive-password
   [cfg password]
   (when password
-    (-> (climit/configure cfg :derive-password)
-        (climit/submit! (partial auth/derive-password password)))))
+    (-> (climit/configure cfg :derive-password/global)
+        (climit/run! (partial auth/derive-password password)
+                     (::wrk/executor cfg)))))
 
 (defn verify-password
   [cfg password password-data]
-  (-> (climit/configure cfg :derive-password)
-      (climit/submit! (partial auth/verify-password password password-data))))
+  (-> (climit/configure cfg :derive-password/global)
+      (climit/run! (partial auth/verify-password password password-data)
+                   (::wrk/executor cfg))))
 
 (defn decode-row
   [{:keys [props] :as row}]

backend/src/app/rpc/commands/projects.clj

@@ -50,11 +50,11 @@
         is-owner (boolean (some :is-owner rows))
         is-admin (boolean (some :is-admin rows))
         can-edit (boolean (some :can-edit rows))]
-     (when (seq rows)
-       {:is-owner is-owner
-        :is-admin (or is-owner is-admin)
-        :can-edit (or is-owner is-admin can-edit)
-        :can-read true})))
+    (when (seq rows)
+      {:is-owner is-owner
+       :is-admin (or is-owner is-admin)
+       :can-edit (or is-owner is-admin can-edit)
+       :can-read true})))
 
 (def has-edit-permissions?
   (perms/make-edition-predicate-fn get-permissions))

backend/src/app/rpc/commands/teams.clj

@@ -9,6 +9,7 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.spec :as us]
@@ -55,11 +56,11 @@
         is-owner (boolean (some :is-owner rows))
         is-admin (boolean (some :is-admin rows))
         can-edit (boolean (some :can-edit rows))]
-     (when (seq rows)
-       {:is-owner is-owner
-        :is-admin (or is-owner is-admin)
-        :can-edit (or is-owner is-admin can-edit)
-        :can-read true})))
+    (when (seq rows)
+      {:is-owner is-owner
+       :is-admin (or is-owner is-admin)
+       :can-edit (or is-owner is-admin can-edit)
+       :can-read true})))
 
 (def has-admin-permissions?
   (perms/make-admin-predicate-fn get-permissions))
@@ -79,22 +80,26 @@
 (def check-read-permissions!
   (perms/make-check-fn has-read-permissions?))
 
+(defn decode-row
+  [{:keys [features] :as row}]
+  (cond-> row
+    (some? features) (assoc :features (db/decode-pgarray features #{}))))
+
 ;; --- Query: Teams
 
-(declare retrieve-teams)
+(declare get-teams)
 
-(def counter (volatile! 0))
-
-(s/def ::get-teams
-  (s/keys :req [::rpc/profile-id]))
+(def ^:private schema:get-teams
+  [:map {:title "get-teams"}])
 
 (sv/defmethod ::get-teams
-  {::doc/added "1.17"}
+  {::doc/added "1.17"
+   ::sm/params schema:get-teams}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
   (dm/with-open [conn (db/open pool)]
-    (retrieve-teams conn profile-id)))
+    (get-teams conn profile-id)))
 
-(def sql:teams
+(def sql:get-teams-with-permissions
   "select t.*,
           tp.is_owner,
           tp.is_admin,
@@ -119,37 +124,77 @@
         (dissoc :is-owner :is-admin :can-edit)
         (assoc :permissions permissions))))
 
-(defn retrieve-teams
+(defn get-teams
   [conn profile-id]
   (let [profile (profile/get-profile conn profile-id)]
-    (->> (db/exec! conn [sql:teams (:default-team-id profile) profile-id])
-         (mapv process-permissions))))
+    (->> (db/exec! conn [sql:get-teams-with-permissions (:default-team-id profile) profile-id])
+         (map decode-row)
+         (map process-permissions)
+         (vec))))
 
 ;; --- Query: Team (by ID)
 
-(declare retrieve-team)
+(declare get-team)
 
-(s/def ::get-team
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::id]))
+(def ^:private schema:get-team
+  [:and
+   [:map {:title "get-team"}
+    [:id {:optional true} ::sm/uuid]
+    [:file-id {:optional true} ::sm/uuid]]
+
+   [:fn (fn [params]
+          (or (contains? params :id)
+              (contains? params :file-id)))]])
 
 (sv/defmethod ::get-team
-  {::doc/added "1.17"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
-  (dm/with-open [conn (db/open pool)]
-    (retrieve-team conn profile-id id)))
+  {::doc/added "1.17"
+   ::sm/params schema:get-team}
+  [{:keys [::db/pool]} {:keys [::rpc/profile-id id file-id]}]
+  (get-team pool :profile-id profile-id :team-id id :file-id file-id))
 
-(defn retrieve-team
-  [conn profile-id team-id]
-  (let [profile (profile/get-profile conn profile-id)
-        sql     (str "WITH teams AS (" sql:teams ") SELECT * FROM teams WHERE id=?")
-        result  (db/exec-one! conn [sql (:default-team-id profile) profile-id team-id])]
+(defn get-team
+  [conn & {:keys [profile-id team-id project-id file-id] :as params}]
+
+  (dm/assert!
+   "connection or pool is mandatory"
+   (or (db/connection? conn)
+       (db/pool? conn)))
+
+  (dm/assert!
+   "profile-id is mandatory"
+   (uuid? profile-id))
+
+  (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
+        result (cond
+                 (some? team-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
+                                ") SELECT * FROM teams WHERE id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id team-id]))
+
+                 (some? project-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
+                                "SELECT t.* FROM teams AS t "
+                                "  JOIN project AS p ON (p.team_id = t.id) "
+                                " WHERE p.id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id project-id]))
+
+                 (some? file-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
+                                "SELECT t.* FROM teams AS t "
+                                "  JOIN project AS p ON (p.team_id = t.id) "
+                                "  JOIN file AS f ON (f.project_id = p.id) "
+                                " WHERE f.id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id file-id]))
+
+                 :else
+                 (throw (IllegalArgumentException. "invalid arguments")))]
 
     (when-not result
       (ex/raise :type :not-found
                 :code :team-does-not-exist))
-
-    (process-permissions result)))
+    (-> result
+        (decode-row)
+        (process-permissions))))
 
 ;; --- Query: Team Members
 
@@ -165,44 +210,48 @@
      join profile as p on (p.id = tp.profile_id)
     where tp.team_id = ?")
 
-(defn retrieve-team-members
+(defn get-team-members
   [conn team-id]
   (db/exec! conn [sql:team-members team-id]))
 
-(s/def ::team-id ::us/uuid)
-(s/def ::get-team-members
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::team-id]))
+(def ^:private schema:get-team-memebrs
+  [:map {:title "get-team-members"}
+   [:team-id ::sm/uuid]])
 
 (sv/defmethod ::get-team-members
-  {::doc/added "1.17"}
+  {::doc/added "1.17"
+   ::sm/params schema:get-team-memebrs}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
-    (retrieve-team-members conn team-id)))
-
+    (get-team-members conn team-id)))
 
 ;; --- Query: Team Users
 
-(declare retrieve-users)
-(declare retrieve-team-for-file)
+(declare get-users)
+(declare get-team-for-file)
 
-(s/def ::get-team-users
-  (s/and (s/keys :req [::rpc/profile-id]
-                 :opt-un [::team-id ::file-id])
-         #(or (:team-id %) (:file-id %))))
+(def ^:private schema:get-team-users
+  [:and {:title "get-team-users"}
+   [:map
+    [:team-id {:optional true} ::sm/uuid]
+    [:file-id {:optional true} ::sm/uuid]]
+   [:fn #(or (contains? % :team-id)
+             (contains? % :file-id))]])
 
 (sv/defmethod ::get-team-users
-  {::doc/added "1.17"}
+  "Get team users by team-id or by file-id"
+  {::doc/added "1.17"
+   ::sm/params schema:get-team-users}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (if team-id
       (do
         (check-read-permissions! conn profile-id team-id)
-        (retrieve-users conn team-id))
-      (let [{team-id :id} (retrieve-team-for-file conn file-id)]
+        (get-users conn team-id))
+      (let [{team-id :id} (get-team-for-file conn file-id)]
         (check-read-permissions! conn profile-id team-id)
-        (retrieve-users conn team-id)))))
+        (get-users conn team-id)))))
 
 ;; This is a similar query to team members but can contain more data
 ;; because some user can be explicitly added to project or file (not
@@ -233,44 +282,44 @@
      join file as f on (p.id = f.project_id)
     where f.id = ?")
 
-(defn retrieve-users
+(defn get-users
   [conn team-id]
   (db/exec! conn [sql:team-users team-id team-id team-id]))
 
-(defn retrieve-team-for-file
+(defn get-team-for-file
   [conn file-id]
   (->> [sql:team-by-file file-id]
        (db/exec-one! conn)))
 
 ;; --- Query: Team Stats
 
-(declare retrieve-team-stats)
+(declare get-team-stats)
 
-(s/def ::get-team-stats
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::team-id]))
+(def ^:private schema:get-team-stats
+  [:map {:title "get-team-stats"}
+   [:team-id ::sm/uuid]])
 
 (sv/defmethod ::get-team-stats
-  {::doc/added "1.17"}
+  {::doc/added "1.17"
+   ::sm/params schema:get-team-stats}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
-    (retrieve-team-stats conn team-id)))
+    (get-team-stats conn team-id)))
 
 (def sql:team-stats
   "select (select count(*) from project where team_id = ?) as projects,
           (select count(*) from file as f join project as p on (p.id = f.project_id) where p.team_id = ?) as files")
 
-(defn retrieve-team-stats
+(defn get-team-stats
   [conn team-id]
   (db/exec-one! conn [sql:team-stats team-id team-id]))
 
-
 ;; --- Query: Team invitations
 
-(s/def ::get-team-invitations
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::team-id]))
+(def ^:private schema:get-team-invitations
+  [:map {:title "get-team-invitations"}
+   [:team-id ::sm/uuid]])
 
 (def sql:team-invitations
   "select email_to as email, role, (valid_until < now()) as expired
@@ -282,7 +331,8 @@
        (mapv #(update % :role keyword))))
 
 (sv/defmethod ::get-team-invitations
-  {::doc/added "1.17"}
+  {::doc/added "1.17"
+   ::sm/params schema:get-team-invitations}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
@@ -297,25 +347,32 @@
 (declare ^:private create-team-role)
 (declare ^:private create-team-default-project)
 
-(s/def ::create-team
-  (s/keys :req [::rpc/profile-id]
-          :req-un [::name]
-          :opt-un [::id]))
+(def ^:private schema:create-team
+  [:map {:title "create-team"}
+   [:name :string]
+   [:features {:optional true} ::cfeat/features]
+   [:id {:optional true} ::sm/uuid]])
 
 (sv/defmethod ::create-team
-  {::doc/added "1.17"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
-                               ::quotes/profile-id profile-id})
+  {::doc/added "1.17"
+   ::sm/params schema:create-team}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
+                                               ::quotes/profile-id profile-id})
 
-    (create-team conn (assoc params :profile-id profile-id))))
+                    (let [features (-> (cfeat/get-enabled-features cf/flags)
+                                       (cfeat/check-client-features! (:features params)))]
+                      (create-team cfg (assoc params
+                                              :profile-id profile-id
+                                              :features features))))))
 
 (defn create-team
   "This is a complete team creation process, it creates the team
   object and all related objects (default role and default project)."
-  [conn params]
-  (let [team    (create-team* conn params)
+  [cfg-or-conn params]
+  (let [conn    (db/get-connection cfg-or-conn)
+        team    (create-team* conn params)
         params  (assoc params
                        :team-id (:id team)
                        :role :owner)
@@ -324,13 +381,16 @@
     (assoc team :default-project-id (:id project))))
 
 (defn- create-team*
-  [conn {:keys [id name is-default] :as params}]
+  [conn {:keys [id name is-default features] :as params}]
   (let [id         (or id (uuid/next))
-        is-default (if (boolean? is-default) is-default false)]
-    (db/insert! conn :team
-                {:id id
-                 :name name
-                 :is-default is-default})))
+        is-default (if (boolean? is-default) is-default false)
+        features   (db/create-array conn "text" features)
+        team       (db/insert! conn :team
+                               {:id id
+                                :name name
+                                :features features
+                                :is-default is-default})]
+    (decode-row team)))
 
 (defn- create-team-role
   [conn {:keys [profile-id team-id role] :as params}]
@@ -396,7 +456,7 @@
 (defn leave-team
   [conn {:keys [profile-id id reassign-to]}]
   (let [perms   (get-permissions conn profile-id id)
-        members (retrieve-team-members conn id)]
+        members (get-team-members conn id)]
 
     (cond
       ;; we can only proceed if there are more members in the team
@@ -480,10 +540,15 @@
 
 (s/def ::team-id ::us/uuid)
 (s/def ::member-id ::us/uuid)
+(s/def ::role #{:owner :admin :editor})
+
 ;; Temporarily disabled viewer role
 ;; https://tree.taiga.io/project/penpot/issue/1083
-;; (s/def ::role #{:owner :admin :editor :viewer})
-(s/def ::role #{:owner :admin :editor})
+(def valid-roles
+  #{:owner :admin :editor #_:viewer})
+
+(def schema:role
+  [::sm/one-of valid-roles])
 
 (defn role->params
   [role]
@@ -500,7 +565,7 @@
   ;; convenience, if this becomes a bottleneck or problematic,
   ;; we will change it to more efficient fetch mechanisms.
   (let [perms   (get-permissions conn profile-id team-id)
-        members (retrieve-team-members conn team-id)
+        members (get-team-members conn team-id)
         member  (d/seek #(= member-id (:id %)) members)
 
         is-owner? (:is-owner perms)
@@ -596,7 +661,7 @@
 
 (defn update-team-photo
   [{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id team-id] :as params}]
-  (let [team  (retrieve-team pool profile-id team-id)
+  (let [team  (get-team pool :profile-id profile-id :team-id team-id)
         photo (profile/upload-photo cfg params)]
 
     (db/with-atomic [conn pool]
@@ -608,8 +673,8 @@
 
       ;; Save new photo
       (db/update! pool :team
-        {:photo-id (:id photo)}
-        {:id team-id})
+                  {:photo-id (:id photo)}
+                  {:id team-id})
 
       (assoc team :photo-id (:id photo)))))
 
@@ -670,7 +735,8 @@
                           (role->params role))]
 
         ;; Insert the invited member to the team
-        (db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
+        (db/insert! conn :team-profile-rel params
+                    {::db/on-conflict-do-nothing? true})
 
         ;; If profile is not yet verified, mark it as verified because
         ;; accepting an invitation link serves as verification.
@@ -784,14 +850,24 @@
   (s/merge ::create-team
            (s/keys :req-un [::emails ::role])))
 
+
+(def ^:private schema:create-team-with-invitations
+  [:map {:title "create-team-with-invitations"}
+   [:name :string]
+   [:features {:optional true} ::cfeat/features]
+   [:id {:optional true} ::sm/uuid]
+   [:emails ::sm/set-of-emails]
+   [:role schema:role]])
+
 (sv/defmethod ::create-team-with-invitations
-  {::doc/added "1.17"}
+  {::doc/added "1.17"
+   ::sm/params schema:create-team-with-invitations}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id emails role] :as params}]
   (db/with-atomic [conn pool]
     (let [params   (assoc params :profile-id profile-id)
-          team     (create-team conn params)
-          profile  (db/get-by-id conn :profile profile-id)
-          cfg      (assoc cfg ::db/conn conn)]
+          cfg      (assoc cfg ::db/conn conn)
+          team     (create-team cfg params)
+          profile  (db/get-by-id conn :profile profile-id)]
 
       ;; Create invitations for all provided emails.
       (->> emails

backend/src/app/rpc/commands/verify_token.clj

@@ -105,7 +105,7 @@
                           ::quotes/team-id team-id})
 
     ;; Insert the invited member to the team
-    (db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
+    (db/insert! conn :team-profile-rel params {::db/on-conflict-do-nothing? true})
 
     ;; If profile is not yet verified, mark it as verified because
     ;; accepting an invitation link serves as verification.

backend/src/app/rpc/commands/viewer.clj

@@ -6,27 +6,49 @@
 
 (ns app.rpc.commands.viewer
   (:require
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.schema :as sm]
+   [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.comments :as comments]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]))
 
 ;; --- QUERY: View Only Bundle
 
-(defn- get-project
-  [conn id]
-  (db/get-by-id conn :project id {:columns [:id :name :team-id]}))
+(defn- remove-not-allowed-pages
+  [data allowed]
+  (-> data
+      (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
+      (update :pages-index select-keys allowed)))
+
+(defn- get-view-only-bundle
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
+  (let [file    (files/get-file cfg file-id)
+
+        project (db/get conn :project
+                        {:id (:project-id file)}
+                        {:columns [:id :name :team-id]})
+
+        team    (-> (db/get conn :team {:id (:team-id project)})
+                    (teams/decode-row))
+
+        _       (-> (cfeat/get-team-enabled-features cf/flags team)
+                    (cfeat/check-client-features! (:features params))
+                    (cfeat/check-file-features! (:features file)))
+
+        file    (cond-> file
+                  (= :share-link (:type perms))
+                  (update :data remove-not-allowed-pages (:pages perms))
+
+                  :always
+                  (update :data select-keys [:id :options :pages :pages-index :components]))
 
-(defn- get-bundle
-  [conn file-id profile-id features]
-  (let [file    (files/get-file conn file-id features)
-        project (get-project conn (:project-id file))
         libs    (files/get-file-libraries conn file-id)
         users   (comments/get-file-comments-users conn file-id profile-id)
         links   (->> (db/query conn :share-link {:file-id file-id})
@@ -41,54 +63,43 @@
                                  (dissoc :flags)))))
 
         fonts   (db/query conn :team-font-variant
-                          {:team-id (:team-id project)
+                          {:team-id (:id team)
                            :deleted-at nil})]
 
-    {:file file
-     :users users
+    {:users users
      :fonts fonts
      :project project
      :share-links links
-     :libraries libs}))
+     :libraries libs
+     :file file
+     :team team
+     :permissions perms}))
 
-(defn- remove-not-allowed-pages
-  [data allowed]
-  (-> data
-      (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
-      (update :pages-index select-keys allowed)))
-
-(defn get-view-only-bundle
-  [conn {:keys [profile-id file-id share-id features] :as params}]
-  (let [perms  (files/get-permissions conn profile-id file-id share-id)
-        bundle (-> (get-bundle conn file-id profile-id features)
-                   (assoc :permissions perms))]
-
-    ;; When we have neither profile nor share, we just return a not
-    ;; found response to the user.
-    (when-not perms
-      (ex/raise :type :not-found
-                :code :object-not-found
-                :hint "object not found"))
-
-    (update bundle :file
-            (fn [file]
-              (cond-> file
-                (= :share-link (:type perms))
-                (update :data remove-not-allowed-pages (:pages perms))
-
-                :always
-                (update :data select-keys [:id :options :pages :pages-index :components]))))))
-
-(sm/def! ::get-view-only-bundle
+(def schema:get-view-only-bundle
   [:map {:title "get-view-only-bundle"}
    [:file-id ::sm/uuid]
    [:share-id {:optional true} ::sm/uuid]
-   [:features {:optional true} ::files/features]])
+   [:features {:optional true} ::cfeat/features]])
 
 (sv/defmethod ::get-view-only-bundle
   {::rpc/auth false
    ::doc/added "1.17"
-   ::sm/params ::get-view-only-bundle}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id] :as params}]
-  (dm/with-open [conn (db/open pool)]
-    (get-view-only-bundle conn (assoc params :profile-id profile-id))))
+   ::sm/params schema:get-view-only-bundle}
+  [system {:keys [::rpc/profile-id file-id share-id] :as params}]
+  (db/run! system
+           (fn [{:keys [::db/conn] :as system}]
+             (let [perms  (files/get-permissions conn profile-id file-id share-id)
+                   params (-> params
+                              (assoc ::perms perms)
+                              (assoc :profile-id profile-id))]
+
+                  ;; When we have neither profile nor share, we just return a not
+               ;; found response to the user.
+               (when-not perms
+                 (ex/raise :type :not-found
+                           :code :object-not-found
+                           :hint "object not found"))
+
+               (get-view-only-bundle system params)))))
+
+

backend/src/app/rpc/cond.clj

@@ -29,7 +29,7 @@
    [app.util.services :as-alias sv]
    [buddy.core.codecs :as bc]
    [buddy.core.hash :as bh]
-   [yetti.response :as yrs]))
+   [ring.response :as-alias rres]))
 
 (def
   ^{:dynamic true
@@ -57,7 +57,7 @@
           (let [key' (when (or key reuse-key?)
                        (some->> (get-object cfg params) (key-fn params) (fmt-key)))]
             (if (and (some? key) (= key key'))
-              (fn [_] {::yrs/status 304})
+              (fn [_] {::rres/status 304})
               (let [result (f cfg params)
                     etag   (or (and reuse-key? key')
                                (some-> result meta ::key fmt-key)

backend/src/app/rpc/doc.clj

@@ -16,6 +16,7 @@
    [app.common.schema.openapi :as oapi]
    [app.common.schema.registry :as sr]
    [app.config :as cf]
+   [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.util.json :as json]
@@ -27,7 +28,7 @@
    [integrant.core :as ig]
    [malli.transform :as mt]
    [pretty-spec.core :as ps]
-   [yetti.response :as yrs]))
+   [ring.response :as-alias rres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DOC (human readable)
@@ -55,6 +56,7 @@
              :module (or (some-> (::module mdata) d/name)
                          (-> (:ns mdata) (str/split ".") last))
              :auth (::rpc/auth mdata true)
+             :sse (::sse/stream? mdata false)
              :webhook (::webhooks/event? mdata false)
              :docs (::sv/docstring mdata)
              :deprecated (::deprecated mdata)
@@ -86,11 +88,11 @@
       (let [params  (:query-params request)
             pstyle  (:type params "js")
             context (assoc context :param-style pstyle)]
-        {::yrs/status 200
-         ::yrs/body (-> (io/resource "app/templates/api-doc.tmpl")
-                        (tmpl/render context))}))
+        {::rres/status 200
+         ::rres/body (-> (io/resource "app/templates/api-doc.tmpl")
+                         (tmpl/render context))}))
     (fn [_]
-      {::yrs/status 404})))
+      {::rres/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OPENAPI / SWAGGER (v3.1)
@@ -141,8 +143,7 @@
 
               {:name (-> mdata ::sv/name d/name)
                :module (-> (:ns mdata) (str/split ".") last)
-               :repr {:post rpost}}))
-          ]
+               :repr {:post rpost}}))]
 
     (let [definitions (atom {})
           options {:registry sr/default-registry
@@ -158,27 +159,27 @@
                          (map (fn [doc]
                                 [(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
                          (into {})))]
-    {:openapi "3.0.0"
-     :info {:version (:main cf/version)}
-     :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
+      {:openapi "3.0.0"
+       :info {:version (:main cf/version)}
+       :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
                 ;; :description "penpot backend"
-                }]
-     :security
-     {:api_key []}
+                  }]
+       :security
+       {:api_key []}
 
-     :paths paths
-     :components {:schemas @definitions}})))
+       :paths paths
+       :components {:schemas @definitions}})))
 
 (defn openapi-json-handler
   [context]
   (if (contains? cf/flags :backend-openapi-doc)
     (fn [_]
-      {::yrs/status 200
-       ::yrs/headers {"content-type" "application/json; charset=utf-8"}
-       ::yrs/body (json/encode context)})
+      {::rres/status 200
+       ::rres/headers {"content-type" "application/json; charset=utf-8"}
+       ::rres/body (json/encode context)})
 
     (fn [_]
-      {::yrs/status 404})))
+      {::rres/status 404})))
 
 (defn openapi-handler
   []
@@ -189,12 +190,12 @@
             context    {:public-uri (cf/get :public-uri)
                         :swagger-js swagger-js
                         :swagger-css swagger-cs}]
-        {::yrs/status 200
-         ::yrs/headers {"content-type" "text/html"}
-         ::yrs/body (-> (io/resource "app/templates/openapi.tmpl")
-                        (tmpl/render context))}))
+        {::rres/status 200
+         ::rres/headers {"content-type" "text/html"}
+         ::rres/body (-> (io/resource "app/templates/openapi.tmpl")
+                         (tmpl/render context))}))
     (fn [_]
-      {::yrs/status 404})))
+      {::rres/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MODULE INIT

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [yetti.response :as-alias yrs]))
+   [ring.response :as-alias rres]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -77,4 +77,4 @@
              (fn [_ response]
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
-                 (update response ::yrs/headers assoc "cache-control" val)))))
+                 (update response ::rres/headers assoc "cache-control" val)))))

backend/src/app/setup/templates.clj

@@ -19,14 +19,18 @@
    [datoteka.fs :as fs]
    [integrant.core :as ig]))
 
-(def ^:private schema:template
-  [:map {:title "Template"}
-   [:id ::sm/word-string]
-   [:name ::sm/word-string]
-   [:file-uri ::sm/word-string]])
+(def ^:private
+  schema:template
+  (sm/define
+    [:map {:title "Template"}
+     [:id ::sm/word-string]
+     [:name ::sm/word-string]
+     [:file-uri ::sm/word-string]]))
 
-(def ^:private schema:templates
-  [:vector schema:template])
+(def ^:private
+  schema:templates
+  (sm/define
+    [:vector schema:template]))
 
 (defmethod ig/init-key ::setup/templates
   [_ _]
@@ -35,13 +39,13 @@
 
     (dm/verify!
      "expected a valid templates file"
-     (sm/valid? schema:templates templates))
+     (sm/check! schema:templates templates))
 
     (doseq [{:keys [id path] :as template} templates]
       (let [path (or path (fs/join dest id))]
         (if (fs/exists? path)
-          (l/debug :hint "template file" :id id :state "present" :path (dm/str path))
-          (l/debug :hint "template file" :id id :state "absent"))))
+          (l/dbg :hint "template file" :id id :state "present" :path (dm/str path))
+          (l/dbg :hint "template file" :id id :state "absent"))))
 
     templates))
 

backend/src/app/srepl.clj

@@ -10,7 +10,7 @@
    [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cf]
-   [app.srepl.ext]
+   [app.srepl.cli]
    [app.srepl.main]
    [app.util.json :as json]
    [app.util.locks :as locks]
@@ -36,7 +36,9 @@
         lock (locks/create)]
     (ccs/prepl *in*
                (fn [m]
-                 (binding [*out* out, *flush-on-newline* true, *print-readably* true]
+                 (binding [*out* out,
+                           *flush-on-newline* true,
+                           *print-readably* true]
                    (locks/locking lock
                      (println (json/encode-str m))))))))
 
@@ -44,13 +46,10 @@
 
 (s/def ::port ::us/integer)
 (s/def ::host ::us/not-empty-string)
-(s/def ::flag #{:urepl-server :prepl-server})
-(s/def ::type #{::prepl ::urepl})
-(s/def ::key (s/tuple ::type ::us/keyword))
 
 (defmethod ig/pre-init-spec ::server
   [_]
-  (s/keys :req [::flag ::host ::port]))
+  (s/keys :req [::host ::port]))
 
 (defmethod ig/prep-key ::server
   [[type _] cfg]
@@ -59,6 +58,12 @@
 (defmethod ig/init-key ::server
   [[type _] {:keys [::flag ::port ::host] :as cfg}]
   (when (contains? cf/flags flag)
+
+    (l/inf :hint "initializing repl server"
+           :name (name type)
+           :port port
+           :host host)
+
     (let [accept (case type
                    ::prepl 'app.srepl/json-repl
                    ::urepl 'app.srepl/user-repl)
@@ -67,14 +72,8 @@
                   :name (name type)
                   :accept accept}]
 
-      (l/info :msg "initializing repl server"
-              :name (name type)
-              :port port
-              :host host)
-
       (ccs/start-server params)
-
-      params)))
+      (assoc params :type type))))
 
 (defmethod ig/halt-key! ::server
   [_ params]

backend/src/app/srepl/cli.clj

@@ -4,14 +4,16 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
-(ns app.srepl.ext
+(ns app.srepl.cli
   "PREPL API for external usage (CLI or ADMIN)"
   (:require
    [app.auth :as auth]
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
    [app.db :as db]
+   [app.main :as main]
    [app.rpc.commands.auth :as cmd.auth]
+   [app.srepl.components-v2]
    [app.util.json :as json]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
@@ -21,18 +23,18 @@
   (or (deref (requiring-resolve 'app.main/system))
       (deref (requiring-resolve 'user/system))))
 
-(defmulti ^:private run-json-cmd* ::cmd)
+(defmulti ^:private exec-command ::cmd)
 
-(defn run-json-cmd
+(defn exec
   "Entry point with external tools integrations that uses PREPL
   interface for interacting with running penpot backend."
   [data]
-  (let [data (json/decode data)
-        params (merge {::cmd (keyword (:cmd data "default"))}
-                      (:params data))]
-    (run-json-cmd* params)))
+  (let [data (json/decode data)]
+    (-> {::cmd (keyword (:cmd data "default"))}
+        (merge (:params data))
+        (exec-command))))
 
-(defmethod run-json-cmd* :create-profile
+(defmethod exec-command :create-profile
   [{:keys [fullname email password is-active]
     :or {is-active true}}]
   (when-let [system (get-current-system)]
@@ -46,7 +48,7 @@
         (->> (cmd.auth/create-profile! conn params)
              (cmd.auth/create-profile-rels! conn))))))
 
-(defmethod run-json-cmd* :update-profile
+(defmethod exec-command :update-profile
   [{:keys [fullname email password is-active]}]
   (when-let [system (get-current-system)]
     (db/with-atomic [conn (:app.db/pool system)]
@@ -67,7 +69,7 @@
                                 {::db/return-keys? false})]
             (pos? (:next.jdbc/update-count res))))))))
 
-(defmethod run-json-cmd* :delete-profile
+(defmethod exec-command :delete-profile
   [{:keys [email soft]}]
   (when-not email
     (ex/raise :type :assertion
@@ -87,7 +89,7 @@
                               {::db/return-keys? false}))]
         (pos? (:next.jdbc/update-count res))))))
 
-(defmethod run-json-cmd* :search-profile
+(defmethod exec-command :search-profile
   [{:keys [email]}]
   (when-not email
     (ex/raise :type :assertion
@@ -101,11 +103,33 @@
                      " where email similar to ? order by created_at desc limit 100")]
         (db/exec! conn [sql email])))))
 
-(defmethod run-json-cmd* :derive-password
+(defmethod exec-command :derive-password
   [{:keys [password]}]
   (auth/derive-password password))
 
-(defmethod run-json-cmd* :default
+(defmethod exec-command :migrate-v2
+  [_]
+  (letfn [(on-start [{:keys [total rollback]}]
+            (println
+             (str/ffmt "The components/v2 migration started (rollback:%, teams:%)"
+                       (if rollback "on" "off")
+                       total)))
+
+          (on-progress [{:keys [total elapsed progress completed]}]
+            (println (str/ffmt "Progress % (total: %, completed: %, elapsed: %)"
+                               progress total completed elapsed)))
+          (on-error [cause]
+            (println "ERR:" (ex-message cause)))
+
+          (on-end [_]
+            (println "Migration finished"))]
+    (app.srepl.components-v2/migrate-teams! main/system
+                                            :on-start on-start
+                                            :on-error on-error
+                                            :on-progress on-progress
+                                            :on-end on-end)))
+
+(defmethod exec-command :default
   [{:keys [::cmd]}]
   (ex/raise :type :internal
             :code :not-implemented

backend/src/app/srepl/components_v2.clj

@@ -0,0 +1,296 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.srepl.components-v2
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.logging :as l]
+   [app.common.pprint :as pp]
+   [app.db :as db]
+   [app.features.components-v2 :as feat]
+   [app.util.time :as dt]
+   [cuerdas.core :as str]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [promesa.exec.semaphore :as ps]
+   [promesa.util :as pu]))
+
+(defn- print-stats!
+  [stats]
+  (->> stats
+       (into (sorted-map))
+       (pp/pprint)))
+
+(defn- report-progress-files
+  [tpoint]
+  (fn [_ _ oldv newv]
+    (when (not= (:processed/files oldv)
+                (:processed/files newv))
+      (let [total     (:total/files newv)
+            completed (:processed/files newv)
+            progress  (/ (* completed 100.0) total)
+            elapsed   (tpoint)]
+        (l/dbg :hint "progress"
+               :completed (:processed/files newv)
+               :total     (:total/files newv)
+               :progress  (str (int progress) "%")
+               :elapsed   (dt/format-duration elapsed))))))
+
+(defn- report-progress-teams
+  [tpoint on-progress]
+  (fn [_ _ oldv newv]
+    (when (not= (:processed/teams oldv)
+                (:processed/teams newv))
+      (let [total     (:total/teams newv)
+            completed (:processed/teams newv)
+            progress  (/ (* completed 100.0) total)
+            progress  (str (int progress) "%")
+            elapsed   (dt/format-duration (tpoint))]
+
+        (when (fn? on-progress)
+          (on-progress {:total total
+                        :elapsed elapsed
+                        :completed completed
+                        :progress progress}))
+
+        (l/dbg :hint "progress"
+               :completed completed
+               :progress progress
+               :elapsed elapsed)))))
+
+(defn- get-total-files
+  [pool & {:keys [team-id]}]
+  (if (some? team-id)
+    (let [sql (str/concat
+               "SELECT count(f.id) AS count FROM file AS f "
+               "  JOIN project AS p ON (p.id = f.project_id) "
+               " WHERE p.team_id = ? AND f.deleted_at IS NULL "
+               "  AND p.deleted_at IS NULL")
+          res (db/exec-one! pool [sql team-id])]
+      (:count res))
+
+    (let [sql (str/concat
+               "SELECT count(id) AS count FROM file "
+               " WHERE deleted_at IS NULL")
+          res (db/exec-one! pool [sql])]
+      (:count res))))
+
+(defn- get-total-teams
+  [pool]
+  (let [sql (str/concat
+             "SELECT count(id) AS count FROM team "
+             " WHERE deleted_at IS NULL")
+        res (db/exec-one! pool [sql])]
+    (:count res)))
+
+(defn migrate-file!
+  [system file-id & {:keys [rollback?] :or {rollback? true}}]
+
+  (l/dbg :hint "migrate:start")
+  (let [tpoint (dt/tpoint)]
+    (try
+      (binding [feat/*stats* (atom {})]
+        (-> (assoc system ::db/rollback rollback?)
+            (feat/migrate-file! file-id))
+
+        (-> (deref feat/*stats*)
+            (assoc :elapsed (dt/format-duration (tpoint)))))
+
+      (catch Throwable cause
+        (l/wrn :hint "migrate:error" :cause cause))
+
+      (finally
+        (let [elapsed (dt/format-duration (tpoint))]
+          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
+
+(defn migrate-files!
+  [{:keys [::db/pool] :as system}
+   & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
+      :or {chunk-size 10
+           skip-on-error true
+           max-jobs 10
+           max-items Long/MAX_VALUE
+           preset :shutdown-on-failure
+           rollback? true
+           validate? false}}]
+  (letfn [(get-chunk [cursor]
+            (let [sql  (str/concat
+                        "SELECT id, created_at FROM file "
+                        " WHERE created_at < ? AND deleted_at IS NULL "
+                        " ORDER BY created_at desc LIMIT ?")
+                  rows (db/exec! pool [sql cursor chunk-size])]
+              [(some->> rows peek :created-at) (seq rows)]))
+
+          (get-candidates []
+            (->> (d/iteration get-chunk
+                              :vf second
+                              :kf first
+                              :initk (or start-at (dt/now)))
+                 (take max-items)
+                 (map :id)))]
+
+    (l/dbg :hint "migrate:start")
+    (let [fsem   (ps/create :permits max-jobs)
+          total  (get-total-files pool)
+          stats  (atom {:files/total total})
+          tpoint (dt/tpoint)]
+
+      (add-watch stats :progress-report (report-progress-files tpoint))
+
+      (binding [feat/*stats* stats
+                feat/*semaphore* fsem
+                feat/*skip-on-error* skip-on-error]
+        (try
+          (pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)]
+
+            (run! (fn [file-id]
+                    (ps/acquire! feat/*semaphore*)
+                    (px/submit! scope (fn []
+                                        (-> (assoc system ::db/rollback rollback?)
+                                            (feat/migrate-file! file-id
+                                                                :validate? validate?
+                                                                :throw-on-validate? (not skip-on-error))))))
+                  (get-candidates))
+
+            (p/await! scope))
+
+          (-> (deref feat/*stats*)
+              (assoc :elapsed (dt/format-duration (tpoint))))
+
+          (catch Throwable cause
+            (l/dbg :hint "migrate:error" :cause cause))
+
+          (finally
+            (let [elapsed (dt/format-duration (tpoint))]
+              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))
+
+(defn migrate-team!
+  [{:keys [::db/pool] :as system} team-id
+   & {:keys [rollback? skip-on-error validate?]
+      :or {rollback? true skip-on-error true validate? false}}]
+  (l/dbg :hint "migrate:start")
+
+  (let [total  (get-total-files pool :team-id team-id)
+        stats  (atom {:total/files total})
+        tpoint (dt/tpoint)]
+
+    (add-watch stats :progress-report (report-progress-files tpoint))
+
+    (try
+      (binding [feat/*stats* stats
+                feat/*skip-on-error* skip-on-error]
+        (-> (assoc system ::db/rollback rollback?)
+            (feat/migrate-team! team-id
+                                :validate? validate?
+                                :throw-on-validate? (not skip-on-error)))
+
+        (print-stats!
+         (-> (deref feat/*stats*)
+             (dissoc :total/files)
+             (assoc :elapsed (dt/format-duration (tpoint))))))
+
+      (catch Throwable cause
+        (l/dbg :hint "migrate:error" :cause cause))
+
+      (finally
+        (let [elapsed (dt/format-duration (tpoint))]
+          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
+
+(defn default-on-end
+  [stats]
+  (print-stats!
+   (-> stats
+       (update :elapsed/total dt/format-duration)
+       (dissoc :total/teams))))
+
+(defn migrate-teams!
+  [{:keys [::db/pool] :as system}
+   & {:keys [chunk-size max-jobs max-items start-at
+             rollback? validate? preset skip-on-error
+             max-time on-start on-progress on-error on-end]
+      :or {chunk-size 10000
+           validate? false
+           rollback? true
+           skip-on-error true
+           on-end default-on-end
+           preset :shutdown-on-failure
+           max-jobs Integer/MAX_VALUE
+           max-items Long/MAX_VALUE}}]
+
+  (letfn [(get-chunk [cursor]
+            (let [sql  (str/concat
+                        "SELECT id, created_at, features FROM team "
+                        " WHERE created_at < ? AND deleted_at IS NULL "
+                        " ORDER BY created_at desc LIMIT ?")
+                  rows (db/exec! pool [sql cursor chunk-size])]
+              [(some->> rows peek :created-at) (seq rows)]))
+
+          (get-candidates []
+            (->> (d/iteration get-chunk
+                              :vf second
+                              :kf first
+                              :initk (or start-at (dt/now)))
+                 (map #(update % :features db/decode-pgarray #{}))
+                 (remove #(contains? (:features %) "ephimeral/v2-migration"))
+                 (take max-items)
+                 (map :id)))
+
+          (migrate-team [team-id]
+            (try
+              (-> (assoc system ::db/rollback rollback?)
+                  (feat/migrate-team! team-id
+                                      :validate? validate?
+                                      :throw-on-validate? (not skip-on-error)))
+              (catch Throwable cause
+                (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
+
+          (process-team [scope tpoint mtime team-id]
+            (ps/acquire! feat/*semaphore*)
+            (let [ts (tpoint)]
+              (if (and mtime (neg? (compare mtime ts)))
+                (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts))
+                (px/submit! scope (partial migrate-team team-id)))))]
+
+    (l/dbg :hint "migrate:start")
+
+    (let [sem    (ps/create :permits max-jobs)
+          total  (get-total-teams pool)
+          stats  (atom {:total/teams (min total max-items)})
+          tpoint (dt/tpoint)
+          mtime  (some-> max-time dt/duration)]
+
+      (when (fn? on-start)
+        (on-start {:total total :rollback rollback?}))
+
+      (add-watch stats :progress-report (report-progress-teams tpoint on-progress))
+
+      (binding [feat/*stats* stats
+                feat/*semaphore* sem
+                feat/*skip-on-error* skip-on-error]
+        (try
+          (pu/with-open [scope (px/structured-task-scope :preset preset
+                                                         :factory :virtual)]
+            (loop [candidates (get-candidates)]
+              (when-let [team-id (first candidates)]
+                (when (process-team scope tpoint mtime team-id)
+                  (recur (rest candidates)))))
+
+            (p/await! scope))
+
+          (when (fn? on-end)
+            (-> (deref stats)
+                (assoc :elapsed/total (tpoint))
+                (on-end)))
+
+          (catch Throwable cause
+            (l/dbg :hint "migrate:error" :cause cause)
+            (when (fn? on-error)
+              (on-error cause)))
+
+          (finally
+            (let [elapsed (dt/format-duration (tpoint))]
+              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))

backend/src/app/srepl/fixes.clj

@@ -1,75 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.srepl.fixes
-  "A collection of adhoc fixes scripts."
-  (:require
-   [app.common.logging :as l]
-   [app.common.uuid :as uuid]
-   [app.srepl.helpers :as h]))
-
-(defn repair-orphaned-shapes
-  "There are some shapes whose parent has been deleted. This function
-  detects them and puts them as children of the root node."
-  ([data]
-   (letfn [(is-orphan? [shape objects]
-             (and (some? (:parent-id shape))
-                  (nil? (get objects (:parent-id shape)))))
-
-           (update-page [page]
-             (let [objects (:objects page)
-                   orphans (into #{} (filter #(is-orphan? % objects)) (vals objects))]
-               (if (seq orphans)
-                 (do
-                   (l/info :hint "found a file with orphans" :file-id (:id data) :broken-shapes (count orphans))
-                   (-> page
-                       (h/update-shapes (fn [shape]
-                                          (if (contains? orphans shape)
-                                            (assoc shape :parent-id uuid/zero)
-                                            shape)))
-                       (update-in [:objects uuid/zero :shapes] into (map :id) orphans)))
-                 page)))]
-
-     (h/update-pages data update-page)))
-
-  ;; special arity for to be called from h/analyze-files to search for
-  ;; files with possible issues
-
-  ([file state]
-   (repair-orphaned-shapes (:data file))
-   (update state :total (fnil inc 0))))
-
-(defn rename-layout-attrs
-  ([file]
-   (let [found? (volatile! false)]
-     (letfn [(update-shape
-               [shape]
-               (when (or (= (:layout-flex-dir shape) :reverse-row)
-                         (= (:layout-flex-dir shape) :reverse-column)
-                         (= (:layout-wrap-type shape) :no-wrap))
-                 (vreset! found? true))
-               (cond-> shape
-                 (= (:layout-flex-dir shape) :reverse-row)
-                 (assoc :layout-flex-dir :row-reverse)
-                 (= (:layout-flex-dir shape) :reverse-column)
-                 (assoc :layout-flex-dir :column-reverse)
-                 (= (:layout-wrap-type shape) :no-wrap)
-                 (assoc :layout-wrap-type :nowrap)))
-
-             (update-page
-               [page]
-               (h/update-shapes page update-shape))]
-
-       (let [new-file (update file :data h/update-pages update-page)]
-         (when @found?
-           (l/info :hint "Found attrs to rename in file"
-                   :id (:id file)
-                   :name (:name file)))
-         new-file))))
-
-   ([file state]
-    (rename-layout-attrs file)
-    (update state :total (fnil inc 0))))

backend/src/app/srepl/helpers.clj

@@ -6,23 +6,27 @@
 
 (ns app.srepl.helpers
   "A  main namespace for server repl."
+  (:refer-clojure :exclude [parse-uuid])
   #_:clj-kondo/ignore
   (:require
-   [app.auth :refer [derive-password]]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.files.features :as ffeat]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as cpc]
+   [app.common.files.migrations :as pmg]
+   [app.common.files.repair :as repair]
+   [app.common.files.validate :as validate]
    [app.common.logging :as l]
-   [app.common.pages :as cp]
-   [app.common.pages.migrations :as pmg]
    [app.common.pprint :refer [pprint]]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cfg]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.features.fdata :as feat.fdata]
    [app.main :refer [system]]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.files-update :as files-update]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
@@ -31,124 +35,278 @@
    [clojure.stacktrace :as strace]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
-   [expound.alpha :as expound]))
+   [expound.alpha :as expound]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [promesa.exec.csp :as sp]))
 
-(def ^:dynamic *conn*)
+(def ^:dynamic *system* nil)
 
-(defn reset-password!
-  "Reset a password to a specific one for a concrete user or all users
-  if email is `:all` keyword."
-  [system & {:keys [email password] :or {password "123123"} :as params}]
-  (us/verify! (contains? params :email) "`email` parameter is mandatory")
-  (db/with-atomic [conn (:app.db/pool system)]
-    (let [password (derive-password password)]
-      (if (= email :all)
-        (db/exec! conn ["update profile set password=?" password])
-        (let [email (str/lower email)]
-          (db/exec! conn ["update profile set password=? where email=?" password email]))))))
+(defn println!
+  [& params]
+  (locking println
+    (apply println params)))
+
+(defn parse-uuid
+  [v]
+  (if (uuid? v)
+    v
+    (d/parse-uuid v)))
 
 (defn reset-file-data!
   "Hardcode replace of the data of one file."
   [system id data]
-  (db/with-atomic [conn (:app.db/pool system)]
-    (db/update! conn :file
-                {:data data}
-                {:id id})))
+  (db/tx-run! system (fn [system]
+                       (db/update! system :file
+                                   {:data data}
+                                   {:id id}))))
 
 (defn get-file
   "Get the migrated data of one file."
+  [system id & {:keys [migrate?] :or {migrate? true}}]
+  (db/run! system
+           (fn [system]
+             (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+               (-> (files/get-file system id :migrate? migrate?)
+                   (update :data feat.fdata/process-pointers deref))))))
+
+(defn validate
+  "Validate structure, referencial integrity and semantic coherence of
+    all contents of a file. Returns a list of errors."
   [system id]
-  (db/with-atomic [conn (:app.db/pool system)]
-    (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
-      (-> (db/get-by-id conn :file id)
-          (update :data blob/decode)
-          (update :data pmg/migrate-data)
-          (files/process-pointers deref)))))
+  (db/tx-run! system
+              (fn [{:keys [::db/conn] :as system}]
+                (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+                  (let [id   (if (string? id) (parse-uuid id) id)
+                        file (files/get-file system id)
+                        libs (->> (files/get-file-libraries conn id)
+                                  (into [file] (map (fn [{:keys [id]}]
+                                                      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+                                                        (-> (files/get-file system id :migrate? false)
+                                                            (feat.fdata/process-pointers deref)
+                                                            (pmg/migrate-file))))))
+                                  (d/index-by :id))]
+                    (validate/validate-file file libs))))))
+
+(defn repair!
+  "Repair the list of errors detected by validation."
+  [system id]
+  (db/tx-run! system
+              (fn [{:keys [::db/conn] :as system}]
+                (binding [pmap/*tracked* (pmap/create-tracked)
+                          pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+                  (let [id      (if (string? id) (parse-uuid id) id)
+                        file    (files/get-file system id)
+                        libs    (->> (files/get-file-libraries conn id)
+                                     (into [file] (map (fn [{:keys [id]}]
+                                                         (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+                                                           (-> (files/get-file system id :migrate? false)
+                                                               (feat.fdata/process-pointers deref)
+                                                               (pmg/migrate-file))))))
+                                     (d/index-by :id))
+                        errors  (validate/validate-file file libs)
+                        changes (-> (repair/repair-file (:data file) libs errors) :redo-changes)
+
+                        file    (-> file
+                                    (update :revn inc)
+                                    (update :data cpc/process-changes changes)
+                                    (update :data blob/encode))]
+
+                    (when (contains? (:features file) "fdata/pointer-map")
+                      (feat.fdata/persist-pointers! system id))
+
+                    (db/update! conn :file
+                                {:revn (:revn file)
+                                 :data (:data file)
+                                 :data-backend nil
+                                 :modified-at (dt/now)
+                                 :has-media-trimmed false}
+                                {:id (:id file)})
+                    :repaired)))))
 
 (defn update-file!
   "Apply a function to the data of one file. Optionally save the changes or not.
   The function receives the decoded and migrated file data."
-  [system & {:keys [update-fn id save? migrate? inc-revn?]
-             :or {save? false migrate? true inc-revn? true}}]
-  (db/with-atomic [conn (:app.db/pool system)]
-    (let [file (-> (db/get-by-id conn :file id {::db/for-update? true})
-                   (update :features db/decode-pgarray #{}))]
-      (binding [*conn* conn
-                pmap/*tracked* (atom {})
-                pmap/*load-fn* (partial files/load-pointer conn id)
-                ffeat/*wrap-with-pointer-map-fn*
-                (if (contains? (:features file) "storage/pointer-map") pmap/wrap identity)
-                ffeat/*wrap-with-objects-map-fn*
-                (if (contains? (:features file) "storage/objectd-map") omap/wrap identity)]
-        (let [file (-> file
-                       (update :data blob/decode)
-                       (cond-> migrate? (update :data pmg/migrate-data))
-                       (update-fn)
-                       (cond-> inc-revn? (update :revn inc)))]
-          (when save?
-            (let [features (db/create-array conn "text" (:features file))
-                  data     (blob/encode (:data file))]
-              (db/update! conn :file
-                          {:data data
-                           :revn (:revn file)
-                           :features features}
-                          {:id id})
+  [system & {:keys [update-fn id rollback? migrate? inc-revn?]
+             :or {rollback? true migrate? true inc-revn? true}}]
+  (letfn [(process-file [{:keys [::db/conn] :as system} {:keys [features] :as file}]
+            (binding [pmap/*tracked* (pmap/create-tracked)
+                      pmap/*load-fn* (partial feat.fdata/load-pointer system id)
+                      cfeat/*wrap-with-pointer-map-fn*
+                      (if (contains? features "fdata/pointer-map") pmap/wrap identity)
+                      cfeat/*wrap-with-objects-map-fn*
+                      (if (contains? features "fdata/objectd-map") omap/wrap identity)]
 
-              (when (contains? (:features file) "storage/pointer-map")
-                (files/persist-pointers! conn id))))
+              (let [file     (cond-> (update-fn file)
+                               inc-revn? (update :revn inc))
+                    features (db/create-array conn "text" (:features file))
+                    data     (blob/encode (:data file))]
 
-          (dissoc file :data))))))
+                (db/update! conn :file
+                            {:data data
+                             :revn (:revn file)
+                             :features features}
+                            {:id id}))
 
-(def ^:private sql:retrieve-files-chunk
-  "SELECT id, name, created_at, data FROM file
-    WHERE created_at < ? AND deleted_at is NULL
-    ORDER BY created_at desc LIMIT ?")
+              (when (contains? (:features file) "fdata/pointer-map")
+                (feat.fdata/persist-pointers! system id))
+
+              (dissoc file :data)))]
+
+    (db/tx-run! system
+                (fn [system]
+                  (binding [*system* system]
+                    (try
+                      (->> (files/get-file system id :migrate? migrate?)
+                           (process-file system))
+                      (finally
+                        (when rollback?
+                          (db/rollback! system)))))))))
 
 (defn analyze-files
   "Apply a function to all files in the database, reading them in
   batches. Do not change data.
 
   The `on-file` parameter should be a function that receives the file
-  and the previous state and returns the new state."
-  [system & {:keys [chunk-size max-items start-at on-file on-error on-end]
+  and the previous state and returns the new state.
+
+  Emits rollback at the end of operation."
+  [system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?]
              :or {chunk-size 10 max-items Long/MAX_VALUE}}]
   (letfn [(get-chunk [conn cursor]
-            (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
-              [(some->> rows peek :created-at) (seq rows)]))
+            (let [sql  (str "SELECT id, created_at FROM file "
+                            " WHERE created_at < ? AND deleted_at is NULL "
+                            " ORDER BY created_at desc LIMIT ?")
+                  rows (db/exec! conn [sql cursor chunk-size])]
+              [(some->> rows peek :created-at) (map :id rows)]))
 
           (get-candidates [conn]
             (->> (d/iteration (partial get-chunk conn)
                               :vf second
                               :kf first
                               :initk (or start-at (dt/now)))
-                 (take max-items)
-                 (map #(update % :data blob/decode))))
+                 (take max-items)))
 
-          (on-error* [file cause]
+          (on-error* [cause file]
             (println "unexpected exception happened on processing file: " (:id file))
-            (strace/print-stack-trace cause))]
+            (strace/print-stack-trace cause))
 
-    (db/with-atomic [conn (:app.db/pool system)]
-      (loop [state {}
-             files (get-candidates conn)]
-        (if-let [file (first files)]
-          (let [state' (try
-                         (on-file file state)
-                         (catch Throwable cause
-                           (let [on-error (or on-error on-error*)]
-                             (on-error file cause))))]
-            (recur (or state' state) (rest files)))
+          (process-file [{:keys [::db/conn] :as system} file-id]
+            (let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)]
+                         (-> (files/get-file system file-id)
+                             (update :data feat.fdata/process-pointers deref)))
 
-          (if (fn? on-end)
-            (on-end state)
-            state))))))
+                  libs (when with-libraries?
+                         (->> (files/get-file-libraries conn file-id)
+                              (into [file] (map (fn [{:keys [id]}]
+                                                  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
+                                                    (-> (files/get-file system id)
+                                                        (update :data feat.fdata/process-pointers deref))))))
+                              (d/index-by :id)))]
+              (try
+                (if with-libraries?
+                  (on-file file libs)
+                  (on-file file))
+                (catch Throwable cause
+                  ((or on-error on-error*) cause file)))))]
 
-(defn update-pages
-  "Apply a function to all pages of one file. The function receives a page and returns an updated page."
-  [data f]
-  (update data :pages-index update-vals f))
+    (db/tx-run! system
+                (fn [{:keys [::db/conn] :as system}]
+                  (try
+                    (binding [*system* system]
+                      (when (fn? on-init) (on-init))
+                      (run! (partial process-file system) (get-candidates conn)))
+                    (finally
+                      (when (fn? on-end)
+                        (ex/ignoring (on-end)))
+                      (db/rollback! system)))))))
 
-(defn update-shapes
-  "Apply a function to all shapes of one page The function receives a shape and returns an updated shape"
-  [page f]
-  (update page :objects update-vals f))
+(defn process-files!
+  "Apply a function to all files in the database, reading them in
+  batches."
+
+  [system & {:keys [chunk-size
+                    max-items
+                    workers
+                    start-at
+                    on-file
+                    on-error
+                    on-end
+                    on-init
+                    rollback?]
+             :or {chunk-size 10
+                  max-items Long/MAX_VALUE
+                  workers 1
+                  rollback? true}}]
+  (letfn [(get-chunk [conn cursor]
+            (let [sql  (str "SELECT id, created_at FROM file "
+                            " WHERE created_at < ? AND deleted_at is NULL "
+                            " ORDER BY created_at desc LIMIT ?")
+                  rows (db/exec! conn [sql cursor chunk-size])]
+              [(some->> rows peek :created-at) (map :id rows)]))
+
+          (get-candidates [conn]
+            (->> (d/iteration (partial get-chunk conn)
+                              :vf second
+                              :kf first
+                              :initk (or start-at (dt/now)))
+                 (take max-items)))
+
+          (on-error* [cause file]
+            (println! "unexpected exception happened on processing file: " (:id file))
+            (strace/print-stack-trace cause))
+
+          (process-file [system file-id]
+            (try
+              (let [{:keys [features] :as file} (files/get-file system file-id)]
+                (binding [pmap/*tracked* (pmap/create-tracked)
+                          pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)
+                          cfeat/*wrap-with-pointer-map-fn*
+                          (if (contains? features "fdata/pointer-map") pmap/wrap identity)
+                          cfeat/*wrap-with-objects-map-fn*
+                          (if (contains? features "fdata/objectd-map") omap/wrap identity)]
+
+                  (on-file file)
+
+                  (when (contains? features "fdata/pointer-map")
+                    (feat.fdata/persist-pointers! system file-id))))
+
+              (catch Throwable cause
+                ((or on-error on-error*) cause file-id))))
+
+          (run-worker [in index]
+            (db/tx-run! system
+                        (fn [system]
+                          (binding [*system* system]
+                            (loop [i 0]
+                              (when-let [file-id (sp/take! in)]
+                                (println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name))
+                                (process-file system file-id)
+                                (recur (inc i)))))
+
+                          (when rollback?
+                            (db/rollback! system)))))
+
+          (run-producer [input]
+            (db/tx-run! system (fn [{:keys [::db/conn]}]
+                                 (doseq [file-id (get-candidates conn)]
+                                   (println! "=> producer:" file-id "|" (px/get-name))
+                                   (sp/put! input file-id))
+                                 (sp/close! input))))]
+
+    (when (fn? on-init) (on-init))
+
+    (let [input    (sp/chan :buf chunk-size)
+          producer (px/thread
+                     {:name "penpot/srepl/producer"}
+                     (run-producer input))
+          threads  (->> (range workers)
+                        (map (fn [index]
+                               (px/thread
+                                 {:name (str "penpot/srepl/worker/" index)}
+                                 (run-worker input index))))
+                        (cons producer)
+                        (doall))]
+
+      (run! p/await! threads)
+      (when (fn? on-end) (on-end)))))

backend/src/app/srepl/main.clj

@@ -8,25 +8,32 @@
   "A collection of adhoc fixes scripts."
   #_:clj-kondo/ignore
   (:require
+   [app.auth :refer [derive-password]]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as p]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.features.fdata :as features.fdata]
+   [app.main :as main]
    [app.msgbus :as mbus]
    [app.rpc.commands.auth :as auth]
+   [app.rpc.commands.files-snapshot :as fsnap]
    [app.rpc.commands.profile :as profile]
-   [app.srepl.fixes :as f]
+   [app.srepl.cli :as cli]
    [app.srepl.helpers :as h]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as wrk]
-   [clojure.pprint :refer [pprint]]
+   [clojure.pprint :refer [pprint print-table]]
+   [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]))
 
 (defn print-available-tasks
@@ -106,44 +113,71 @@
         (db/delete! conn :http-session {:profile-id (:id profile)})
         :blocked))))
 
+(defn reset-password!
+  "Reset a password to a specific one for a concrete user or all users
+  if email is `:all` keyword."
+  [system & {:keys [email password] :or {password "123123"} :as params}]
+  (us/verify! (contains? params :email) "`email` parameter is mandatory")
+  (db/with-atomic [conn (:app.db/pool system)]
+    (let [password (derive-password password)]
+      (if (= email :all)
+        (db/exec! conn ["update profile set password=?" password])
+        (let [email (str/lower email)]
+          (db/exec! conn ["update profile set password=? where email=?" password email]))))))
 
 (defn enable-objects-map-feature-on-file!
   [system & {:keys [save? id]}]
-  (letfn [(update-file [{:keys [features] :as file}]
-            (if (contains? features "storage/objects-map")
-              file
-              (-> file
-                  (update :data migrate)
-                  (update :features conj "storage/objects-map"))))
-
-          (migrate [data]
-            (-> data
-                (update :pages-index update-vals #(update % :objects omap/wrap))
-                (update :components update-vals #(update % :objects omap/wrap))))]
-
-    (h/update-file! system
-                    :id id
-                    :update-fn update-file
-                    :save? save?)))
+  (h/update-file! system
+                  :id id
+                  :update-fn features.fdata/enable-objects-map
+                  :save? save?))
 
 (defn enable-pointer-map-feature-on-file!
   [system & {:keys [save? id]}]
-  (letfn [(update-file [{:keys [features] :as file}]
-            (if (contains? features "storage/pointer-map")
-              file
-              (-> file
-                  (update :data migrate)
-                  (update :features conj "storage/pointer-map"))))
+  (h/update-file! system
+                  :id id
+                  :update-fn features.fdata/enable-pointer-map
+                  :save? save?))
 
-          (migrate [data]
-            (-> data
-                (update :pages-index update-vals pmap/wrap)
-                (update :components pmap/wrap)))]
+(defn enable-team-feature!
+  [system team-id feature]
+  (dm/verify!
+   "feature should be supported"
+   (contains? cfeat/supported-features feature))
 
-    (h/update-file! system
-                    :id id
-                    :update-fn update-file
-                    :save? save?)))
+  (let [team-id (if (string? team-id)
+                  (parse-uuid team-id)
+                  team-id)]
+    (db/tx-run! system
+                (fn [{:keys [::db/conn]}]
+                  (let [team     (-> (db/get conn :team {:id team-id})
+                                     (update :features db/decode-pgarray #{}))
+                        features (conj (:features team) feature)]
+                    (when (not= features (:features team))
+                      (db/update! conn :team
+                                  {:features (db/create-array conn "text" features)}
+                                  {:id team-id})
+                      :enabled))))))
+
+(defn disable-team-feature!
+  [system team-id feature]
+  (dm/verify!
+   "feature should be supported"
+   (contains? cfeat/supported-features feature))
+
+  (let [team-id (if (string? team-id)
+                  (parse-uuid team-id)
+                  team-id)]
+    (db/tx-run! system
+                (fn [{:keys [::db/conn]}]
+                  (let [team     (-> (db/get conn :team {:id team-id})
+                                     (update :features db/decode-pgarray #{}))
+                        features (disj (:features team) feature)]
+                    (when (not= features (:features team))
+                      (db/update! conn :team
+                                  {:features (db/create-array conn "text" features)}
+                                  {:id team-id})
+                      :disabled))))))
 
 (defn enable-storage-features-on-file!
   [system & {:as params}]
@@ -169,6 +203,33 @@
   (alter-var-root var (fn [f]
                         (or (::original (meta f)) f))))
 
+(defn take-file-snapshot!
+  "An internal helper that persist the file snapshot using non-gc
+  collectable file-changes entry."
+  [system & {:keys [file-id label]}]
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! system fsnap/take-file-snapshot! {:file-id file-id :label label})))
+
+(defn restore-file-snapshot!
+  [system & {:keys [file-id id]}]
+  (db/tx-run! system
+              (fn [cfg]
+                (let [file-id (h/parse-uuid file-id)
+                      id      (h/parse-uuid id)]
+
+                  (if (and (uuid? id) (uuid? file-id))
+                    (fsnap/restore-file-snapshot! cfg {:id id :file-id file-id})
+                    (println "=> invalid parameters"))))))
+
+
+(defn list-file-snapshots!
+  [system & {:keys [file-id limit]}]
+  (db/tx-run! system (fn [system]
+                       (let [params {:file-id (h/parse-uuid file-id)
+                                     :limit limit}]
+                         (->> (fsnap/get-file-snapshots system (d/without-nils params))
+                              (print-table [:id :revn :created-at :label]))))))
+
 (defn notify!
   [{:keys [::mbus/msgbus ::db/pool]} & {:keys [dest code message level]
                                         :or {code :generic level :info}
@@ -265,8 +326,7 @@
                   (= op :profile-id)
                   (if (coll? param)
                     (sequence (keep parse-uuid) param)
-                    (resolve-dest param))))))
-          ]
+                    (resolve-dest param))))))]
 
     (->> (resolve-dest dest)
          (filter some?)

backend/src/app/storage.clj

@@ -18,7 +18,6 @@
    [app.storage.impl :as impl]
    [app.storage.s3 :as ss3]
    [app.util.time :as dt]
-   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
@@ -40,7 +39,7 @@
                    :fs ::sfs/backend))))
 
 (defmethod ig/pre-init-spec ::storage [_]
-  (s/keys :req [::db/pool ::wrk/executor ::backends]))
+  (s/keys :req [::db/pool ::backends]))
 
 (defmethod ig/init-key ::storage
   [_ {:keys [::backends ::db/pool] :as cfg}]
@@ -251,53 +250,59 @@
 
 (defmethod ig/init-key ::gc-deleted-task
   [_ {:keys [::db/pool ::storage ::min-age]}]
-  (letfn [(retrieve-deleted-objects-chunk [conn min-age cursor]
-            (let [min-age (db/interval min-age)
-                  rows    (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
-              [(some-> rows peek :created-at)
+  (letfn [(get-to-delete-chunk [cursor]
+            (let [sql  (str "select s.* "
+                            "  from storage_object as s "
+                            " where s.deleted_at is not null "
+                            "   and s.deleted_at < ? "
+                            " order by s.deleted_at desc "
+                            " limit 25")
+                  rows (db/exec! pool [sql cursor])]
+              [(some-> rows peek :deleted-at)
                (some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
 
-          (retrieve-deleted-objects [conn min-age]
-            (d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
-                         :initk (dt/now)
+          (get-to-delete-chunks [min-age]
+            (d/iteration get-to-delete-chunk
+                         :initk (dt/minus (dt/now) min-age)
                          :vf second
                          :kf first))
 
-          (delete-in-bulk [backend-id ids]
-            (let [backend (impl/resolve-backend storage backend-id)]
+          (delete-in-bulk! [backend-id ids]
+            (try
+              (db/with-atomic [conn pool]
+                (let [sql   "delete from storage_object where id = ANY(?)"
+                      ids'  (db/create-array conn "uuid" ids)
 
-              (doseq [id ids]
-                (l/debug :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
+                      total (-> (db/exec-one! conn [sql ids'])
+                                (db/get-update-count))]
 
-              (impl/del-objects-in-bulk backend ids)))]
+                  (-> (impl/resolve-backend storage backend-id)
+                      (impl/del-objects-in-bulk ids))
+
+                  (doseq [id ids]
+                    (l/dbg :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
+
+                  total))
+
+              (catch Throwable cause
+                (l/err :hint "gc-deleted: unexpected error on bulk deletion"
+                       :ids (vec ids)
+                       :cause cause)
+                0)))]
 
     (fn [params]
-      (let [min-age (or (:min-age params) min-age)]
-        (db/with-atomic [conn pool]
-          (loop [total  0
-                 groups (retrieve-deleted-objects conn min-age)]
-            (if-let [[backend-id ids] (first groups)]
-              (do
-                (delete-in-bulk backend-id ids)
-                (recur (+ total (count ids))
-                       (rest groups)))
-              (do
-                (l/info :hint "gc-deleted: task finished" :min-age (dt/format-duration min-age) :total total)
-                {:deleted total}))))))))
-
-(def sql:retrieve-deleted-objects-chunk
-  "with items_part as (
-     select s.id
-       from storage_object as s
-      where s.deleted_at is not null
-        and s.deleted_at < (now() - ?::interval)
-        and s.created_at < ?
-      order by s.created_at desc
-      limit 25
-   )
-   delete from storage_object
-    where id in (select id from items_part)
-   returning *;")
+      (let [min-age (or (some-> params :min-age dt/duration) min-age)]
+        (loop [total  0
+               chunks (get-to-delete-chunks min-age)]
+          (if-let [[backend-id ids] (first chunks)]
+            (let [deleted (delete-in-bulk! backend-id ids)]
+              (recur (+ total deleted)
+                     (rest chunks)))
+            (do
+              (l/inf :hint "gc-deleted: task finished"
+                     :min-age (dt/format-duration min-age)
+                     :total total)
+              {:deleted total})))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Garbage Collection: Analyze touched objects
@@ -316,10 +321,11 @@
 ;; and the object is still valid) or deleted (no more references to
 ;; this object so is ready to be deleted).
 
-(declare sql:retrieve-touched-objects-chunk)
 (declare sql:retrieve-file-media-object-nrefs)
-(declare sql:retrieve-team-font-variant-nrefs)
+(declare sql:retrieve-file-object-thumbnail-nrefs)
 (declare sql:retrieve-profile-nrefs)
+(declare sql:retrieve-team-font-variant-nrefs)
+(declare sql:retrieve-touched-objects-chunk)
 
 (defmethod ig/pre-init-spec ::gc-touched-task [_]
   (s/keys :req [::db/pool]))
@@ -335,6 +341,9 @@
           (get-profile-nrefs [conn id]
             (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs))
 
+          (get-file-object-thumbnails [conn id]
+            (-> (db/exec-one! conn [sql:retrieve-file-object-thumbnail-nrefs id]) :nrefs))
+
           (mark-freeze-in-bulk [conn ids]
             (db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)"
                                 (db/create-array conn "uuid" ids)]))
@@ -394,8 +403,7 @@
                 (do
                   (some->> (seq to-freeze) (mark-freeze-in-bulk conn))
                   (some->> (seq to-delete) (mark-delete-in-bulk conn))
-                  [(count to-freeze) (count to-delete)]))))
-          ]
+                  [(count to-freeze) (count to-delete)]))))]
 
     (fn [_]
       (db/with-atomic [conn pool]
@@ -404,9 +412,10 @@
                groups    (retrieve-touched conn)]
           (if-let [[bucket ids] (first groups)]
             (let [[f d] (case bucket
-                          "file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket)
-                          "team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket)
-                          "profile"           (process-objects! conn get-profile-nrefs ids bucket)
+                          "file-media-object"     (process-objects! conn get-file-media-object-nrefs ids bucket)
+                          "team-font-variant"     (process-objects! conn get-team-font-variant-nrefs ids bucket)
+                          "file-object-thumbnail" (process-objects! conn get-file-object-thumbnails ids bucket)
+                          "profile"               (process-objects! conn get-profile-nrefs ids bucket)
                           (ex/raise :type :internal
                                     :code :unexpected-unknown-reference
                                     :hint (dm/fmt "unknown reference %" bucket)))]
@@ -429,6 +438,9 @@
   "select ((select count(*) from file_media_object where media_id = ?) +
            (select count(*) from file_media_object where thumbnail_id = ?)) as nrefs")
 
+(def sql:retrieve-file-object-thumbnail-nrefs
+  "select (select count(*) from file_tagged_object_thumbnail where media_id = ?) as nrefs")
+
 (def sql:retrieve-team-font-variant-nrefs
   "select ((select count(*) from team_font_variant where woff1_file_id = ?) +
            (select count(*) from team_font_variant where woff2_file_id = ?) +

backend/src/app/storage/fs.clj

@@ -18,8 +18,8 @@
    [datoteka.io :as io]
    [integrant.core :as ig])
   (:import
-   java.nio.file.Path
-   java.nio.file.Files))
+   java.nio.file.Files
+   java.nio.file.Path))
 
 ;; --- BACKEND INIT
 

backend/src/app/storage/impl.clj

@@ -11,7 +11,6 @@
    [app.common.exceptions :as ex]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
-   [app.worker :as-alias wrk]
    [buddy.core.codecs :as bc]
    [buddy.core.hash :as bh]
    [clojure.java.io :as jio]
@@ -201,7 +200,7 @@
     (str "blake2b:" result)))
 
 (defn resolve-backend
-  [{:keys [::db/pool ::wrk/executor] :as storage} backend-id]
+  [{:keys [::db/pool] :as storage} backend-id]
   (let [backend (get-in storage [::sto/backends backend-id])]
     (when-not backend
       (ex/raise :type :internal
@@ -209,7 +208,6 @@
                 :hint (dm/fmt "backend '%' not configured" backend-id)))
     (-> backend
         (assoc ::sto/id backend-id)
-        (assoc ::wrk/executor executor)
         (assoc ::db/pool pool))))
 
 (defrecord StorageObject [id size created-at expired-at touched-at backend])

backend/src/app/storage/s3.clj

@@ -17,7 +17,6 @@
    [app.storage.impl :as impl]
    [app.storage.tmp :as tmp]
    [app.util.time :as dt]
-   [app.worker :as wrk]
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [datoteka.fs :as fs]
@@ -40,7 +39,6 @@
    software.amazon.awssdk.core.async.AsyncRequestBody
    software.amazon.awssdk.core.async.AsyncResponseTransformer
    software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
-   software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
    software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
    software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
    software.amazon.awssdk.regions.Region
@@ -77,9 +75,10 @@
 (s/def ::bucket ::us/string)
 (s/def ::prefix ::us/string)
 (s/def ::endpoint ::us/string)
+(s/def ::io-threads ::us/integer)
 
 (defmethod ig/pre-init-spec ::backend [_]
-  (s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor]))
+  (s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads]))
 
 (defmethod ig/prep-key ::backend
   [_ {:keys [::prefix ::region] :as cfg}]
@@ -114,8 +113,7 @@
                 ::client
                 ::presigner]
           :opt [::prefix
-                ::sto/id
-                ::wrk/executor]))
+                ::sto/id]))
 
 ;; --- API IMPL
 
@@ -161,7 +159,6 @@
 
 ;; --- HELPERS
 
-(def default-eventloop-threads 4)
 (def default-timeout
   (dt/duration {:seconds 30}))
 
@@ -171,18 +168,18 @@
   (Region/of (name region)))
 
 (defn- build-s3-client
-  [{:keys [::region ::endpoint ::wrk/executor]}]
+  [{:keys [::region ::endpoint ::io-threads]}]
   (let [aconfig (-> (ClientAsyncConfiguration/builder)
-                    (.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
                     (.build))
 
         sconfig (-> (S3Configuration/builder)
                     (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
                     (.build))
 
+        thr-num (or io-threads (min 16 (px/get-available-processors)))
         hclient (-> (NettyNioAsyncHttpClient/builder)
                     (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
-                                                (.numberOfThreads (int default-eventloop-threads))))
+                                                (.numberOfThreads (int thr-num))))
                     (.connectionAcquisitionTimeout default-timeout)
                     (.connectionTimeout default-timeout)
                     (.readTimeout default-timeout)
@@ -197,9 +194,7 @@
                       builder (cond-> ^S3AsyncClientBuilder builder
                                 (some? endpoint)
                                 (.endpointOverride (URI. endpoint)))]
-                  (.build ^S3AsyncClientBuilder builder))
-
-        ]
+                  (.build ^S3AsyncClientBuilder builder))]
 
     (reify
       clojure.lang.IDeref
@@ -226,6 +221,7 @@
   [id subscriber sem content]
   (px/thread
     {:name "penpot/s3/uploader"
+     :virtual true
      :daemon true}
     (l/trace :hint "start upload thread"
              :object-id (str id)
@@ -267,15 +263,15 @@
       (Optional/of (long (impl/get-size content))))
 
     (^void subscribe [_ ^Subscriber subscriber]
-     (let [sem (Semaphore. 0)
-           thr (upload-thread id subscriber sem content)]
-       (.onSubscribe subscriber
-                     (reify Subscription
-                       (cancel [_]
-                         (px/interrupt! thr)
-                         (.release sem 1))
-                       (request [_ n]
-                         (.release sem (int n)))))))))
+      (let [sem (Semaphore. 0)
+            thr (upload-thread id subscriber sem content)]
+        (.onSubscribe subscriber
+                      (reify Subscription
+                        (cancel [_]
+                          (px/interrupt! thr)
+                          (.release sem 1))
+                        (request [_ n]
+                          (.release sem (int n)))))))))
 
 
 (defn- put-object
@@ -313,7 +309,7 @@
     ;; to the filesystem and then read with buffered inputstream; if
     ;; not, read the contento into memory using bytearrays.
     (if (> ^long size (* 1024 1024 2))
-      (let [path (tmp/tempfile :prefix "penpot.storage.s3.")
+      (let [path (tmp/tempfile :prefix "penpot.storage.s3." :min-age "6h")
             rxf  (AsyncResponseTransformer/toFile ^Path path)]
         (->> (.getObject ^S3AsyncClient client
                          ^GetObjectRequest gor

backend/src/app/storage/tmp.clj

@@ -29,7 +29,7 @@
 
 (defmethod ig/prep-key ::cleaner
   [_ cfg]
-  (assoc cfg ::min-age (dt/duration "30m")))
+  (assoc cfg ::min-age (dt/duration "60m")))
 
 (defmethod ig/init-key ::cleaner
   [_ cfg]
@@ -42,14 +42,15 @@
 
 (defn- io-loop
   [{:keys [::min-age] :as cfg}]
-  (l/info :hint "started tmp file cleaner")
+  (l/inf :hint "started tmp cleaner" :default-min-age (dt/format-duration min-age))
   (try
     (loop []
-      (when-let [path (sp/take! queue)]
-        (l/debug :hint "schedule tempfile deletion" :path path
+      (when-let [[path min-age'] (sp/take! queue)]
+        (let [min-age (or min-age' min-age)]
+          (l/dbg :hint "schedule tempfile deletion" :path path
                  :expires-at (dt/plus (dt/now) min-age))
-        (px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
-        (recur)))
+          (px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
+          (recur))))
     (catch InterruptedException _
       (l/trace :hint "cleaner interrupted"))
     (finally
@@ -57,11 +58,11 @@
 
 (defn- remove-temp-file
   "Permanently delete tempfile"
-  [{:keys [::wrk/executor path]}]
+  [{:keys [::wrk/executor]} path]
   (when (fs/exists? path)
     (px/run! executor
              (fn []
-               (l/debug :hint "permanently delete tempfile" :path path)
+               (l/dbg :hint "permanently delete tempfile" :path path)
                (fs/delete path)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -70,17 +71,17 @@
 
 (defn tempfile
   "Returns a tmpfile candidate (without creating it)"
-  [& {:keys [suffix prefix]
+  [& {:keys [suffix prefix min-age]
       :or {prefix "penpot."
            suffix ".tmp"}}]
-  (let [candidate (fs/tempfile :suffix suffix :prefix prefix)]
-    (sp/offer! queue candidate)
-    candidate))
+  (let [path (fs/tempfile :suffix suffix :prefix prefix)]
+    (sp/offer! queue [path (some-> min-age dt/duration)])
+    path))
 
 (defn create-tempfile
-  [& {:keys [suffix prefix]
+  [& {:keys [suffix prefix min-age]
       :or {prefix "penpot."
            suffix ".tmp"}}]
   (let [path (fs/create-tempfile :suffix suffix :prefix prefix)]
-    (sp/offer! queue path)
+    (sp/offer! queue [path (some-> min-age dt/duration)])
     path))

backend/src/app/tasks/file_gc.clj

@@ -11,15 +11,16 @@
   inactivity (the default threshold is 72h)."
   (:require
    [app.common.data :as d]
+   [app.common.files.migrations :as pmg]
    [app.common.logging :as l]
-   [app.common.pages.migrations :as pmg]
+   [app.common.thumbnails :as thc]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
    [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
+   [app.features.fdata :as feat.fdata]
    [app.media :as media]
-   [app.rpc.commands.files :as files]
    [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
@@ -111,18 +112,21 @@
   (let [xform (comp
                (map :objects)
                (mapcat vals)
-               (keep (fn [{:keys [type] :as obj}]
-                       (case type
-                         :path  (get-in obj [:fill-image :id])
-                         :bool  (get-in obj [:fill-image :id])
+               (mapcat (fn [obj]
                          ;; NOTE: because of some bug, we ended with
                          ;; many shape types having the ability to
                          ;; have fill-image attribute (which initially
                          ;; designed for :path shapes).
-                         :group (get-in obj [:fill-image :id])
-                         :image (get-in obj [:metadata :id])
-
-                         nil))))
+                         (sequence
+                          (keep :id)
+                          (concat [(:fill-image obj)
+                                   (:metadata obj)]
+                                  (map :fill-image (:fills obj))
+                                  (map :stroke-image (:strokes obj))
+                                  (->> (:content obj)
+                                       (tree-seq map? :children)
+                                       (mapcat :fills)
+                                       (map :fill-image)))))))
         pages (concat
                (vals (:pages-index data))
                (vals (:components data)))]
@@ -138,10 +142,10 @@
                     (remove #(contains? used (:id %))))]
 
     (doseq [mobj unused]
-      (l/debug :hint "delete file media object"
-               :id (:id mobj)
-               :media-id (:media-id mobj)
-               :thumbnail-id (:thumbnail-id mobj))
+      (l/dbg :hint "delete file media object"
+             :id (:id mobj)
+             :media-id (:media-id mobj)
+             :thumbnail-id (:thumbnail-id mobj))
 
       ;; NOTE: deleting the file-media-object in the database
       ;; automatically marks as touched the referenced storage
@@ -152,34 +156,40 @@
 
 (defn- clean-file-object-thumbnails!
   [{:keys [::db/conn ::sto/storage]} file-id data]
-  (let [stored (->> (db/query conn :file-object-thumbnail
+  (let [stored (->> (db/query conn :file-tagged-object-thumbnail
                               {:file-id file-id}
                               {:columns [:object-id]})
                     (into #{} (map :object-id)))
 
         using  (into #{}
-                     (mapcat (fn [{:keys [id objects]}]
-                               (->> (ctt/get-frames objects)
-                                    (map #(str id (:id %))))))
+                     (comp
+                      (mapcat (fn [{:keys [id objects]}]
+                                (->> (ctt/get-frames objects)
+                                     (map #(assoc % :page-id id)))))
+                      (mapcat (fn [{:keys [id page-id]}]
+                                (list
+                                 (thc/fmt-object-id file-id page-id id "frame")
+                                 (thc/fmt-object-id file-id page-id id "component")))))
+
                      (vals (:pages-index data)))
 
         unused (set/difference stored using)]
 
     (when (seq unused)
-      (let [sql (str "delete from file_object_thumbnail "
+      (let [sql (str "delete from file_tagged_object_thumbnail "
                      " where file_id=? and object_id=ANY(?)"
                      " returning media_id")
             res (db/exec! conn [sql file-id (db/create-array conn "text" unused)])]
 
+        (l/dbg :hint "delete file object thumbnails"
+               :file-id (str file-id)
+               :total (count res))
+
         (doseq [media-id (into #{} (keep :media-id) res)]
           ;; Mark as deleted the storage object related with the
           ;; photo-id field.
-          (l/trace :hint "mark storage object as deleted" :id media-id)
-          (sto/del-object! storage media-id))
-
-        (l/debug :hint "delete file object thumbnails"
-                 :file-id file-id
-                 :total (count res))))))
+          (l/trc :hint "touch file object thumbnail storage object" :id (str media-id))
+          (sto/touch-object! storage media-id))))))
 
 (defn- clean-file-thumbnails!
   [{:keys [::db/conn ::sto/storage]} file-id revn]
@@ -189,15 +199,15 @@
         res (db/exec! conn [sql file-id revn])]
 
     (when (seq res)
+      (l/dbg :hint "delete file thumbnails"
+             :file-id (str file-id)
+             :total (count res))
+
       (doseq [media-id (into #{} (keep :media-id) res)]
         ;; Mark as deleted the storage object related with the
         ;; media-id field.
-        (l/trace :hint "mark storage object as deleted" :id media-id)
-        (sto/del-object! storage media-id))
-
-      (l/debug :hint "delete file thumbnails"
-               :file-id file-id
-               :total (count res)))))
+        (l/trc :hint "delete file thumbnail storage object" :id (str media-id))
+        (sto/del-object! storage media-id)))))
 
 (def ^:private
   sql:get-files-for-library
@@ -242,7 +252,7 @@
                        (mapv :id))]
 
       (when (seq unused)
-        (l/debug :hint "clean deleted components" :total (count unused))
+        (l/dbg :hint "clean deleted components" :total (count unused))
 
         (let [data (reduce ctkl/delete-component data unused)]
           (db/update! conn :file
@@ -261,9 +271,9 @@
                             " limit 1;")
                   rows (db/exec! conn [sql file-id cursor])]
               [(some-> rows peek :created-at)
-               (mapcat (comp files/get-all-pointer-ids blob/decode :data) rows)]))]
+               (mapcat (comp feat.fdata/get-used-pointer-ids blob/decode :data) rows)]))]
 
-    (let [used (into (files/get-all-pointer-ids data)
+    (let [used (into (feat.fdata/get-used-pointer-ids data)
                      (d/iteration get-pointers-chunk
                                   :vf second
                                   :kf first
@@ -275,15 +285,15 @@
           rows (db/exec! conn [sql file-id used])]
 
       (doseq [fragment-id (map :id rows)]
-        (l/trace :hint "remove unused file data fragment" :id (str fragment-id))
+        (l/trc :hint "remove unused file data fragment" :id (str fragment-id))
         (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id})))))
 
 (defn- process-file
   [{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}]
-  (l/debug :hint "processing file" :id id :modified-at modified-at)
+  (l/dbg :hint "processing file" :file-id (str id) :modified-at modified-at)
 
-  (binding [pmap/*load-fn* (partial files/load-pointer conn id)
-            pmap/*tracked* (atom {})]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+            pmap/*tracked* (pmap/create-tracked)]
     (let [data (-> (blob/decode data)
                    (assoc :id id)
                    (pmg/migrate-data))]
@@ -293,7 +303,7 @@
       (clean-file-thumbnails! cfg id revn)
       (clean-deleted-components! conn id data)
 
-      (when (contains? features "storage/pointer-map")
+      (when (contains? features "fdata/pointer-map")
         (clean-data-fragments! conn id data))
 
       ;; Mark file as trimmed
@@ -301,4 +311,4 @@
                   {:has-media-trimmed true}
                   {:id id})
 
-      (files/persist-pointers! conn id))))
+      (feat.fdata/persist-pointers! cfg id))))

backend/src/app/tasks/file_xlog_gc.clj

@@ -17,7 +17,8 @@
 (def ^:private
   sql:delete-files-xlog
   "delete from file_change
-    where created_at < now() - ?::interval")
+    where created_at < now() - ?::interval
+      and label is NULL")
 
 (defmethod ig/pre-init-spec ::handler [_]
   (s/keys :req [::db/pool]))

backend/src/app/tasks/objects_gc.clj

@@ -151,7 +151,7 @@
                                      {::db/return-keys? false})
                   count  (db/get-update-count result)]
               (when (pos? count)
-                (l/debug :hint "mark team for deletion" :id (str id) ))
+                (l/debug :hint "mark team for deletion" :id (str id)))
 
               (+ total count)))]
 

backend/src/app/tasks/telemetry.clj

@@ -178,7 +178,7 @@
 (defn- retrieve-enabled-auth-providers
   [conn]
   (let [sql  (str "select auth_backend as backend, count(*) as total "
-                 "  from profile group by 1")
+                  "  from profile group by 1")
         rows (db/exec! conn [sql])]
     (->> rows
          (map (fn [{:keys [backend total]}]

backend/src/app/util/async.clj

@@ -1,113 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.util.async
-  (:require
-   [app.common.exceptions :as ex]
-   [clojure.core.async :as a]
-   [clojure.core.async.impl.protocols :as ap]
-   [clojure.spec.alpha :as s])
-  (:import
-   java.util.concurrent.Executor
-   java.util.concurrent.RejectedExecutionException))
-
-(s/def ::executor #(instance? Executor %))
-(s/def ::channel #(satisfies? ap/Channel %))
-
-(defonce processors
-  (delay (.availableProcessors (Runtime/getRuntime))))
-
-(defmacro go-try
-  [& body]
-  `(a/go
-     (try
-       ~@body
-       (catch Exception e# e#))))
-
-(defmacro thread
-  [& body]
-  `(a/thread
-     (try
-       ~@body
-       (catch Exception e#
-         e#))))
-
-(defmacro <?
-  [ch]
-  `(let [r# (a/<! ~ch)]
-     (if (instance? Exception r#)
-       (throw r#)
-       r#)))
-
-(defmacro with-closing
-  [ch & body]
-  `(try
-     ~@body
-     (finally
-       (some-> ~ch a/close!))))
-
-(defn thread-call
-  [^Executor executor f]
-  (let [ch (a/chan 1)
-        f' (fn []
-             (try
-               (let [ret (ex/try* f identity)]
-                 (when (some? ret) (a/>!! ch ret)))
-               (finally
-                 (a/close! ch))))]
-    (try
-      (.execute executor f')
-      (catch RejectedExecutionException _cause
-        (a/close! ch)))
-
-    ch))
-
-(defmacro with-thread
-  [executor & body]
-  (if (= executor ::default)
-    `(a/thread-call (^:once fn* [] (try ~@body (catch Exception e# e#))))
-    `(thread-call ~executor (^:once fn* [] ~@body))))
-
-(defn batch
-  [in {:keys [max-batch-size
-              max-batch-age
-              buffer-size
-              init]
-       :or {max-batch-size 200
-            max-batch-age (* 30 1000)
-            buffer-size 128
-            init #{}}
-       :as opts}]
-  (let [out (a/chan buffer-size)]
-    (a/go-loop [tch (a/timeout max-batch-age) buf init]
-      (let [[val port] (a/alts! [tch in])]
-        (cond
-          (identical? port tch)
-          (if (empty? buf)
-            (recur (a/timeout max-batch-age) buf)
-            (do
-              (a/>! out [:timeout buf])
-              (recur (a/timeout max-batch-age) init)))
-
-          (nil? val)
-          (if (empty? buf)
-            (a/close! out)
-            (do
-              (a/offer! out [:timeout buf])
-              (a/close! out)))
-
-          (identical? port in)
-          (let [buf (conj buf val)]
-            (if (>= (count buf) max-batch-size)
-              (do
-                (a/>! out [:size buf])
-                (recur (a/timeout max-batch-age) init))
-              (recur tch buf))))))
-    out))
-
-(defn thread-sleep
-  [ms]
-  (Thread/sleep (long ms)))

backend/src/app/util/locks.clj

@@ -8,8 +8,8 @@
   "A syntactic helpers for using locks."
   (:refer-clojure :exclude [locking])
   (:import
-   java.util.concurrent.locks.ReentrantLock
-   java.util.concurrent.locks.Lock))
+   java.util.concurrent.locks.Lock
+   java.util.concurrent.locks.ReentrantLock))
 
 (defn create
   []

backend/src/app/util/objects_map.clj

@@ -335,8 +335,7 @@
   Iterable
   (iterator [this]
     (when-not loaded? (load! this))
-    (ObjectsMapIterator. (.iterator ^Iterable positions) this))
-  )
+    (ObjectsMapIterator. (.iterator ^Iterable positions) this)))
 
 (defn create
   ([]

backend/src/app/util/pointer_map.clj

@@ -60,6 +60,10 @@
 
 (declare create)
 
+(defn create-tracked
+  []
+  (atom {}))
+
 (defprotocol IPointerMap
   (get-id [_])
   (load! [_])
@@ -73,7 +77,7 @@
 
   IPointerMap
   (load! [_]
-    (l/trace :hint "pointer-map:load" :id id)
+    (l/trace :hint "pointer-map:load" :id (str id))
 
     (when-not *load-fn*
       (throw (UnsupportedOperationException. "load is not supported when *load-fn* is not bind")))

backend/src/app/util/svg.clj

@@ -1,51 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.util.svg
-  (:require
-   [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex]
-   [app.common.logging :as l]
-   [clojure.xml :as xml]
-   [cuerdas.core :as str])
-  (:import
-   javax.xml.XMLConstants
-   java.io.InputStream
-   javax.xml.parsers.SAXParserFactory
-   clojure.lang.XMLHandler
-   org.apache.commons.io.IOUtils))
-
-(defn- secure-parser-factory
-  [^InputStream input ^XMLHandler handler]
-  (.. (doto (SAXParserFactory/newInstance)
-        (.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
-        (.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
-      (newSAXParser)
-      (parse input handler)))
-
-(defn parse
-  [^String data]
-  (try
-    (dm/with-open [istream (IOUtils/toInputStream data "UTF-8")]
-      (xml/parse istream secure-parser-factory))
-    (catch Exception e
-      (l/warn :hint "error on processing svg"
-              :message (ex-message e))
-      (ex/raise :type :validation
-                :code :invalid-svg-file
-                :hint "invalid svg file"
-                :cause e))))
-
-
-;; --- PROCESSORS
-
-(defn strip-doctype
-  [data]
-  (cond-> data
-    (str/includes? data "<!DOCTYPE")
-    (str/replace #"<\!DOCTYPE[^>]*>" "")))
-
-(def pre-process strip-doctype)