wip: Add rxjs-spy library to monitor RX streams

.circleci/config.yml

@@ -11,11 +11,11 @@ jobs:
       - image: cimg/redis:7.0.5
 
     working_directory: ~/repo
-    resource_class: medium+
+    resource_class: large
 
     environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx4g
 
     steps:
       - checkout
@@ -28,59 +28,50 @@ jobs:
            - v1-dependencies-
 
       - run: cd .clj-kondo && cat config.edn
-      - run: cat .cljfmt.edn
-      - run: clj-kondo --version
 
-      # - run:
-      #     name: "fmt check [clj]"
-      #     command: |
-      #       yarn run fmt:clj:check
+      - run:
+          name: frontend styles prettier
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run lint:scss
 
       - run:
           name: common lint
           working_directory: "./common"
           command: |
-            yarn install
-            yarn run lint:clj
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
           name: frontend lint
           working_directory: "./frontend"
           command: |
-            yarn install
-            yarn run lint:scss
-            yarn run lint:clj
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
           name: backend lint
           working_directory: "./backend"
           command: |
-            yarn install
-            yarn run lint:clj
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
-          name: exporter lint
-          working_directory: "./exporter"
-          command: |
-            yarn install
-            yarn run lint:clj
-
-      - run:
-          name: "common tests"
           working_directory: "./common"
+          name: common tests
           command: |
+            yarn install
             yarn test
             clojure -X:dev:test :patterns '["common-tests.*-test"]'
 
-      - run:
-          name: "frontend tests"
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn test
+          environment:
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+            JVM_OPTS: -Xmx4g
+            NODE_OPTIONS: --max-old-space-size=4096
 
       - run:
-          name: "backend tests"
+          name: backend test
           working_directory: "./backend"
           command: |
             clojure -X:dev:test :patterns '["backend-tests.*-test"]'
@@ -90,6 +81,18 @@ jobs:
             PENPOT_TEST_DATABASE_USERNAME: penpot_test
             PENPOT_TEST_DATABASE_PASSWORD: penpot_test
             PENPOT_TEST_REDIS_URI: "redis://localhost/1"
+            JVM_OPTS: -Xmx4g
+
+      - run:
+          name: frontend tests
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn test
+
+          environment:
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+            NODE_OPTIONS: --max-old-space-size=4096
 
       - save_cache:
          paths:

.clj-kondo/config.edn

@@ -5,7 +5,6 @@
   promesa.exec.csp/go-loop clojure.core/loop
   rumext.v2/defc clojure.core/defn
   rumext.v2/fnc clojure.core/fn
-  promesa.util/with-open clojure.core/with-open
   app.common.data/export clojure.core/def
   app.common.data.macros/get-in clojure.core/get-in
   app.common.data.macros/with-open clojure.core/with-open
@@ -15,7 +14,7 @@
  :hooks
  {:analyze-call
   {app.common.data.macros/export hooks.export/export
-   potok.v2.core/reify hooks.export/potok-reify
+   potok.core/reify hooks.export/potok-reify
    app.util.services/defmethod hooks.export/service-defmethod
    app.common.record/defrecord hooks.export/penpot-defrecord
    app.db/with-atomic hooks.export/penpot-with-atomic
@@ -25,16 +24,9 @@
  :output
  {:exclude-files
   ["data_readers.clj"
-   "src/app/util/perf.cljs"
-   "src/app/common/logging.cljc"
-   "src/app/common/exceptions.cljc"
-   "^(?:backend|frontend|exporter|common)/build.clj"
-   "^(?:backend|frontend|exporter|common)/deps.edn"
-   "^(?:backend|frontend|exporter|common)/scripts/"
-   "^(?:backend|frontend|exporter|common)/dev/"
-   "^(?:backend|frontend|exporter|common)/test/"]
-
-  :linter-name true}
+   "app/util/perf.cljs"
+   "app/common/logging.cljc"
+   "app/common/exceptions.cljc"]}
 
  :linters
  {:unsorted-required-namespaces
@@ -70,3 +62,4 @@
    :exclude-destructured-keys-in-fn-args false
    }
   }}
+

.cljfmt.edn

@@ -1,8 +0,0 @@
-{:sort-ns-references? true
- :remove-multiple-non-indenting-spaces? false
- :remove-surrounding-whitespace? true
- :remove-consecutive-blank-lines? false
- :extra-indents {rumext.v2/fnc [[:inner 0]]
-                 promesa.exec/thread [[:inner 0]]
-                 specify! [[:inner 0] [:inner 1]]}
- }

.gitignore

@@ -1,12 +1,4 @@
-.pnp.*
-.yarn/*
-!.yarn/patches
-!.yarn/plugins
-!.yarn/releases
-!.yarn/sdks
-!.yarn/versions
 *-init.clj
-*.css.json
 *.jar
 *.orig
 *.penpot
@@ -66,4 +58,3 @@
 /web
 clj-profiler/
 node_modules
-frontend/.storybook/preview-body.html

.vscode/settings.json

@@ -1,9 +1,9 @@
 {
-  "files.exclude": {
-    "**/.clj-kondo": true,
-    "**/.cpcache": true,
-    "**/.lsp": true,
-    "**/.shadow-cljs": true,
-    "**/node_modules": true
-  }
+    "files.exclude": {
+        "**/.clj-kondo": true,
+        "**/.cpcache": true,
+        "**/.lsp": true,
+        "**/.shadow-cljs": true,
+        "**/node_modules": true
+    }
 }

.yarnrc.yml

@@ -1,9 +0,0 @@
-enableGlobalCache: true
-
-enableImmutableCache: false
-
-enableImmutableInstalls: false
-
-enableTelemetry: false
-
-nodeLinker: node-modules

CHANGES.md

@@ -1,17 +1,5 @@
 # CHANGELOG
 
-## :rocket: Next
-
-### :boom: Breaking changes & Deprecations
-
-### :sparkles: New features
-
-### :bug: Bugs fixed
-
-### :arrow_up: Deps updates
-
-### :heart: Community contributions by (Thank you!)
-
 ## 1.20.0
 
 ### :boom: Breaking changes & Deprecations
@@ -19,23 +7,13 @@
 ### :sparkles: New features
 
 - Select through stroke only rectangle [Taiga #5484](https://tree.taiga.io/project/penpot/issue/5484)
-- Override browser Ctrl+ and Ctrl- zoom with Penpot Zoom [Taiga #3200](https://tree.taiga.io/project/penpot/us/3200)
 
 ### :bug: Bugs fixed
 
-- Fix pixelated thumbnails [Github
-  #3681](https://github.com/penpot/penpot/issues/3681) [Github #3661](https://github.com/penpot/penpot/issues/3661)
-
 ### :arrow_up: Deps updates
 
 ### :heart: Community contributions by (Thank you!)
 
-## 1.19.5
-
-### :bug: New features
-
-- Fix problem with alignment performance
-
 ## 1.19.4
 
 ### :sparkles: New features

README.md

@@ -109,7 +109,6 @@ Every sort of contribution will be very helpful to enhance Penpot. How you’ll
 - Create and [share Libraries & templates](https://penpot.app/libraries-templates.html) that will be helpful for the community
 - Become a [translator](https://help.penpot.app/contributing-guide/translations)
 - Give feedback: [Mail us](mailto:support@penpot.app)
-- **Contribute to Penpot's code:** [Watch this video](https://www.youtube.com/watch?v=TpN0osiY-8k) by Alejandro Alonso, CIO and developer at Penpot, where he gives us a hands-on demo of how to use Penpot’s repository and make changes in both front and back end
 
 To find (almost) everything you need to know on how to contribute to Penpot, refer to the [contributing-guide](https://help.penpot.app/contributing-guide/).
 

backend/.gitignore

@@ -1,7 +0,0 @@
-.pnp.*
-.yarn/*
-!.yarn/patches
-!.yarn/plugins
-!.yarn/releases
-!.yarn/sdks
-!.yarn/versions

backend/deps.edn

@@ -3,10 +3,8 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.0-alpha5"}
-  org.clojure/tools.namespace {:mvn/version "1.4.4"}
-
-  com.github.luben/zstd-jni {:mvn/version "1.5.5-10"}
+  org.clojure/clojure {:mvn/version "1.11.1"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.5-5"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -21,19 +19,17 @@
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
   funcool/yetti
-  {:git/tag "v10.0"
-   :git/sha "520613f"
+  {:git/tag "v9.16"
+   :git/sha "7df3e08"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
-  com.github.seancorfield/next.jdbc {:mvn/version "1.3.894"}
+  com.github.seancorfield/next.jdbc {:mvn/version "1.3.883"}
   metosin/reitit-core {:mvn/version "0.6.0"}
-  nrepl/nrepl {:mvn/version "1.1.0"}
-  cider/cider-nrepl {:mvn/version "0.43.1"}
 
   org.postgresql/postgresql {:mvn/version "42.6.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "5.1.0"}
+  com.zaxxer/HikariCP {:mvn/version "5.0.1"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
@@ -42,7 +38,7 @@
 
   com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.8"}
 
-  org.jsoup/jsoup {:mvn/version "1.16.2"}
+  org.jsoup/jsoup {:mvn/version "1.16.1"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -51,9 +47,10 @@
   org.lz4/lz4-java {:mvn/version "1.8.0"}
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
+  integrant/integrant {:mvn/version "0.8.1"}
 
   dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.11.7"}
+  markdown-clj/markdown-clj {:mvn/version "1.11.4"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
@@ -65,6 +62,7 @@
  {:dev
   {:extra-deps
    {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
+    org.clojure/tools.namespace {:mvn/version "RELEASE"}
     clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
     org.clojure/data.csv {:mvn/version "RELEASE"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}

backend/dev/user.clj

@@ -21,8 +21,7 @@
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
-   [app.db :as db]
+   [app.config :as cfg]
    [app.main :as main]
    [app.srepl.helpers :as srepl.helpers]
    [app.srepl.main :as srepl]
@@ -56,6 +55,8 @@
 (repl/disable-reload! (find-ns 'integrant.core))
 (set! *warn-on-reflection* true)
 
+(defonce system nil)
+
 ;; --- Benchmarking Tools
 
 (defmacro run-quick-bench
@@ -93,14 +94,20 @@
 (defn- start
   []
   (try
-    (main/start)
+    (alter-var-root #'system (fn [sys]
+                               (when sys (ig/halt! sys))
+                               (-> (merge main/system-config main/worker-config)
+                                   (ig/prep)
+                                   (ig/init))))
     :started
     (catch Throwable cause
       (ex/print-throwable cause))))
 
 (defn- stop
   []
-  (main/stop)
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             nil))
   :stopped)
 
 (defn restart
@@ -113,26 +120,61 @@
   (stop)
   (repl/refresh-all :after 'user/start))
 
-;; (defn compression-bench
-;;   [data]
-;;   (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
-;;         v1 (time (humanize (alength (blob/encode data {:version 1}))))
-;;         v3 (time (humanize (alength (blob/encode data {:version 3}))))
-;;         v4 (time (humanize (alength (blob/encode data {:version 4}))))
-;;         v5 (time (humanize (alength (blob/encode data {:version 5}))))
-;;         v6 (time (humanize (alength (blob/encode data {:version 6}))))
-;;         ]
-;;     (print-table
-;;      [{
-;;        :v1 v1
-;;        :v3 v3
-;;        :v4 v4
-;;        :v5 v5
-;;        :v6 v6
-;;        }])))
+(defn compression-bench
+  [data]
+  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
+        v1 (time (humanize (alength (blob/encode data {:version 1}))))
+        v3 (time (humanize (alength (blob/encode data {:version 3}))))
+        v4 (time (humanize (alength (blob/encode data {:version 4}))))
+        v5 (time (humanize (alength (blob/encode data {:version 5}))))
+        v6 (time (humanize (alength (blob/encode data {:version 6}))))
+        ]
+    (print-table
+     [{
+       :v1 v1
+       :v3 v3
+       :v4 v4
+       :v5 v5
+       :v6 v6
+       }])))
 
 (defonce debug-tap
   (do
     (add-tap #(locking debug-tap
                 (prn "tap debug:" %)))
     1))
+
+
+(sm/def! ::test
+  [:map {:title "Foo"}
+   [:x :int]
+   [:y {:min 0} :double]
+   [:bar
+    [:map {:title "Bar"}
+     [:z :string]
+     [:v ::sm/uuid]]]
+   [:items
+    [:vector ::dt/instant]]])
+
+(sm/def! ::test2
+  [:multi {:title "Foo" :dispatch :type}
+   [:x
+    [:map {:title "FooX"}
+     [:type [:= :x]]
+     [:x :int]]]
+   [:y
+    [:map
+     [:type [:= :x]]
+     [:y [::sm/one-of #{:a :b :c}]]]]
+   [:z
+    [:map {:title "FooZ"}
+     [:z
+      [:multi {:title "Bar" :dispatch :type}
+       [:a
+        [:map
+         [:type [:= :a]]
+         [:a :int]]]
+       [:b
+        [:map
+         [:type [:= :b]]
+         [:b :int]]]]]]]])

backend/package.json

@@ -1,26 +1,18 @@
 {
-  "name": "backend",
-  "version": "1.0.0",
-  "license": "MPL-2.0",
-  "author": "Kaleidos INC",
-  "private": true,
-  "packageManager": "yarn@4.0.2",
+  "name": "uxbox-back",
+  "version": "0.1.0",
+  "description": "The Open-Source prototyping tool",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build-emails": "./scripts/build-email-templates.sh"
+  },
   "repository": {
     "type": "git",
-    "url": "https://github.com/penpot/penpot"
-  },
-  "dependencies": {
-    "luxon": "^3.4.2",
-    "sax": "^1.2.4"
+    "url": "git+https://github.com/uxbox/uxbox.git"
   },
+  "author": "Uxbox",
+  "license": "SEE LICENSE IN <LICENSE>",
   "devDependencies": {
-    "nodemon": "^3.0.1",
-    "source-map-support": "^0.5.21",
-    "ws": "^8.13.0"
-  },
-  "scripts": {
-    "fmt:clj:check": "cljfmt check --parallel=false src/ test/",
-    "fmt:clj": "cljfmt fix --parallel=true src/ test/",
-    "lint:clj": "clj-kondo --parallel --lint src/"
+    "mjml": "^4.6.3"
   }
 }

backend/resources/app/templates/api-doc-entry.tmpl

@@ -25,12 +25,6 @@
           <span>SCHEMA</span>
         </span>
       {% endif %}
-
-      {% if item.sse %}
-        <span class="tag">
-          <span>SSE</span>
-        </span>
-      {% endif %}
     </div>
   </div>
   <div class="rpc-row-detail hidden">

backend/resources/app/templates/api-doc.tmpl

@@ -56,17 +56,6 @@
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
 
-        <h3>SSE (Server-Sent Events)</h3>
-        <p>The methods marked with <b>SSE</b> returns
-        a <a href="https://html.spec.whatwg.org/multipage/server-sent-events.html"> SSE
-        formatted</a> stream on the response body, always with status 200. The events are
-        always encoded using `application/transit+json` encoding (for now no content
-        negotiation is possible on methods that return SSE streams). </p>
-        <p>On the javascript side you can use
-        the <a href="https://github.com/rexxars/eventsource-parser">eventsoure-parser</a>
-        library for propertly parsing the response body using the
-        standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
-        API</a></p>
 
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share

backend/resources/app/templates/debug.tmpl

@@ -6,17 +6,11 @@ Debug Main Page
 
 {% block content %}
 <nav>
-  <div class="title">
-    <h1>ADMIN DEBUG INTERFACE</h1>
-  </div>
+  <h1>Debug INDEX:</h1>
+  <div>[<a href="/dbg/error">ERRORS</a>]</div>
 </nav>
-<main class="dashboard">
+<main class="index">
   <section class="widget">
-    <fieldset>
-      <legend>Error reports</legend>
-      <desc><a href="/dbg/error">CLICK HERE TO SEE THE ERROR REPORTS</a> </desc>
-    </fieldset>
-
     <fieldset>
       <legend>Download file data:</legend>
       <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
@@ -43,42 +37,9 @@ Debug Main Page
           <input type="checkbox" name="reuseid" />
         </div>
 
-        <div class="row">
-          <input type="submit" value="Upload" />
-        </div>
+        <input type="submit" value="Upload" />
       </form>
     </fieldset>
-
-    <fieldset>
-      <legend>Profile Management</legend>
-      <form method="post" action="/dbg/actions/resend-email-verification">
-        <div class="row">
-          <input type="email" name="email" placeholder="example@example.com" value="" />
-        </div>
-
-        <div class="row">
-          <label for="force-verify">Are you sure?</label>
-          <input id="force-verify" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-        <div class="row">
-          <input type="submit" name="resend" value="Resend Verification" />
-          <input type="submit" name="verify" value="Verify" />
-        </div>
-
-        <div class="row">
-          <input type="submit" class="danger" name="block" value="Block" />
-          <input type="submit" class="danger" name="unblock" value="Unblock" />
-        </div>
-      </form>
-    </fieldset>
-
-
-
   </section>
 
   <section class="widget">
@@ -162,35 +123,5 @@ Debug Main Page
       </form>
     </fieldset>
   </section>
-
-  <section class="widget">
-    <fieldset>
-      <legend>Reset file version</legend>
-      <desc>Allows reset file data version to a specific number/</desc>
-
-      <form method="post" action="/dbg/actions/reset-file-data-version">
-        <div class="row">
-          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
-        </div>
-        <div class="row">
-          <input type="number" style="width:100px" name="version" placeholder="version" value="32" />
-        </div>
-
-        <div class="row">
-          <label for="force-version">Are you sure?</label>
-          <input id="force-version" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-
-        <div class="row">
-          <input type="submit" value="Submit" />
-        </div>
-      </form>
-    </fieldset>
-  </section>
 </main>
 {% endblock %}

backend/resources/app/templates/styles.css

@@ -116,50 +116,29 @@ nav > div:not(:last-child) {
   width: unset;
 }
 
-.dashboard {
+.index {
   margin-top: 40px;
   display: flex;
 }
 
-.widget {
+.index > section {
+  padding: 10px;
+  background-color: #e3e3e3;
   max-width: 400px;
   margin: 5px;
   height: fit-content;
 }
 
-.widget input[type=submit] {
-  outline: none;
-  border: 1px solid gray;
-  border-radius: 2px;
-  padding: 3px 5px;
-}
-
-.widget input[type=submit].danger {
-  outline: none;
-  border: 1px solid red;
-  border-radius: 2px;
-  padding: 3px 5px;
-}
-
-
-
-.widget > fieldset {
-  padding: 10px;
-  background-color: #f9f9f9;
-}
-
-.widget > fieldset:not(:last-child) {
-  margin-bottom: 10px;
-}
-
-
-
-.dashboard fieldset:not(:first-child) {
+.index fieldset:not(:first-child) {
   margin-top: 15px;
 }
 
+/* .index > section:not(:last-child) { */
+/*   margin-bottom: 10px; */
+/* } */
 
-.widget > h2 {
+
+.index > section > h2 {
   margin-top: 0px;
 }
 

backend/resources/climit.edn

@@ -3,17 +3,12 @@
 ;; Optional: queue, ommited means Integer/MAX_VALUE
 ;; Optional: timeout, ommited means no timeout
 ;; Note: queue and timeout are excluding
-{:update-file/by-profile
- {:permits 1 :queue 5}
+{:update-file-by-id {:permits 1 :queue 3}
+ :update-file {:permits 20}
 
- :update-file/global {:permits 20}
+ :derive-password {:permits 8}
+ :process-font  {:permits 4 :queue 32}
+ :process-image {:permits 8 :queue 32}
 
- :derive-password/global {:permits 8}
- :process-font/global  {:permits 4}
- :process-image/global {:permits 8}
-
- :file-thumbnail-ops/by-profile
- {:permits 2}
-
- :submit-audit-events/by-profile
+ :submit-audit-events-by-profile
  {:permits 1 :queue 3}}

backend/resources/log4j2-devenv.xml

@@ -30,7 +30,8 @@
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />
     <Logger name="app.rpc.climit" level="info" />
-    <Logger name="app.common.files.migrations" level="info" />
+    <Logger name="app.rpc.mutations.files" level="info" />
+    <Logger name="app.common.files.migrations" level="debug" />
 
     <Logger name="app.loggers" level="debug" additivity="false">
       <AppenderRef ref="main" level="debug" />

backend/resources/log4j2.xml

@@ -13,14 +13,10 @@
     <Logger name="org.postgresql" level="error" />
 
     <Logger name="app.util" level="info" />
-
-    <Logger name="app.loggers" level="debug" />
-
     <Logger name="app" level="info" additivity="false">
       <AppenderRef ref="console" />
     </Logger>
 
-
     <Root level="info">
       <AppenderRef ref="console" />
     </Root>

backend/scripts/manage.py

@@ -44,16 +44,11 @@ def send_eval(expr):
         s.send(b":repl/quit\n\n")
 
         with s.makefile() as f:
-            while True:
-                line = f.readline()
-                result = json.loads(line)
-                tag = result.get("tag", None)
-                if tag == "ret":
-                    return result.get("val", None), result.get("exception", None)
-                elif tag == "out":
-                    print(result.get("val"), end="")
-                else:
-                    raise RuntimeError("unexpected response from PREPL")
+            result = json.load(f)
+            tag = result.get("tag", None)
+            if tag != "ret":
+                raise RuntimeError("unexpected response from PREPL")
+            return result.get("val", None), result.get("exception", None)
 
 def encode(val):
     return json.dumps(json.dumps(val))
@@ -65,7 +60,7 @@ def print_error(res):
 
 def run_cmd(params):
     try:
-        expr = "(app.srepl.cli/exec {})".format(encode(params))
+        expr = "(app.srepl.ext/run-json-cmd {})".format(encode(params))
         res, failed = send_eval(expr)
         if failed:
             print_error(res)
@@ -145,22 +140,12 @@ def derive_password(password):
     res = run_cmd(params)
     print(f"Derived password: \"{res}\"")
 
-
-def migrate_components_v2():
-    params = {
-        "cmd": "migrate-v2",
-        "params": {}
-    }
-
-    run_cmd(params)
-
 available_commands = (
     "create-profile",
     "update-profile",
     "delete-profile",
     "search-profile",
     "derive-password",
-    "migrate-components-v2",
 )
 
 parser = argparse.ArgumentParser(
@@ -232,8 +217,3 @@ elif args.action == "search-profile":
         email = input("Email: ")
 
     search_profile(email)
-
-elif args.action == "migrate-components-v2":
-    migrate_components_v2()
-
-

backend/scripts/nrepl

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-clojure -J-Xms50m -J-Xmx256m -J-XX:+UseSerialGC -Sdeps '{:deps {reply/reply {:mvn/version "0.5.0"}}}' -M -m reply.main --attach localhost:6064 -e "(in-ns 'app.main)"

backend/scripts/repl

@@ -10,10 +10,9 @@ export PENPOT_FLAGS="\
        enable-login-with-google \
        enable-login-with-github \
        enable-login-with-gitlab \
-       enable-backend-worker \
        enable-backend-asserts \
-       enable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
+       enable-fdata-storage-pointer-map \
+       enable-fdata-storage-objets-map \
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
@@ -25,12 +24,7 @@ export PENPOT_FLAGS="\
        enable-rpc-rlimit \
        enable-soft-rpc-rlimit \
        enable-webhooks \
-       enable-access-tokens \
-       disable-feature-components-v2 \
-       enable-file-validation \
-       enable-file-schema-validation \
-       disable-soft-file-schema-validation \
-       disable-soft-file-validation";
+       enable-access-tokens";
 
 # export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
 # export PENPOT_DATABASE_USERNAME="penpot"
@@ -45,13 +39,10 @@ export PENPOT_FLAGS="\
 # export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
 
 # Initialize MINIO config
-mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
-mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-if [ "$?" = "1" ]; then
-    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-fi
-mc mb penpot-s3/penpot -p -q
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
+mc admin user add penpot-s3 penpot-devenv penpot-devenv
+mc admin policy attach penpot-s3 readwrite --user=penpot-devenv
+mc mb penpot-s3/penpot -p
 
 export AWS_ACCESS_KEY_ID=penpot-devenv
 export AWS_SECRET_ACCESS_KEY=penpot-devenv
@@ -59,23 +50,21 @@ export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
 export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
 export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
 
+#-J-Djdk.virtualThreadScheduler.parallelism=16
+
 export OPTIONS="
        -A:jmx-remote -A:dev \
        -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
        -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
        -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -J-XX:+EnableDynamicAgentLoading \
        -J-XX:-OmitStackTraceInFastThrow \
        -J-XX:+UnlockDiagnosticVMOptions \
        -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full"
-
-# Enable preview
-export OPTIONS="$OPTIONS -J--enable-preview"
+       -J-Djdk.tracePinnedThreads=full \
+       -J--enable-preview";
 
 # Setup HEAP
-# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
+export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
 # export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
 
 # Increase virtual thread pool size
@@ -88,7 +77,7 @@ export OPTIONS="$OPTIONS -J--enable-preview"
 # export OPTIONS="$OPTIONS -J-Xint"
 
 # Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
+export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
 
 # Setup GC
 # export OPTIONS="$OPTIONS -J-XX:+UseZGC"

backend/scripts/start-dev

@@ -6,90 +6,33 @@ export PENPOT_FLAGS="\
        $PENPOT_FLAGS \
        enable-prepl-server \
        enable-urepl-server \
-       enable-nrepl-server \
        enable-webhooks \
        enable-backend-asserts \
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
-       enable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
+       enable-fdata-storage-pointer-map \
+       enable-fdata-storage-objets-map \
        disable-secure-session-cookies \
        enable-smtp \
-       enable-access-tokens \
-       disable-feature-components-v2 \
-       enable-file-validation \
-       enable-file-schema-validation \
-       disable-soft-file-schema-validation \
-       disable-soft-file-validation";
+       enable-access-tokens";
 
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-Dlog4j2.configurationFile=log4j2.xml \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints"
-
-# Setup HEAP
-# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
-# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
-
-# Increase virtual thread pool size
-# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
-
-# Disable C2 Compiler
-# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
-
-# Disable all compilers
-# export OPTIONS="$OPTIONS -J-Xint"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
-
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
-
-# Initialize MINIO config
-mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
-mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-if [ "$?" = "1" ]; then
-    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-fi
-mc mb penpot-s3/penpot -p -q
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
-export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
-export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
+set -ex
 
 if [ "$1" = "--watch" ]; then
-    trap "exit" INT TERM ERR
-    trap "kill 0" EXIT
-
     echo "Start Watch..."
 
-    clojure $OPTIONS -A:dev -M -m app.main &
+    clojure -A:dev -M -m app.main &
+    PID=$!
 
     npx nodemon \
         --watch src \
         --watch ../common \
         --ext "clj" \
         --signal SIGKILL \
-        --exec 'echo "(app.main/stop)\n\r(repl/refresh)\n\r(app.main/start)\n" | nc -N localhost 6062'
-
-    wait;
+        --exec 'echo "(user/restart)" | nc -N localhost 6062'
 
+    kill -9 $PID
 else
-    set -x
-    clojure $OPTIONS -A:dev -M -m app.main;
+    clojure -A:dev -M -m app.main
 fi

backend/src/app/auth/oidc.clj

@@ -31,7 +31,7 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [ring.response :as-alias rres]))
+   [yetti.response :as-alias yrs]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -353,7 +353,8 @@
           (get-name [props]
             (let [attr-kw (cf/get :oidc-name-attr "name")
                   attr-ph (parse-attr-path provider attr-kw)]
-              (get-in props attr-ph)))]
+              (get-in props attr-ph)))
+          ]
 
     (let [props (qualify-props provider info)
           email (get-email props)]
@@ -478,8 +479,8 @@
 
 (defn- redirect-response
   [uri]
-  {::rres/status 302
-   ::rres/headers {"location" (str uri)}})
+  {::yrs/status 302
+   ::yrs/headers {"location" (str uri)}})
 
 (defn- generate-error-redirect
   [_ cause]
@@ -556,8 +557,8 @@
                                 :props props
                                 :exp (dt/in-future "4h")})
         uri   (build-auth-uri cfg state)]
-    {::rres/status 200
-     ::rres/body {:redirect-uri uri}}))
+    {::yrs/status 200
+     ::yrs/body {:redirect-uri uri}}))
 
 (defn- callback-handler
   [cfg request]

backend/src/app/config.clj

@@ -203,7 +203,6 @@
 (s/def ::storage-assets-s3-bucket ::us/string)
 (s/def ::storage-assets-s3-region ::us/keyword)
 (s/def ::storage-assets-s3-endpoint ::us/string)
-(s/def ::storage-assets-s3-io-threads ::us/integer)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
@@ -295,7 +294,6 @@
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::rpc-rlimit-config
-                   ::rpc-climit-config
 
                    ::semaphore-process-font
                    ::semaphore-process-image
@@ -321,7 +319,6 @@
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
                    ::storage-assets-s3-endpoint
-                   ::storage-assets-s3-io-threads
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer

backend/src/app/db.clj

@@ -97,7 +97,7 @@
             :with-credentials (and (contains? cfg ::username)
                                    (contains? cfg ::password))
             :min-size (::min-size cfg)
-            :max-size (::max-size cfg))
+          :max-size (::max-size cfg))
     (create-pool cfg)))
 
 (defmethod ig/halt-key! ::pool
@@ -231,82 +231,62 @@
     `(jdbc/with-transaction ~@args)))
 
 (defn open
-  [system-or-pool]
-  (if (pool? system-or-pool)
-    (jdbc/get-connection system-or-pool)
-    (if (map? system-or-pool)
-      (open (::pool system-or-pool))
-      (ex/raise :type :internal
-                :code :unable-resolve-pool))))
+  [pool]
+  (jdbc/get-connection pool))
 
-(defn get-connection
-  [cfg-or-conn]
-  (if (connection? cfg-or-conn)
-    cfg-or-conn
-    (if (map? cfg-or-conn)
-      (get-connection (::conn cfg-or-conn))
-      (ex/raise :type :internal
-                :code :unable-resolve-connection
-                :hint "expected conn or system map"))))
-
-(defn connection-map?
-  "Check if the provided value is a map like data structure that
-  contains a database connection."
+(defn- resolve-connectable
   [o]
-  (and (map? o) (connection? (::conn o))))
+  (if (connection? o)
+    o
+    (if (pool? o)
+      o
+      (or (::conn o) (::pool o)))))
 
-(defn- get-connectable
-  [o]
-  (cond
-    (connection? o) o
-    (pool? o)       o
-    (map? o)        (get-connectable (or (::conn o) (::pool o)))
-    :else           (ex/raise :type :internal
-                              :code :unable-resolve-connectable
-                              :hint "expected conn, pool or system")))
 
 (def ^:private default-opts
   {:builder-fn sql/as-kebab-maps})
 
 (defn exec!
   ([ds sv]
-   (-> (get-connectable ds)
+   (-> (resolve-connectable ds)
        (jdbc/execute! sv default-opts)))
   ([ds sv opts]
-   (-> (get-connectable ds)
-       (jdbc/execute! sv (into default-opts (sql/adapt-opts opts))))))
+   (-> (resolve-connectable ds)
+       (jdbc/execute! sv (merge default-opts opts)))))
 
 (defn exec-one!
   ([ds sv]
-   (-> (get-connectable ds)
+   (-> (resolve-connectable ds)
        (jdbc/execute-one! sv default-opts)))
   ([ds sv opts]
-   (-> (get-connectable ds)
-       (jdbc/execute-one! sv (into default-opts (sql/adapt-opts opts))))))
+   (-> (resolve-connectable ds)
+       (jdbc/execute-one! sv
+                          (-> (merge default-opts opts)
+                              (assoc :return-keys (::return-keys? opts false)))))))
 
 (defn insert!
-  [ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
-  (-> (get-connectable ds)
+  [ds table params & {:as opts}]
+  (-> (resolve-connectable ds)
       (exec-one! (sql/insert table params opts)
-                 (assoc opts ::return-keys? return-keys?))))
+                 (merge {::return-keys? true} opts))))
 
 (defn insert-multi!
-  [ds table cols rows & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
-  (-> (get-connectable ds)
+  [ds table cols rows & {:as opts}]
+  (-> (resolve-connectable ds)
       (exec! (sql/insert-multi table cols rows opts)
-             (assoc opts ::return-keys? return-keys?))))
+             (merge {::return-keys? true} opts))))
 
 (defn update!
-  [ds table params where & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
-  (-> (get-connectable ds)
+  [ds table params where & {:as opts}]
+  (-> (resolve-connectable ds)
       (exec-one! (sql/update table params where opts)
-                 (assoc opts ::return-keys? return-keys?))))
+                 (merge {::return-keys? true} opts))))
 
 (defn delete!
-  [ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
-  (-> (get-connectable ds)
+  [ds table params & {:as opts}]
+  (-> (resolve-connectable ds)
       (exec-one! (sql/delete table params opts)
-                 (assoc opts ::return-keys? return-keys?))))
+                 (merge {::return-keys? true} opts))))
 
 (defn is-row-deleted?
   [{:keys [deleted-at]}]
@@ -338,7 +318,7 @@
 
 (defn plan
   [ds sql]
-  (-> (get-connectable ds)
+  (-> (resolve-connectable ds)
       (jdbc/plan sql sql/default-opts)))
 
 (defn get-by-id
@@ -416,65 +396,57 @@
    (.setSavepoint conn (name label))))
 
 (defn release!
-  [^Connection conn ^Savepoint sp]
+  [^Connection conn ^Savepoint sp ]
   (.releaseSavepoint conn sp))
 
 (defn rollback!
-  ([conn]
-   (let [^Connection conn (get-connection conn)]
-     (l/trc :hint "explicit rollback requested")
-     (.rollback conn)))
-  ([conn ^Savepoint sp]
-   (let [^Connection conn (get-connection conn)]
-     (l/trc :hint "explicit rollback requested")
-     (.rollback conn sp))))
+  ([^Connection conn]
+   (.rollback conn))
+  ([^Connection conn ^Savepoint sp]
+   (.rollback conn sp)))
 
 (defn tx-run!
-  [system f & params]
+  [cfg f]
   (cond
-    (connection? system)
-    (tx-run! {::conn system} f)
+    (connection? cfg)
+    (tx-run! {::conn cfg} f)
 
-    (pool? system)
-    (tx-run! {::pool system} f)
+    (pool? cfg)
+    (tx-run! {::pool cfg} f)
 
-    (::conn system)
-    (let [conn (::conn system)
+    (::conn cfg)
+    (let [conn (::conn cfg)
           sp   (savepoint conn)]
       (try
-        (let [result (apply f system params)]
+        (let [result (f cfg)]
           (release! conn sp)
           result)
         (catch Throwable cause
-          (rollback! conn sp)
+          (rollback! sp)
           (throw cause))))
 
-    (::pool system)
-    (with-atomic [conn (::pool system)]
-      (let [system (assoc system ::conn conn)
-            result (apply f system params)]
-        (when (::rollback system)
-          (rollback! conn))
-        result))
+    (::pool cfg)
+    (with-atomic [conn (::pool cfg)]
+      (f (assoc cfg ::conn conn)))
 
     :else
     (throw (IllegalArgumentException. "invalid arguments"))))
 
 (defn run!
-  [system f & params]
+  [cfg f]
   (cond
-    (connection? system)
-    (run! {::conn system} f)
+    (connection? cfg)
+    (run! {::conn cfg} f)
 
-    (pool? system)
-    (run! {::pool system} f)
+    (pool? cfg)
+    (run! {::pool cfg} f)
 
-    (::conn system)
-    (apply f system params)
+    (::conn cfg)
+    (f cfg)
 
-    (::pool system)
-    (with-open [^Connection conn (open (::pool system))]
-      (apply f (assoc system ::conn conn) params))
+    (::pool cfg)
+    (with-open [^Connection conn (open (::pool cfg))]
+      (f (assoc cfg ::conn conn)))
 
     :else
     (throw (IllegalArgumentException. "invalid arguments"))))

backend/src/app/db/sql.clj

@@ -8,7 +8,6 @@
   (:refer-clojure :exclude [update])
   (:require
    [app.db :as-alias db]
-   [clojure.set :as set]
    [clojure.string :as str]
    [next.jdbc.optional :as jdbc-opt]
    [next.jdbc.sql.builder :as sql]))
@@ -20,14 +19,6 @@
   {:table-fn snake-case
    :column-fn snake-case})
 
-(def params-mapping
-  {::db/return-keys? :return-keys
-   ::db/columns :columns})
-
-(defn adapt-opts
-  [opts]
-  (set/rename-keys opts params-mapping))
-
 (defn as-kebab-maps
   [rs opts]
   (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
@@ -38,7 +29,7 @@
   ([table key-map opts]
    (let [opts (merge default-opts opts)
          opts (cond-> opts
-                (::db/on-conflict-do-nothing? opts)
+                (:on-conflict-do-nothing opts)
                 (assoc :suffix "ON CONFLICT DO NOTHING"))]
      (sql/for-insert table key-map opts))))
 
@@ -53,7 +44,6 @@
   ([table where-params opts]
    (let [opts (merge default-opts opts)
          opts (cond-> opts
-                (::db/columns opts)     (assoc :columns (::db/columns opts))
                 (::db/for-update? opts) (assoc :suffix "FOR UPDATE")
                 (::db/for-share? opts)  (assoc :suffix "FOR KEY SHARE")
                 (:for-update opts)      (assoc :suffix "FOR UPDATE")
@@ -64,13 +54,7 @@
   ([table key-map where-params]
    (update table key-map where-params nil))
   ([table key-map where-params opts]
-   (let [opts (into default-opts opts)
-         opts (if-let [columns (::db/columns opts)]
-                (let [columns   (if (seq columns)
-                                  (sql/as-cols columns opts)
-                                  "*")]
-                  (assoc opts :suffix (str "RETURNING " columns)))
-                opts)]
+   (let [opts (merge default-opts opts)]
      (sql/for-update table key-map where-params opts))))
 
 (defn delete

backend/src/app/email.clj

@@ -311,12 +311,6 @@
                     ^String (::password cfg))
 
           (let [^MimeMessage message (create-smtp-message cfg session params)]
-            (l/dbg :hint "sendmail"
-                   :id (:id params)
-                   :to (:to params)
-                   :subject (str/trim (:subject params))
-                   :body (str/join "," (map :type (:body params))))
-
             (.sendMessage ^Transport transport
                           ^MimeMessage message
                           (.getAllRecipients message))))))

backend/src/app/features/components_v2.clj

@@ -1,927 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.features.components-v2
-  (:require
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.exceptions :as ex]
-   [app.common.files.changes :as cp]
-   [app.common.files.changes-builder :as fcb]
-   [app.common.files.helpers :as cfh]
-   [app.common.files.libraries-helpers :as cflh]
-   [app.common.files.migrations :as fmg]
-   [app.common.files.shapes-helpers :as cfsh]
-   [app.common.files.validate :as cfv]
-   [app.common.geom.point :as gpt]
-   [app.common.geom.rect :as grc]
-   [app.common.geom.shapes :as gsh]
-   [app.common.logging :as l]
-   [app.common.svg :as csvg]
-   [app.common.svg.shapes-builder :as sbuilder]
-   [app.common.types.component :as ctk]
-   [app.common.types.components-list :as ctkl]
-   [app.common.types.container :as ctn]
-   [app.common.types.file :as ctf]
-   [app.common.types.pages-list :as ctpl]
-   [app.common.types.shape :as cts]
-   [app.common.types.shape-tree :as ctst]
-   [app.common.uuid :as uuid]
-   [app.db :as db]
-   [app.features.fdata :as fdata]
-   [app.http.sse :as sse]
-   [app.media :as media]
-   [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-snapshot :as fsnap]
-   [app.rpc.commands.media :as cmd.media]
-   [app.storage :as sto]
-   [app.storage.tmp :as tmp]
-   [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]
-   [app.util.time :as dt]
-   [buddy.core.codecs :as bc]
-   [cuerdas.core :as str]
-   [datoteka.io :as io]
-   [promesa.exec :as px]
-   [promesa.exec.semaphore :as ps]
-   [promesa.util :as pu]))
-
-(def ^:dynamic *system* nil)
-(def ^:dynamic *stats* nil)
-(def ^:dynamic *file-stats* nil)
-(def ^:dynamic *team-stats* nil)
-(def ^:dynamic *semaphore* nil)
-(def ^:dynamic *skip-on-error* true)
-
-(def grid-gap 50)
-(def frame-gap 200)
-(def max-group-size 50)
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; FILE PREPARATION BEFORE MIGRATION
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- prepare-file-data
-  "Apply some specific migrations or fixes to things that are allowed in v1 but not in v2,
-   or that are the result of old bugs."
-  [file-data libraries]
-  (let [detached-ids  (volatile! #{})
-
-        detach-shape
-        (fn [container shape]
-          ;; Detach a shape. If it's inside a component, add it to detached-ids, for further use.
-          (let [is-component? (let [root-shape (ctst/get-shape container (:id container))]
-                                (and (some? root-shape) (nil? (:parent-id root-shape))))]
-            (when is-component?
-              (vswap! detached-ids conj (:id shape)))
-            (ctk/detach-shape shape)))
-
-        fix-orphan-shapes
-        (fn [file-data]
-          ;; Find shapes that are not listed in their parent's children list.
-          ;; Remove them, and also their children
-          (letfn [(fix-container [container]
-                    (reduce fix-shape container (ctn/shapes-seq container)))
-
-                  (fix-shape
-                    [container shape]
-                    (if-not (or (= (:id shape) uuid/zero)
-                                (nil? (:parent-id shape)))
-                      (let [parent (ctst/get-shape container (:parent-id shape))
-                            exists? (d/index-of (:shapes parent) (:id shape))]
-                        (if (nil? exists?)
-                          (let [ids (cfh/get-children-ids-with-self (:objects container) (:id shape))]
-                            (update container :objects #(reduce dissoc % ids)))
-                          container))
-                      container))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        remove-nested-roots
-        (fn [file-data]
-          ;; Remove :component-root in head shapes that are nested.
-          (letfn [(fix-container [container]
-                    (update container :objects update-vals (partial fix-shape container)))
-
-                  (fix-shape [container shape]
-                    (let [parent (ctst/get-shape container (:parent-id shape))]
-                      (if (and (ctk/instance-root? shape)
-                               (ctn/in-any-component? (:objects container) parent))
-                        (dissoc shape :component-root)
-                        shape)))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        add-not-nested-roots
-        (fn [file-data]
-          ;; Add :component-root in head shapes that are not nested.
-          (letfn [(fix-container [container]
-                    (update container :objects update-vals (partial fix-shape container)))
-
-                  (fix-shape [container shape]
-                    (let [parent (ctst/get-shape container (:parent-id shape))]
-                      (if (and (ctk/subinstance-head? shape)
-                               (not (ctn/in-any-component? (:objects container) parent)))
-                        (assoc shape :component-root true)
-                        shape)))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        fix-orphan-copies
-        (fn [file-data]
-          ;; Detach shapes that were inside a copy (have :shape-ref) but now they aren't.
-          (letfn [(fix-container [container]
-                    (update container :objects update-vals (partial fix-shape container)))
-
-                  (fix-shape [container shape]
-                    (let [parent (ctst/get-shape container (:parent-id shape))]
-                      (if (and (ctk/in-component-copy? shape)
-                               (not (ctk/instance-head? shape))
-                               (not (ctk/in-component-copy? parent)))
-                        (detach-shape container shape)
-                        shape)))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        remap-refs
-        (fn [file-data]
-          ;; Remap shape-refs so that they point to the near main.
-          ;; At the same time, if there are any dangling ref, detach the shape and its children.
-          (letfn [(fix-container [container]
-                    (reduce fix-shape container (ctn/shapes-seq container)))
-
-                  (fix-shape [container shape]
-                    (if (ctk/in-component-copy? shape)
-                      ;; First look for the direct shape.
-                      (let [root         (ctn/get-component-shape (:objects container) shape)
-                            libraries    (assoc-in libraries [(:id file-data) :data] file-data)
-                            library      (get libraries (:component-file root))
-                            component    (ctkl/get-component (:data library) (:component-id root) true)
-                            direct-shape (ctf/get-component-shape (:data library) component (:shape-ref shape))]
-                        (if (some? direct-shape)
-                          ;; If it exists, there is nothing else to do.
-                          container
-                          ;; If not found, find the near shape.
-                          (let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape))
-                                                   (ctf/get-component-shapes (:data library) component))]
-                            (if (some? near-shape)
-                              ;; If found, update the ref to point to the near shape.
-                              (ctn/update-shape container (:id shape) #(assoc % :shape-ref (:id near-shape)))
-                              ;; If not found, it may be a fostered component. Try to locate a direct shape
-                              ;; in the head component.
-                              (let [head           (ctn/get-head-shape (:objects container) shape)
-                                    library-2      (get libraries (:component-file head))
-                                    component-2    (ctkl/get-component (:data library-2) (:component-id head) true)
-                                    direct-shape-2 (ctf/get-component-shape (:data library-2) component-2 (:shape-ref shape))]
-                                (if (some? direct-shape-2)
-                                  ;; If it exists, there is nothing else to do.
-                                  container
-                                  ;; If not found, detach shape and all children (stopping if a nested instance is reached)
-                                  (let [children (ctn/get-children-in-instance (:objects container) (:id shape))]
-                                    (reduce #(ctn/update-shape %1 (:id %2) (partial detach-shape %1))
-                                            container
-                                            children))))))))
-                      container))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        fix-copies-of-detached
-        (fn [file-data]
-          ;; Find any copy that is referencing a detached shape inside a component, and
-          ;; undo the nested copy, converting it into a direct copy.
-          (letfn [(fix-container [container]
-                    (update container :objects update-vals fix-shape))
-
-                  (fix-shape [shape]
-                    (cond-> shape
-                      (@detached-ids (:shape-ref shape))
-                      (dissoc shape
-                              :component-id
-                              :component-file
-                              :component-root)))]
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        transform-to-frames
-        (fn [file-data]
-          ;; Transform component and copy heads to frames, and set the
-          ;; frame-id of its childrens
-          (letfn [(fix-container
-                    [container]
-                    (update container :objects update-vals fix-shape))
-
-                  (fix-shape
-                    [shape]
-                    (if (or (nil? (:parent-id shape)) (ctk/instance-head? shape))
-                      (assoc shape
-                        :type :frame                  ; Old groups must be converted
-                        :fills (or (:fills shape) []) ; to frames and conform to spec
-                        :hide-in-viewer (or (:hide-in-viewer shape) true)
-                        :rx (or (:rx shape) 0)
-                        :ry (or (:ry shape) 0))
-                      shape))]
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        remap-frame-ids
-        (fn [file-data]
-          ;; Remap the frame-ids of the primary childs of the head instances
-          ;; to point to the head instance.
-          (letfn [(fix-container
-                    [container]
-                    (update container :objects update-vals (partial fix-shape container)))
-
-                  (fix-shape
-                    [container shape]
-                    (let [parent (ctst/get-shape container (:parent-id shape))]
-                      (if (ctk/instance-head? parent)
-                        (assoc shape :frame-id (:id parent))
-                        shape)))]
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        fix-frame-ids
-        (fn [file-data]
-          ;; Ensure that frame-id of all shapes point to the parent or to the frame-id
-          ;; of the parent, and that the destination is indeed a frame.
-          (letfn [(fix-container [container]
-                    (update container :objects #(cfh/reduce-objects % fix-shape %)))
-
-                  (fix-shape [objects shape]
-                    (let [parent (when (:parent-id shape)
-                                   (get objects (:parent-id shape)))
-                          error? (when (some? parent)
-                                   (if (= (:type parent) :frame)
-                                     (not= (:frame-id shape) (:id parent))
-                                     (not= (:frame-id shape) (:frame-id parent))))]
-                      (if error?
-                        (let [nearest-frame (cfh/get-frame objects (:parent-id shape))
-                              frame-id      (or (:id nearest-frame) uuid/zero)]
-                          (update objects (:id shape) assoc :frame-id frame-id))
-                        objects)))]
-
-            (-> file-data
-                (update :pages-index update-vals fix-container)
-                (update :components update-vals fix-container))))
-
-        fix-component-nil-objects
-        (fn [file-data]
-          ;; Ensure that objects of all components is not null
-          (letfn [(fix-component [component]
-                    (if (and (contains? component :objects) (nil? (:objects component)))
-                      (if (:deleted component)
-                        (assoc component :objects {})
-                        (dissoc component :objects))
-                      component))]
-            (-> file-data
-                (update :components update-vals fix-component))))]
-
-    (-> file-data
-        (fix-orphan-shapes)
-        (remove-nested-roots)
-        (add-not-nested-roots)
-        (fix-orphan-copies)
-        (remap-refs)
-        (fix-copies-of-detached)
-        (transform-to-frames)
-        (remap-frame-ids)
-        (fix-frame-ids)
-        (fix-component-nil-objects))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; COMPONENTS MIGRATION
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- get-asset-groups
-  [assets generic-name]
-  (let [; Group by first element of the path.
-        groups (d/group-by #(first (cfh/split-path (:path %))) assets)
-
-        ; Split large groups in chunks of max-group-size elements
-        groups (loop [groups (seq groups)
-                      result {}]
-                 (if (empty? groups)
-                   result
-                   (let [[group-name assets] (first groups)
-                         group-name (if (or (nil? group-name) (str/empty? group-name))
-                                      generic-name
-                                      group-name)]
-                     (if (<= (count assets) max-group-size)
-                       (recur (next groups)
-                              (assoc result group-name assets))
-                       (let [splits (-> (partition-all max-group-size assets)
-                                        (d/enumerate))]
-                         (recur (next groups)
-                                (reduce (fn [result [index split]]
-                                          (let [split-name (str group-name " " (inc index))]
-                                            (assoc result split-name split)))
-                                        result
-                                        splits)))))))
-
-        ; Sort assets in each group by path
-        groups (update-vals groups (fn [assets]
-                                     (sort-by (fn [{:keys [path name]}]
-                                                (str/lower (cfh/merge-path-item path name)))
-                                              assets)))
-
-        ; Sort groups by name
-        groups (into (sorted-map) groups)]
-    groups))
-
-(defn- create-frame
-  [name position width height]
-  (cts/setup-shape
-   {:type :frame
-    :x (:x position)
-    :y (:y position)
-    :width (+ width (* 2 grid-gap))
-    :height (+ height (* 2 grid-gap))
-    :name name
-    :frame-id uuid/zero
-    :parent-id uuid/zero}))
-
-(defn- migrate-components
-  "If there is any component in the file library, add a new 'Library
-  backup', generate main instances for all components there and remove
-  shapes from library components.  Mark the file with
-  the :components-v2 option."
-  [file-data libraries]
-  (sse/tap {:type :migration-progress
-            :section :components})
-  (let [components (ctkl/components-seq file-data)]
-    (if (empty? components)
-      (assoc-in file-data [:options :components-v2] true)
-      (let [[file-data page-id start-pos]
-            (ctf/get-or-add-library-page file-data frame-gap)
-
-            migrate-component-shape
-            (fn [shape delta component-file component-id frame-id]
-              (cond-> shape
-                (nil? (:parent-id shape))
-                (assoc :parent-id frame-id
-                       :main-instance true
-                       :component-root true
-                       :component-file component-file
-                       :component-id component-id)
-
-                (nil? (:frame-id shape))
-                (assoc :frame-id frame-id)
-
-                :always
-                (gsh/move delta)))
-
-            add-main-instance
-            (fn [file-data component frame-id position]
-              (let [shapes (cfh/get-children-with-self (:objects component)
-                                                       (:id component))
-
-                    root-shape (first shapes)
-                    orig-pos   (gpt/point (:x root-shape) (:y root-shape))
-                    delta      (gpt/subtract position orig-pos)
-
-                    xf-shape (map #(migrate-component-shape %
-                                                            delta
-                                                            (:id file-data)
-                                                            (:id component)
-                                                            frame-id))
-                    new-shapes
-                    (into [] xf-shape shapes)
-
-                    find-frame-id ; if its parent is a frame, the frame-id should be the parent-id
-                    (fn [page shape]
-                      (let [parent (ctst/get-shape page (:parent-id shape))]
-                        (if (= :frame (:type parent))
-                          (:id parent)
-                          (:frame-id parent))))
-
-                    add-shapes
-                    (fn [page]
-                      (reduce (fn [page shape]
-                                (ctst/add-shape (:id shape)
-                                                shape
-                                                page
-                                                (find-frame-id page shape)
-                                                (:parent-id shape)
-                                                nil     ; <- As shapes are ordered, we can safely add each
-                                                true))  ;    one at the end of the parent's children list.
-                              page
-                              new-shapes))
-
-                    update-component
-                    (fn [component]
-                      (-> component
-                          (assoc :main-instance-id (:id root-shape)
-                                 :main-instance-page page-id)
-                          (dissoc :objects)))]
-
-                (-> file-data
-                    (ctpl/update-page page-id add-shapes)
-                    (ctkl/update-component (:id component) update-component))))
-
-            add-instance-grid
-            (fn [fdata frame-id grid assets]
-              (reduce (fn [result [component position]]
-                        (sse/tap {:type :migration-progress
-                                  :section :components
-                                  :name (:name component)})
-                        (add-main-instance result component frame-id (gpt/add position
-                                                                              (gpt/point grid-gap grid-gap))))
-                      fdata
-                      (d/zip assets grid)))
-
-            add-instance-grids
-            (fn [fdata]
-              (let [components (ctkl/components-seq fdata)
-                    groups     (get-asset-groups components "Components")]
-                (loop [groups   (seq groups)
-                       fdata    fdata
-                       position start-pos]
-                  (if (empty? groups)
-                    fdata
-                    (let [[group-name assets]    (first groups)
-                          grid                   (ctst/generate-shape-grid
-                                                  (map (partial ctf/get-component-root fdata) assets)
-                                                  position
-                                                  grid-gap)
-                          {:keys [width height]} (meta grid)
-                          frame                  (create-frame group-name position width height)
-                          fdata                  (ctpl/update-page fdata
-                                                                   page-id
-                                                                   #(ctst/add-shape (:id frame)
-                                                                                    frame
-                                                                                    %
-                                                                                    (:id frame)
-                                                                                    (:id frame)
-                                                                                    nil
-                                                                                    true))]
-                      (recur (next groups)
-                             (add-instance-grid fdata (:id frame) grid assets)
-                             (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))]
-
-        (let [total (count components)]
-          (some-> *stats* (swap! update :processed/components (fnil + 0) total))
-          (some-> *team-stats* (swap! update :processed/components (fnil + 0) total))
-          (some-> *file-stats* (swap! assoc :processed/components total)))
-
-        (-> file-data
-            (prepare-file-data libraries)
-            (add-instance-grids))))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; GRAPHICS MIGRATION
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- create-shapes-for-bitmap
-  "Convert a media object that contains a bitmap image into shapes,
-  one shape of type :image and one group that contains it."
-  [{:keys [name width height id mtype]} frame-id position]
-  (let [frame-shape (cts/setup-shape
-                     {:type :frame
-                      :x (:x position)
-                      :y (:y position)
-                      :width width
-                      :height height
-                      :name name
-                      :frame-id frame-id
-                      :parent-id frame-id})
-
-        img-shape   (cts/setup-shape
-                     {:type :image
-                      :x (:x position)
-                      :y (:y position)
-                      :width width
-                      :height height
-                      :metadata {:id id
-                                 :width width
-                                 :height height
-                                 :mtype mtype}
-                      :name name
-                      :frame-id (:id frame-shape)
-                      :parent-id (:id frame-shape)})]
-    [frame-shape [img-shape]]))
-
-(defn- parse-datauri
-  [data]
-  (let [[mtype b64-data] (str/split data ";base64," 2)
-        mtype (subs mtype (inc (str/index-of mtype ":")))
-        data  (-> b64-data bc/str->bytes bc/b64->bytes)]
-    [mtype data]))
-
-(defn- extract-name
-  [href]
-  (let [query-idx (d/nilv (str/last-index-of href "?") 0)
-        href (if (> query-idx 0) (subs href 0 query-idx) href)
-        filename (->> (str/split href "/") (last))
-        ext-idx (str/last-index-of filename ".")]
-    (if (> ext-idx 0) (subs filename 0 ext-idx) filename)))
-
-(defn- collect-and-persist-images
-  [svg-data file-id]
-  (letfn [(process-image [{:keys [href] :as item}]
-            (try
-              (let [item (if (str/starts-with? href "data:")
-                           (let [[mtype data] (parse-datauri href)
-                                 size         (alength data)
-                                 path         (tmp/tempfile :prefix "penpot.media.download.")
-                                 written      (io/write-to-file! data path :size size)]
-
-                             (when (not= written size)
-                               (ex/raise :type :internal
-                                         :code :mismatch-write-size
-                                         :hint "unexpected state: unable to write to file"))
-
-                             (-> item
-                                 (assoc :size size)
-                                 (assoc :path path)
-                                 (assoc :filename "tempfile")
-                                 (assoc :mtype mtype)))
-
-                           (let [result (cmd.media/download-image *system* href)]
-                             (-> (merge item result)
-                                 (assoc :name (extract-name href)))))]
-
-                ;; The media processing adds the data to the
-                ;; input map and returns it.
-                (media/run {:cmd :info :input item}))
-
-              (catch Throwable cause
-                (l/warn :hint "unexpected exception on processing internal image shape (skiping)"
-                        :cause cause)
-                (when-not *skip-on-error*
-                  (throw cause)))))
-
-          (persist-image [acc {:keys [path size width height mtype href] :as item}]
-            (let [storage (::sto/storage *system*)
-                  conn    (::db/conn *system*)
-                  hash    (sto/calculate-hash path)
-                  content (-> (sto/content path size)
-                              (sto/wrap-with-hash hash))
-                  params  {::sto/content content
-                           ::sto/deduplicate? true
-                           ::sto/touched-at (:ts item)
-                           :content-type mtype
-                           :bucket "file-media-object"}
-                  image   (sto/put-object! storage params)
-                  fmo-id  (uuid/next)]
-
-              (db/exec-one! conn
-                            [cmd.media/sql:create-file-media-object
-                             fmo-id
-                             file-id true (:name item "image")
-                             (:id image)
-                             nil
-                             width
-                             height
-                             mtype])
-
-              (assoc acc href {:id fmo-id
-                               :mtype mtype
-                               :width width
-                               :height height})))]
-
-    (let [images (->> (csvg/collect-images svg-data)
-                      (transduce (keep process-image)
-                                 (completing persist-image) {}))]
-      (assoc svg-data :image-data images))))
-
-(defn- get-svg-content
-  [id]
-  (let [storage  (::sto/storage *system*)
-        conn     (::db/conn *system*)
-        fmobject (db/get conn :file-media-object {:id id})
-        sobject  (sto/get-object storage (:media-id fmobject))]
-
-    (with-open [stream (sto/get-object-data storage sobject)]
-      (slurp stream))))
-
-(defn- create-shapes-for-svg
-  [{:keys [id] :as mobj} file-id objects frame-id position]
-  (let [svg-text  (get-svg-content id)
-
-        optimizer (::csvg/optimizer *system*)
-        svg-text  (csvg/optimize optimizer svg-text)
-
-        svg-data  (-> (csvg/parse svg-text)
-                      (assoc :name (:name mobj))
-                      (collect-and-persist-images file-id))]
-
-    (sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
-
-(defn- process-media-object
-  [fdata page-id frame-id mobj position]
-  (let [page    (ctpl/get-page fdata page-id)
-        file-id (get fdata :id)
-
-        [shape children]
-        (if (= (:mtype mobj) "image/svg+xml")
-          (create-shapes-for-svg mobj file-id (:objects page) frame-id position)
-          (create-shapes-for-bitmap mobj frame-id position))
-
-        shape (assoc shape :name (-> "Graphics"
-                                     (cfh/merge-path-item (:path mobj))
-                                     (cfh/merge-path-item (:name mobj))))
-
-        changes
-        (-> (fcb/empty-changes nil)
-            (fcb/set-save-undo? false)
-            (fcb/with-page page)
-            (fcb/with-objects (:objects page))
-            (fcb/with-library-data fdata)
-            (fcb/delete-media (:id mobj))
-            (fcb/add-objects (cons shape children)))
-
-        ;; NOTE: this is a workaround for `generate-add-component`, it
-        ;; is needed because that function always starts from empty
-        ;; changes; so in this case we need manually add all shapes to
-        ;; the page and then use that page for the
-        ;; `generate-add-component` function
-        page
-        (reduce (fn [page shape]
-                  (ctst/add-shape (:id shape)
-                                  shape
-                                  page
-                                  frame-id
-                                  frame-id
-                                  nil
-                                  true))
-                page
-                (cons shape children))
-
-        [_ _ changes2]
-        (cflh/generate-add-component nil
-                                     [shape]
-                                     (:objects page)
-                                     (:id page)
-                                     file-id
-                                     true
-                                     nil
-                                     cfsh/prepare-create-artboard-from-selection)
-        changes (fcb/concat-changes changes changes2)]
-
-    (:redo-changes changes)))
-
-(defn- create-media-grid
-  [fdata page-id frame-id grid media-group]
-  (let [factory  (px/thread-factory :virtual true)
-        executor (px/fixed-executor :parallelism 10 :factory factory)
-        process  (fn [mobj position]
-                   (let [position (gpt/add position (gpt/point grid-gap grid-gap))
-                         tp1      (dt/tpoint)]
-                     (try
-                       (process-media-object fdata page-id frame-id mobj position)
-                       (catch Throwable cause
-                         (l/wrn :hint "unable to process file media object (skiping)"
-                                :file-id (str (:id fdata))
-                                :id (str (:id mobj))
-                                :cause cause)
-
-                         (if-not *skip-on-error*
-                           (throw cause)
-                           nil))
-                       (finally
-                         (l/trc :hint "graphic processed"
-                                :file-id (str (:id fdata))
-                                :media-id (str (:id mobj))
-                                :elapsed (dt/format-duration (tp1)))))))]
-    (try
-      (->> (d/zip media-group grid)
-           (map (fn [[mobj position]]
-                  (sse/tap {:type :migration-progress
-                            :section :graphics
-                            :name (:name mobj)})
-                  (px/submit! executor (partial process mobj position))))
-           (reduce (fn [fdata promise]
-                     (if-let [changes (deref promise)]
-                       (-> (assoc-in fdata [:options :components-v2] true)
-                           (cp/process-changes changes false))
-                       fdata))
-                   fdata))
-      (finally
-        (pu/close! executor)))))
-
-(defn- migrate-graphics
-  [fdata]
-  (sse/tap {:type :migration-progress
-            :section :graphics})
-  (if (empty? (:media fdata))
-    fdata
-    (let [[fdata page-id start-pos]
-          (ctf/get-or-add-library-page fdata frame-gap)
-
-          media (->> (vals (:media fdata))
-                     (map (fn [{:keys [width height] :as media}]
-                            (let [points (-> (grc/make-rect 0 0 width height)
-                                             (grc/rect->points))]
-                              (assoc media :points points)))))
-
-          groups (get-asset-groups media "Graphics")]
-
-      (let [total (count media)]
-        (some-> *stats* (swap! update :processed/graphics (fnil + 0) total))
-        (some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total))
-        (some-> *file-stats* (swap! assoc :processed/graphics total)))
-
-      (loop [groups (seq groups)
-             fdata fdata
-             position start-pos]
-        (if (empty? groups)
-          fdata
-          (let [[group-name assets]    (first groups)
-                grid                   (ctst/generate-shape-grid assets position grid-gap)
-                {:keys [width height]} (meta grid)
-                frame                  (create-frame group-name position width height)
-                fdata                  (ctpl/update-page fdata
-                                                         page-id
-                                                         #(ctst/add-shape (:id frame)
-                                                                          frame
-                                                                          %
-                                                                          (:id frame)
-                                                                          (:id frame)
-                                                                          nil
-                                                                          true))]
-            (recur (next groups)
-                   (create-media-grid fdata page-id (:id frame) grid assets)
-                   (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap))))))))))
-
-(defn- migrate-fdata
-  [fdata libs]
-  (let [migrated? (dm/get-in fdata [:options :components-v2])]
-    (if migrated?
-      fdata
-      (let [fdata (migrate-components fdata libs)
-            fdata (migrate-graphics fdata)]
-        (update fdata :options assoc :components-v2 true)))))
-
-(defn- get-file
-  [system id]
-  (binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
-    (-> (files/get-file system id :migrate? false)
-        (update :data assoc :id id)
-        (update :data fdata/process-pointers deref)
-        (fmg/migrate-file))))
-
-(defn- validate-file!
-  [file libs throw-on-validate?]
-  (try
-    (cfv/validate-file! file libs)
-    (cfv/validate-file-schema! file)
-    (catch Throwable cause
-      (if throw-on-validate?
-        (throw cause)
-        (l/wrn :hint "migrate:file:validation-error" :cause cause)))))
-
-(defn- process-file
-  [{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}]
-  (let [file  (get-file system id)
-
-        libs  (->> (files/get-file-libraries conn id)
-                   (into [file] (comp (map :id) (map (partial get-file system))))
-                   (d/index-by :id))
-
-        file  (-> file
-                  (update :data migrate-fdata libs)
-                  (update :features conj "components/v2"))
-
-        _     (when validate?
-                (validate-file! file libs throw-on-validate?))
-
-        file (if (contains? (:features file) "fdata/objects-map")
-               (fdata/enable-objects-map file)
-               file)
-
-        file (if (contains? (:features file) "fdata/pointer-map")
-               (binding [pmap/*tracked* (pmap/create-tracked)]
-                 (let [file (fdata/enable-pointer-map file)]
-                   (fdata/persist-pointers! system id)
-                   file))
-               file)]
-
-    (db/update! conn :file
-                {:data (blob/encode (:data file))
-                 :features (db/create-array conn "text" (:features file))
-                 :revn (:revn file)}
-                {:id (:id file)}
-                {::db/return-keys? false})
-
-    (dissoc file :data)))
-
-(defn migrate-file!
-  [system file-id & {:keys [validate? throw-on-validate?]}]
-  (let [tpoint  (dt/tpoint)
-        file-id (if (string? file-id)
-                  (parse-uuid file-id)
-                  file-id)]
-    (binding [*file-stats* (atom {})]
-      (try
-        (l/dbg :hint "migrate:file:start" :file-id (str file-id))
-
-        (let [system (update system ::sto/storage media/configure-assets-storage)]
-          (db/tx-run! system
-                      (fn [system]
-                        (binding [*system* system]
-                          (fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
-                          (process-file system file-id
-                                        :validate? validate?
-                                        :throw-on-validate? throw-on-validate?)))))
-
-        (finally
-          (let [elapsed    (tpoint)
-                components (get @*file-stats* :processed/components 0)
-                graphics   (get @*file-stats* :processed/graphics 0)]
-
-            (l/dbg :hint "migrate:file:end"
-                   :file-id (str file-id)
-                   :graphics graphics
-                   :components components
-                   :elapsed (dt/format-duration elapsed))
-
-            (some-> *stats* (swap! update :processed/files (fnil inc 0)))
-            (some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
-
-(defn migrate-team!
-  [system team-id & {:keys [validate? throw-on-validate?]}]
-  (let [tpoint  (dt/tpoint)
-        team-id (if (string? team-id)
-                  (parse-uuid team-id)
-                  team-id)]
-    (l/dbg :hint "migrate:team:start" :team-id (dm/str team-id))
-    (binding [*team-stats* (atom {})]
-      (try
-        ;; We execute this out of transaction because we want this
-        ;; change to be visible to all other sessions before starting
-        ;; the migration
-        (let [sql (str "UPDATE team SET features = "
-                       "    array_append(features, 'ephimeral/v2-migration') "
-                       " WHERE id = ?")]
-          (db/exec-one! system [sql team-id]))
-
-        (db/tx-run! system
-                    (fn [{:keys [::db/conn] :as system}]
-                      ;; Lock the team
-                      (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
-
-                      (let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
-                                                            (update :features db/decode-pgarray #{}))]
-
-                        (if (contains? features "components/v2")
-                          (l/dbg :hint "team already migrated")
-                          (let [sql  (str/concat
-                                      "SELECT f.id FROM file AS f "
-                                      "  JOIN project AS p ON (p.id = f.project_id) "
-                                      "WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
-                                      "FOR UPDATE")]
-
-                            (doseq [file-id (->> (db/exec! conn [sql team-id])
-                                                 (map :id))]
-                              (migrate-file! system file-id
-                                             :validate? validate?
-                                             :throw-on-validate? throw-on-validate?))
-
-                            (let [features (-> features
-                                               (disj "ephimeral/v2-migration")
-                                               (conj "components/v2")
-                                               (conj "layout/grid")
-                                               (conj "styles/v2"))]
-                              (db/update! conn :team
-                                          {:features (db/create-array conn "text" features)}
-                                          {:id team-id})))))))
-        (finally
-          (some-> *semaphore* ps/release!)
-          (let [elapsed (tpoint)]
-            (some-> *stats* (swap! update :processed/teams (fnil inc 0)))
-
-            ;; We execute this out of transaction because we want this
-            ;; change to be visible to all other sessions before starting
-            ;; the migration
-            (let [sql (str "UPDATE team SET features = "
-                           "    array_remove(features, 'ephimeral/v2-migration') "
-                           " WHERE id = ?")]
-              (db/exec-one! system [sql team-id]))
-
-            (let [components (get @*team-stats* :processed/components 0)
-                  graphics   (get @*team-stats* :processed/graphics 0)
-                  files      (get @*team-stats* :processed/files 0)]
-              (l/dbg :hint "migrate:team:end"
-                     :team-id (dm/str team-id)
-                     :files files
-                     :components components
-                     :graphics graphics
-                     :elapsed (dt/format-duration elapsed)))))))))
-
-

backend/src/app/features/fdata.clj

@@ -1,96 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.features.fdata
-  "A `fdata/*` related feature migration helpers"
-  (:require
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.logging :as l]
-   [app.db :as db]
-   [app.util.blob :as blob]
-   [app.util.objects-map :as omap]
-   [app.util.pointer-map :as pmap]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; OBJECTS-MAP
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn enable-objects-map
-  [file]
-  (let [update-fn #(d/update-when % :objects omap/wrap)]
-    (-> file
-        (update :data (fn [fdata]
-                        (-> fdata
-                            (update :pages-index update-vals update-fn)
-                            (update :components update-vals update-fn))))
-        (update :features conj "fdata/objects-map"))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; POINTER-MAP
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn load-pointer
-  "A database loader pointer helper"
-  [system file-id id]
-  (let [{:keys [content]} (db/get system :file-data-fragment
-                                  {:id id :file-id file-id}
-                                  {::db/columns [:content]
-                                   ::db/check-deleted? false})]
-    (when-not content
-      (ex/raise :type :internal
-                :code :fragment-not-found
-                :hint "fragment not found"
-                :file-id file-id
-                :fragment-id id))
-
-    (blob/decode content)))
-
-(defn persist-pointers!
-  "Given a database connection and the final file-id, persist all
-  pointers to the underlying storage (the database)."
-  [system file-id]
-  (doseq [[id item] @pmap/*tracked*]
-    (when (pmap/modified? item)
-      (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
-      (let [content (-> item deref blob/encode)]
-        (db/insert! system :file-data-fragment
-                    {:id id
-                     :file-id file-id
-                     :content content})))))
-
-(defn process-pointers
-  "Apply a function to all pointers on the file. Usuly used for
-  dereference the pointer to a plain value before some processing."
-  [fdata update-fn]
-  (cond-> fdata
-    (contains? fdata :pages-index)
-    (update :pages-index process-pointers update-fn)
-
-    :always
-    (update-vals (fn [val]
-                   (if (pmap/pointer-map? val)
-                     (update-fn val)
-                     val)))))
-
-(defn get-used-pointer-ids
-  "Given a file, return all pointer ids used in the data."
-  [fdata]
-  (->> (concat (vals fdata)
-               (vals (:pages-index fdata)))
-       (into #{} (comp (filter pmap/pointer-map?)
-                       (map pmap/get-id)))))
-
-(defn enable-pointer-map
-  "Enable the fdata/pointer-map feature on the file."
-  [file]
-  (-> file
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (update :pages-index update-vals pmap/wrap)
-                          (update :components pmap/wrap))))
-
-      (update :features conj "fdata/pointer-map")))

backend/src/app/http.clj

@@ -23,14 +23,15 @@
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
    [app.rpc.doc :as-alias rpc.doc]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
-   [ring.request :as rreq]
-   [ring.response :as-alias rres]
-   [yetti.adapter :as yt]))
+   [yetti.adapter :as yt]
+   [yetti.request :as yrq]
+   [yetti.response :as-alias yrs]))
 
 (declare router-handler)
 
@@ -62,7 +63,8 @@
                 ::max-multipart-body-size
                 ::router
                 ::handler
-                ::io-threads]))
+                ::io-threads
+                ::wrk/executor]))
 
 (defmethod ig/init-key ::server
   [_ {:keys [::handler ::router ::host ::port] :as cfg}]
@@ -73,9 +75,11 @@
                  :http/max-multipart-body-size (::max-multipart-body-size cfg)
                  :xnio/io-threads (or (::io-threads cfg)
                                       (max 3 (px/get-available-processors)))
-                 :xnio/dispatch :virtual
-                 :ring/compat :ring2
-                 :socket/backlog 4069}
+                 :xnio/worker-threads (or (::worker-threads cfg)
+                                          (max 6 (px/get-available-processors)))
+                 :xnio/dispatch true
+                 :socket/backlog 4069
+                 :ring/async true}
 
         handler (cond
                   (some? router)
@@ -98,13 +102,13 @@
   (yt/stop! server))
 
 (defn- not-found-handler
-  [_]
-  {::rres/status 404})
+  [_ respond _]
+  (respond {::yrs/status 404}))
 
 (defn- router-handler
   [router]
   (letfn [(resolve-handler [request]
-            (if-let [match (r/match-by-path router (rreq/path request))]
+            (if-let [match (r/match-by-path router (yrq/path request))]
               (let [params  (:path-params match)
                     result  (:result match)
                     handler (or (:handler result) not-found-handler)
@@ -116,15 +120,18 @@
             (let [{:keys [body] :as response} (errors/handle cause request)]
               (cond-> response
                 (map? body)
-                (-> (update ::rres/headers assoc "content-type" "application/transit+json")
-                    (assoc ::rres/body (t/encode-str body {:type :json-verbose}))))))]
+                (-> (update ::yrs/headers assoc "content-type" "application/transit+json")
+                    (assoc ::yrs/body (t/encode-str body {:type :json-verbose}))))))]
 
-    (fn [request]
-      (let [handler (resolve-handler request)]
-        (try
-          (handler)
-          (catch Throwable cause
-            (on-error cause request)))))))
+    (fn [request respond _]
+      (let [handler  (resolve-handler request)
+            exchange (yrq/exchange request)]
+        (handler
+         (fn [response]
+           (yt/dispatch! exchange (partial respond response)))
+         (fn [cause]
+           (let [response (on-error cause request)]
+             (yt/dispatch! exchange (partial respond response)))))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HTTP ROUTER
@@ -153,7 +160,8 @@
                       [session/soft-auth cfg]
                       [actoken/soft-auth cfg]
                       [mw/errors errors/handle]
-                      [mw/restrict-methods]]}
+                      [mw/restrict-methods]
+                      [mw/with-dispatch :vthread]]}
 
      (::mtx/routes cfg)
      (::assets/routes cfg)

backend/src/app/http/access_token.clj

@@ -11,13 +11,13 @@
    [app.db :as db]
    [app.main :as-alias main]
    [app.tokens :as tokens]
-   [ring.request :as rreq]))
+   [yetti.request :as yrq]))
 
 (def header-re #"^Token\s+(.*)")
 
 (defn- get-token
   [request]
-  (some->> (rreq/get-header request "authorization")
+  (some->> (yrq/get-header request "authorization")
            (re-matches header-re)
            (second)))
 
@@ -30,7 +30,7 @@
   "SELECT perms, profile_id, expires_at
      FROM access_token
     WHERE id = ?
-      AND (expires_at IS NULL
+      AND (expires_at IS NULL 
            OR (expires_at > now()));")
 
 (defn- get-token-data
@@ -54,8 +54,9 @@
                 (l/trace :hint "exception on decoding malformed token" :cause cause)
                 request)))]
 
-    (fn [request]
-      (handler (handle-request request)))))
+    (fn [request respond raise]
+      (let [request (handle-request request)]
+        (handler request respond raise)))))
 
 (defn- wrap-authz
   "Authorization middleware, will be executed synchronously on vthread."

backend/src/app/http/assets.clj

@@ -16,7 +16,7 @@
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
-   [ring.response :as-alias rres]))
+   [yetti.response :as-alias yrs]))
 
 (def ^:private cache-max-age
   (dt/duration {:hours 24}))
@@ -37,11 +37,11 @@
 (defn- serve-object-from-s3
   [{:keys [::sto/storage] :as cfg} obj]
   (let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
-    {::rres/status  307
-     ::rres/headers {"location" (str url)
-                     "x-host"   (cond-> host port (str ":" port))
-                     "x-mtype"  (-> obj meta :content-type)
-                     "cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
+    {::yrs/status  307
+     ::yrs/headers {"location" (str url)
+                    "x-host"   (cond-> host port (str ":" port))
+                    "x-mtype"  (-> obj meta :content-type)
+                    "cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
 
 (defn- serve-object-from-fs
   [{:keys [::path]} obj]
@@ -51,8 +51,8 @@
         headers {"x-accel-redirect" (:path purl)
                  "content-type" (:content-type mdata)
                  "cache-control" (str "max-age=" (inst-ms cache-max-age))}]
-    {::rres/status 204
-     ::rres/headers headers}))
+    {::yrs/status 204
+     ::yrs/headers headers}))
 
 (defn- serve-object
   "Helper function that returns the appropriate response depending on
@@ -70,7 +70,7 @@
         obj (sto/get-object storage id)]
     (if obj
       (serve-object cfg obj)
-      {::rres/status 404})))
+      {::yrs/status 404})))
 
 (defn- generic-handler
   "A generic handler helper/common code for file-media based handlers."
@@ -81,7 +81,7 @@
         sobj (sto/get-object storage (kf mobj))]
     (if sobj
       (serve-object cfg sobj)
-      {::rres/status 404})))
+      {::yrs/status 404})))
 
 (defn file-objects-handler
   "Handler that serves storage objects by file media id."

backend/src/app/http/awsns.clj

@@ -20,8 +20,8 @@
    [integrant.core :as ig]
    [jsonista.core :as j]
    [promesa.exec :as px]
-   [ring.request :as rreq]
-   [ring.response :as-alias rres]))
+   [yetti.request :as yrq]
+   [yetti.response :as-alias yrs]))
 
 (declare parse-json)
 (declare handle-request)
@@ -31,14 +31,15 @@
 (defmethod ig/pre-init-spec ::routes [_]
   (s/keys :req [::http/client
                 ::main/props
-                ::db/pool]))
+                ::db/pool
+                ::wrk/executor]))
 
 (defmethod ig/init-key ::routes
-  [_ cfg]
+  [_ {:keys [::wrk/executor] :as cfg}]
   (letfn [(handler [request]
-            (let [data (-> request rreq/body slurp)]
-              (px/run! :vthread (partial handle-request cfg data)))
-            {::rres/status 200})]
+            (let [data (-> request yrq/body slurp)]
+              (px/run! executor #(handle-request cfg data)))
+            {::yrs/status 200})]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 

backend/src/app/http/client.clj

@@ -8,6 +8,7 @@
   "Http client abstraction layer."
   (:require
    [app.common.spec :as us]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [java-http-clj.core :as http]
@@ -20,11 +21,12 @@
   (s/keys :req [::client]))
 
 (defmethod ig/pre-init-spec ::client [_]
-  (s/keys :req []))
+  (s/keys :req [::wrk/executor]))
 
 (defmethod ig/init-key ::client
-  [_ _]
-  (http/build-client {:connect-timeout 30000 ;; 10s
+  [_ {:keys [::wrk/executor] :as cfg}]
+  (http/build-client {:executor executor
+                      :connect-timeout 30000 ;; 10s
                       :follow-redirects :always}))
 
 (defn send!

backend/src/app/http/debug.clj

@@ -7,7 +7,6 @@
 (ns app.http.debug
   (:refer-clojure :exclude [error-handler])
   (:require
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
@@ -15,12 +14,9 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http.session :as session]
-   [app.main :as-alias main]
-   [app.rpc.commands.auth :as auth]
    [app.rpc.commands.binfile :as binf]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
-   [app.srepl.helpers :as srepl]
    [app.storage :as-alias sto]
    [app.util.blob :as blob]
    [app.util.template :as tmpl]
@@ -32,41 +28,55 @@
    [integrant.core :as ig]
    [markdown.core :as md]
    [markdown.transformers :as mdt]
-   [ring.request :as rreq]
-   [ring.response :as rres]))
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
 ;; (selmer.parser/cache-off!)
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn authorized?
+  [pool {:keys [::session/profile-id]}]
+  (or (= "devenv" (cf/get :host))
+      (let [profile (ex/ignoring (profile/get-profile pool profile-id))
+            admins  (or (cf/get :admins) #{})]
+        (contains? admins (:email profile)))))
+
+(defn prepare-response
+  [body]
+  (let [headers {"content-type" "application/transit+json"}]
+    {::yrs/status 200
+     ::yrs/body body
+     ::yrs/headers headers}))
+
+(defn prepare-download-response
+  [body filename]
+  (let [headers {"content-disposition" (str "attachment; filename=" filename)
+                 "content-type" "application/octet-stream"}]
+    {::yrs/status 200
+     ::yrs/body body
+     ::yrs/headers headers}))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; INDEX
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn index-handler
-  [_cfg _request]
-  {::rres/status  200
-   ::rres/headers {"content-type" "text/html"}
-   ::rres/body    (-> (io/resource "app/templates/debug.tmpl")
-                      (tmpl/render {}))})
+  [{:keys [::db/pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+  {::yrs/status  200
+   ::yrs/headers {"content-type" "text/html"}
+   ::yrs/body    (-> (io/resource "app/templates/debug.tmpl")
+                     (tmpl/render {}))})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE CHANGES
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn prepare-response
-  [body]
-  (let [headers {"content-type" "application/transit+json"}]
-    {::rres/status 200
-     ::rres/body body
-     ::rres/headers headers}))
-
-(defn prepare-download-response
-  [body filename]
-  (let [headers {"content-disposition" (str "attachment; filename=" filename)
-                 "content-type" "application/octet-stream"}]
-    {::rres/status 200
-     ::rres/body body
-     ::rres/headers headers}))
-
 (def sql:retrieve-range-of-changes
   "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
 
@@ -75,6 +85,10 @@
 
 (defn- retrieve-file-data
   [{:keys [::db/pool]} {:keys [params ::session/profile-id] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
   (let [file-id  (some-> params :file-id parse-uuid)
         revn     (some-> params :revn parse-long)
         filename (str file-id)]
@@ -107,8 +121,8 @@
                           (db/update! conn :file
                                       {:data data}
                                       {:id file-id})
-                          {::rres/status 201
-                           ::rres/body "OK CREATED"})))
+                          {::yrs/status 201
+                           ::yrs/body "OK CREATED"})))
 
         :else
         (prepare-response (blob/decode data))))))
@@ -137,8 +151,8 @@
                         {:data data
                          :deleted-at nil}
                         {:id file-id})
-            {::rres/status 200
-             ::rres/body "OK UPDATED"})
+            {::yrs/status 200
+             ::yrs/body "OK UPDATED"})
 
           (db/run! pool (fn [{:keys [::db/conn]}]
                           (create-file conn {:id file-id
@@ -148,15 +162,15 @@
                           (db/update! conn :file
                                       {:data data}
                                       {:id file-id})
-                          {::rres/status 201
-                           ::rres/body "OK CREATED"}))))
+                          {::yrs/status 201
+                           ::yrs/body "OK CREATED"}))))
 
-      {::rres/status 500
-       ::rres/body "ERROR"})))
+      {::yrs/status 500
+       ::yrs/body "ERROR"})))
 
 (defn file-data-handler
   [cfg request]
-  (case (rreq/method request)
+  (case (yrq/method request)
     :get (retrieve-file-data cfg request)
     :post (upload-file-data cfg request)
     (ex/raise :type :http
@@ -164,6 +178,10 @@
 
 (defn file-changes-handler
   [{:keys [::db/pool]} {:keys [params] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
   (letfn [(retrieve-changes [file-id revn]
             (if (str/includes? revn ":")
               (let [[start end] (->> (str/split revn #":")
@@ -230,19 +248,24 @@
             (-> (io/resource "app/templates/error-report.v3.tmpl")
                 (tmpl/render (-> content
                                  (assoc :id id)
-                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))]
+                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))
+          ]
+
+    (when-not (authorized? pool request)
+      (ex/raise :type :authentication
+                :code :only-admins-allowed))
 
     (if-let [report (get-report request)]
       (let [result (case (:version report)
                      1 (render-template-v1 report)
                      2 (render-template-v2 report)
                      3 (render-template-v3 report))]
-        {::rres/status 200
-         ::rres/body result
-         ::rres/headers {"content-type" "text/html; charset=utf-8"
-                         "x-robots-tag" "noindex"}})
-      {::rres/status 404
-       ::rres/body "not found"})))
+        {::yrs/status 200
+         ::yrs/body result
+         ::yrs/headers {"content-type" "text/html; charset=utf-8"
+                        "x-robots-tag" "noindex"}})
+      {::yrs/status 404
+       ::yrs/body "not found"})))
 
 (def sql:error-reports
   "SELECT id, created_at,
@@ -252,14 +275,17 @@
     LIMIT 200")
 
 (defn error-list-handler
-  [{:keys [::db/pool]} _request]
+  [{:keys [::db/pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
   (let [items (->> (db/exec! pool [sql:error-reports])
                    (map #(update % :created-at dt/format-instant :rfc1123)))]
-    {::rres/status 200
-     ::rres/body (-> (io/resource "app/templates/error-list.tmpl")
-                     (tmpl/render {:items items}))
-     ::rres/headers {"content-type" "text/html; charset=utf-8"
-                     "x-robots-tag" "noindex"}}))
+    {::yrs/status 200
+     ::yrs/body (-> (io/resource "app/templates/error-list.tmpl")
+                    (tmpl/render {:items items}))
+     ::yrs/headers {"content-type" "text/html; charset=utf-8"
+                    "x-robots-tag" "noindex"}}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; EXPORT/IMPORT
@@ -295,14 +321,14 @@
                   ::binf/profile-id profile-id
                   ::binf/project-id project-id))
 
-          {::rres/status  200
-           ::rres/headers {"content-type" "text/plain"}
-           ::rres/body    "OK CLONED"})
+          {::yrs/status  200
+           ::yrs/headers {"content-type" "text/plain"}
+           ::yrs/body    "OK CLONED"})
 
-        {::rres/status  200
-         ::rres/body    (io/input-stream path)
-         ::rres/headers {"content-type" "application/octet-stream"
-                         "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
+        {::yrs/status  200
+         ::yrs/body    (io/input-stream path)
+         ::yrs/headers {"content-type" "application/octet-stream"
+                        "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
 
 
 
@@ -333,96 +359,9 @@
             ::binf/profile-id profile-id
             ::binf/project-id project-id))
 
-    {::rres/status  200
-     ::rres/headers {"content-type" "text/plain"}
-     ::rres/body    "OK"}))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ACTIONS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- resend-email-notification
-  [{:keys [::db/pool ::main/props] :as cfg} {:keys [params] :as request}]
-
-  (when-not (contains? params :force)
-    (ex/raise :type :validation
-              :code :missing-force
-              :hint "missing force checkbox"))
-
-  (let [profile  (some->> params :email (profile/get-profile-by-email pool))]
-
-    (when-not profile
-      (ex/raise :type :validation
-                :code :missing-profile
-                :hint "unable to find profile by email"))
-
-    (cond
-      (contains? params :block)
-      (do
-        (db/update! pool :profile {:is-blocked true} {:id (:id profile)})
-        (db/delete! pool :http-session {:profile-id (:id profile)})
-
-        {::rres/status  200
-         ::rres/headers {"content-type" "text/plain"}
-         ::rres/body    (str/ffmt "PROFILE '%' BLOCKED" (:email profile))})
-
-      (contains? params :unblock)
-      (do
-        (db/update! pool :profile {:is-blocked false} {:id (:id profile)})
-        {::rres/status  200
-         ::rres/headers {"content-type" "text/plain"}
-         ::rres/body    (str/ffmt "PROFILE '%' UNBLOCKED" (:email profile))})
-
-      (contains? params :resend)
-      (if (:is-blocked profile)
-        {::rres/status  200
-         ::rres/headers {"content-type" "text/plain"}
-         ::rres/body    "PROFILE ALREADY BLOCKED"}
-        (do
-          (auth/send-email-verification! pool props profile)
-          {::rres/status  200
-           ::rres/headers {"content-type" "text/plain"}
-           ::rres/body    (str/ffmt "RESENDED FOR '%'" (:email profile))}))
-
-      :else
-      (do
-        (db/update! pool :profile {:is-active true} {:id (:id profile)})
-        {::rres/status  200
-         ::rres/headers {"content-type" "text/plain"}
-         ::rres/body    (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))
-
-
-(defn- reset-file-data-version
-  [cfg {:keys [params] :as request}]
-  (let [file-id (some-> params :file-id d/parse-uuid)
-        version (some-> params :version d/parse-integer)]
-
-    (when-not (contains? params :force)
-      (ex/raise :type :validation
-                :code :missing-force
-                :hint "missing force checkbox"))
-
-    (when (nil? file-id)
-      (ex/raise :type :validation
-                :code :invalid-file-id
-                :hint "provided invalid file id"))
-
-    (when (nil? version)
-      (ex/raise :type :validation
-                :code :invalid-version
-                :hint "provided invalid version"))
-
-    (srepl/update-file! cfg
-                        :id file-id
-                        :update-fn (fn [file]
-                                     (update file :data assoc :version version))
-                        :migrate? false
-                        :inc-revn? false
-                        :save? true)
-    {::rres/status  200
-     ::rres/headers {"content-type" "text/plain"}
-     ::rres/body    "OK"}))
-
+    {::yrs/status  200
+     ::yrs/headers {"content-type" "text/plain"}
+     ::yrs/body    "OK"}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
@@ -433,13 +372,13 @@
   [{:keys [::db/pool]} _]
   (try
     (db/exec-one! pool ["select count(*) as count from server_prop;"])
-    {::rres/status 200
-     ::rres/body "OK"}
+    {::yrs/status 200
+     ::yrs/body "OK"}
     (catch Throwable cause
       (l/warn :hint "unable to execute query on health handler"
               :cause cause)
-      {::rres/status 503
-       ::rres/body "KO"})))
+      {::yrs/status 503
+       ::yrs/body "KO"})))
 
 (defn changelog-handler
   [_ _]
@@ -448,23 +387,16 @@
           (md->html [text]
             (md/md-to-html-string text :replacement-transformers (into [transform-emoji] mdt/transformer-vector)))]
     (if-let [clog (io/resource "changelog.md")]
-      {::rres/status 200
-       ::rres/headers {"content-type" "text/html; charset=utf-8"}
-       ::rres/body (-> clog slurp md->html)}
-      {::rres/status 404
-       ::rres/body "NOT FOUND"})))
+      {::yrs/status 200
+       ::yrs/headers {"content-type" "text/html; charset=utf-8"}
+       ::yrs/body (-> clog slurp md->html)}
+      {::yrs/status 404
+       ::yrs/body "NOT FOUND"})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; INIT
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn authorized?
-  [pool {:keys [::session/profile-id]}]
-  (or (= "devenv" (cf/get :host))
-      (let [profile (ex/ignoring (profile/get-profile pool profile-id))
-            admins  (or (cf/get :admins) #{})]
-        (contains? admins (:email profile)))))
-
 (def with-authorization
   {:compile
    (fn [& _]
@@ -488,10 +420,6 @@
     ["/changelog" {:handler (partial changelog-handler cfg)}]
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
-    ["/actions/resend-email-verification"
-     {:handler (partial resend-email-notification cfg)}]
-    ["/actions/reset-file-data-version"
-     {:handler (partial reset-file-data-version cfg)}]
     ["/file/export" {:handler (partial export-handler cfg)}]
     ["/file/import" {:handler (partial import-handler cfg)}]
     ["/file/data" {:handler (partial file-data-handler cfg)}]

backend/src/app/http/errors.clj

@@ -9,21 +9,21 @@
   (:require
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
-   [app.common.schema :as-alias sm]
+   [app.common.schema :as sm]
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
    [app.http.session :as-alias session]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [ring.request :as rreq]
-   [ring.response :as rres]))
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
 (defn- parse-client-ip
   [request]
-  (or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
-      (rreq/get-header request "x-real-ip")
-      (rreq/remote-addr request)))
+  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
+      (yrq/get-header request "x-real-ip")
+      (yrq/remote-addr request)))
 
 (defn request->context
   "Extracts error report relevant context data from request."
@@ -34,206 +34,184 @@
     {:request/path       (:path request)
      :request/method     (:method request)
      :request/params     (:params request)
-     :request/user-agent (rreq/get-header request "user-agent")
+     :request/user-agent (yrq/get-header request "user-agent")
      :request/ip-addr    (parse-client-ip request)
      :request/profile-id (:uid claims)
-     :version/frontend   (or (rreq/get-header request "x-frontend-version") "unknown")
+     :version/frontend   (or (yrq/get-header request "x-frontend-version") "unknown")
      :version/backend    (:full cf/version)}))
 
-(defmulti handle-error
-  (fn [cause _ _]
-    (-> cause ex-data :type)))
-
 (defmulti handle-exception
-  (fn [cause _ _]
-    (class cause)))
+  (fn [err & _rest]
+    (let [edata (ex-data err)]
+      (or (:type edata)
+          (class err)))))
 
-(defmethod handle-error :authentication
-  [err _ _]
-  {::rres/status 401
-   ::rres/body (ex-data err)})
+(defmethod handle-exception :authentication
+  [err _]
+  {::yrs/status 401
+   ::yrs/body (ex-data err)})
 
-(defmethod handle-error :authorization
-  [err _ _]
-  {::rres/status 403
-   ::rres/body (ex-data err)})
+(defmethod handle-exception :authorization
+  [err _]
+  {::yrs/status 403
+   ::yrs/body (ex-data err)})
 
-(defmethod handle-error :restriction
-  [err _ _]
-  {::rres/status 400
-   ::rres/body (ex-data err)})
+(defmethod handle-exception :restriction
+  [err _]
+  {::yrs/status 400
+   ::yrs/body (ex-data err)})
 
-(defmethod handle-error :rate-limit
-  [err _ _]
+(defmethod handle-exception :rate-limit
+  [err _]
   (let [headers (-> err ex-data ::http/headers)]
-    {::rres/status 429
-     ::rres/headers headers}))
+    {::yrs/status 429
+     ::yrs/headers headers}))
 
-(defmethod handle-error :concurrency-limit
-  [err _ _]
+(defmethod handle-exception :concurrency-limit
+  [err _]
   (let [headers (-> err ex-data ::http/headers)]
-    {::rres/status 429
-     ::rres/headers headers}))
+    {::yrs/status 429
+     ::yrs/headers headers}))
 
-(defmethod handle-error :validation
-  [err request parent-cause]
+(defmethod handle-exception :validation
+  [err request]
   (let [{:keys [code] :as data} (ex-data err)]
     (cond
-      (or (= code :spec-validation)
-          (= code :params-validation)
-          (= code :schema-validation)
-          (= code :data-validation))
+      (= code :spec-validation)
       (let [explain (ex/explain data)]
-        {::rres/status 400
-         ::rres/body   (-> data
-                           (dissoc ::s/problems ::s/value ::s/spec ::sm/explain)
-                           (cond-> explain (assoc :explain explain)))})
+        {::yrs/status 400
+         ::yrs/body   (-> data
+                          (dissoc ::s/problems ::s/value ::s/spec)
+                          (cond-> explain (assoc :explain explain)))})
+
+      (= code :params-validation)
+      (let [explain (::sm/explain data)
+            payload (sm/humanize-data explain)]
+        {::yrs/status 400
+         ::yrs/body   (-> data
+                          (dissoc ::sm/explain)
+                          (assoc :data payload))})
 
       (= code :request-body-too-large)
-      {::rres/status 413 ::rres/body data}
+      {::yrs/status 413 ::yrs/body data}
 
       (= code :invalid-image)
       (binding [l/*context* (request->context request)]
-        (let [cause (or parent-cause err)]
-          (l/error :hint "unexpected error on processing image" :cause cause)
-          {::rres/status 400 ::rres/body data}))
+        (l/error :hint "unexpected error on processing image" :cause err)
+        {::yrs/status 400 ::yrs/body data})
 
       :else
-      {::rres/status 400 ::rres/body data})))
+      {::yrs/status 400 ::yrs/body data})))
 
-(defmethod handle-error :assertion
-  [error request parent-cause]
+(defmethod handle-exception :assertion
+  [error request]
   (binding [l/*context* (request->context request)]
-    (let [{:keys [code] :as data} (ex-data error)
-          cause (or parent-cause error)]
+    (let [{:keys [code] :as data} (ex-data error)]
       (cond
         (= code :data-validation)
-        (let [explain (ex/explain data)]
-          (l/error :hint "data assertion error" :cause cause)
-          {::rres/status 500
-           ::rres/body   {:type :server-error
-                          :code :assertion
-                          :data (-> data
-                                    (dissoc ::sm/explain)
-                                    (cond-> explain (assoc :explain explain)))}})
+        (let [explain (::sm/explain data)
+              payload (sm/humanize-data explain)]
+          (l/error :hint "Data assertion error" :message (ex-message error) :cause error)
+          {::yrs/status 500
+           ::yrs/body   {:type :server-error
+                         :code :assertion
+                         :data (-> data
+                                   (dissoc ::sm/explain)
+                                   (assoc :data payload))}})
 
         (= code :spec-validation)
         (let [explain (ex/explain data)]
-          (l/error :hint "spec assertion error" :cause cause)
-          {::rres/status 500
-           ::rres/body   {:type :server-error
-                          :code :assertion
-                          :data (-> data
-                                    (dissoc ::s/problems ::s/value ::s/spec)
-                                    (cond-> explain (assoc :explain explain)))}})
+          (l/error :hint "Spec assertion error" :message (ex-message error) :cause error)
+          {::yrs/status 500
+           ::yrs/body   {:type :server-error
+                         :code :assertion
+                         :data (-> data
+                                   (dissoc ::s/problems ::s/value ::s/spec)
+                                   (cond-> explain (assoc :explain explain)))}})
 
         :else
         (do
-          (l/error :hint "assertion error" :cause cause)
-          {::rres/status 500
-           ::rres/body   {:type :server-error
-                          :code :assertion
-                          :data data}})))))
+          (l/error :hint "Assertion error" :message (ex-message error) :cause error)
+          {::yrs/status 500
+           ::yrs/body   {:type :server-error
+                         :code :assertion
+                         :data data}})))))
 
-(defmethod handle-error :not-found
-  [err _ _]
-  {::rres/status 404
-   ::rres/body (ex-data err)})
 
-(defmethod handle-error :internal
-  [error request parent-cause]
+(defmethod handle-exception :not-found
+  [err _]
+  {::yrs/status 404
+   ::yrs/body (ex-data err)})
+
+(defmethod handle-exception :internal
+  [error request]
   (binding [l/*context* (request->context request)]
-    (let [cause (or parent-cause error)]
-      (l/error :hint "internal error" :cause cause)
-      {::rres/status 500
-       ::rres/body {:type :server-error
-                    :code :unhandled
-                    :hint (ex-message error)
-                    :data (ex-data error)}})))
-
-(defmethod handle-error :default
-  [error request parent-cause]
-  (let [edata (ex-data error)]
-    ;; This is a special case for the idle-in-transaction error;
-    ;; when it happens, the connection is automatically closed and
-    ;; next-jdbc combines the two errors in a single ex-info. We
-    ;; only need the :handling error, because the :rollback error
-    ;; will be always "connection closed".
-    (if (and (ex/exception? (:rollback edata))
-             (ex/exception? (:handling edata)))
-      (handle-exception (:handling edata) request error)
-      (handle-exception error request parent-cause))))
+    (l/error :hint "Internal error" :message (ex-message error) :cause error)
+    {::yrs/status 500
+     ::yrs/body {:type :server-error
+                 :code :unhandled
+                 :hint (ex-message error)
+                 :data (ex-data error)}}))
 
 (defmethod handle-exception org.postgresql.util.PSQLException
-  [error request parent-cause]
-  (let [state (.getSQLState ^java.sql.SQLException error)
-        cause (or parent-cause error)]
+  [error request]
+  (let [state (.getSQLState ^java.sql.SQLException error)]
     (binding [l/*context* (request->context request)]
-      (l/error :hint "PSQL error"
-               :cause cause)
+      (l/error :hint "PSQL error" :message (ex-message error) :cause error)
       (cond
         (= state "57014")
-        {::rres/status 504
-         ::rres/body {:type :server-error
-                      :code :statement-timeout
-                      :hint (ex-message error)}}
+        {::yrs/status 504
+         ::yrs/body {:type :server-error
+                     :code :statement-timeout
+                     :hint (ex-message error)}}
 
         (= state "25P03")
-        {::rres/status 504
-         ::rres/body {:type :server-error
-                      :code :idle-in-transaction-timeout
-                      :hint (ex-message error)}}
+        {::yrs/status 504
+         ::yrs/body {:type :server-error
+                     :code :idle-in-transaction-timeout
+                     :hint (ex-message error)}}
 
         :else
-        {::rres/status 500
-         ::rres/body {:type :server-error
-                      :code :unexpected
-                      :hint (ex-message error)
-                      :state state}}))))
+        {::yrs/status 500
+         ::yrs/body {:type :server-error
+                     :code :unexpected
+                     :hint (ex-message error)
+                     :state state}}))))
 
 (defmethod handle-exception :default
-  [error request parent-cause]
-  (let [edata (ex-data error)
-        cause (or parent-cause error)]
+  [error request]
+  (let [edata (ex-data error)]
     (cond
       ;; This means that exception is not a controlled exception.
       (nil? edata)
       (binding [l/*context* (request->context request)]
-        (l/error :hint "unexpected error" :cause cause)
-        {::rres/status 500
-         ::rres/body {:type :server-error
-                      :code :unexpected
-                      :hint (ex-message error)}})
+        (l/error :hint "Unexpected error" :message (ex-message error) :cause error)
+        {::yrs/status 500
+         ::yrs/body {:type :server-error
+                     :code :unexpected
+                     :hint (ex-message error)}})
+
+      ;; This is a special case for the idle-in-transaction error;
+      ;; when it happens, the connection is automatically closed and
+      ;; next-jdbc combines the two errors in a single ex-info. We
+      ;; only need the :handling error, because the :rollback error
+      ;; will be always "connection closed".
+      (and (ex/exception? (:rollback edata))
+           (ex/exception? (:handling edata)))
+      (handle-exception (:handling edata) request)
 
       :else
       (binding [l/*context* (request->context request)]
-        (l/error :hint "unhandled error" :cause cause)
-        {::rres/status 500
-         ::rres/body {:type :server-error
-                      :code :unhandled
-                      :hint (ex-message error)
-                      :data edata}}))))
-
-(defmethod handle-exception java.util.concurrent.CompletionException
-  [cause request _]
-  (let [cause' (ex-cause cause)]
-    (if (ex/error? cause')
-      (handle-error cause' request cause)
-      (handle-exception cause' request cause))))
-
-(defmethod handle-exception java.util.concurrent.ExecutionException
-  [cause request _]
-  (let [cause' (ex-cause cause)]
-    (if (ex/error? cause')
-      (handle-error cause' request cause)
-      (handle-exception cause' request cause))))
+        (l/error :hint "Unhandled error" :message (ex-message error) :cause error)
+        {::yrs/status 500
+         ::yrs/body {:type :server-error
+                     :code :unhandled
+                     :hint (ex-message error)
+                     :data edata}}))))
 
 (defn handle
   [cause request]
-  (if (ex/error? cause)
-    (handle-error cause request nil)
-    (handle-exception cause request nil)))
-
-(defn handle'
-  [cause request]
-  (::rres/body (handle cause request)))
+  (if (or (instance? java.util.concurrent.CompletionException cause)
+          (instance? java.util.concurrent.ExecutionException cause))
+    (handle-exception (ex-cause cause) request)
+    (handle-exception cause request)))

backend/src/app/http/middleware.clj

@@ -12,10 +12,13 @@
    [app.config :as cf]
    [app.util.json :as json]
    [cuerdas.core :as str]
-   [ring.request :as rreq]
-   [ring.response :as rres]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [promesa.util :as pu]
    [yetti.adapter :as yt]
-   [yetti.middleware :as ymw])
+   [yetti.middleware :as ymw]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs])
   (:import
    com.fasterxml.jackson.core.JsonParseException
    com.fasterxml.jackson.core.io.JsonEOFException
@@ -43,17 +46,17 @@
 (defn wrap-parse-request
   [handler]
   (letfn [(process-request [request]
-            (let [header (rreq/get-header request "content-type")]
+            (let [header (yrq/get-header request "content-type")]
               (cond
                 (str/starts-with? header "application/transit+json")
-                (with-open [^InputStream is (rreq/body request)]
+                (with-open [^InputStream is (yrq/body request)]
                   (let [params (t/read! (t/reader is))]
                     (-> request
                         (assoc :body-params params)
                         (update :params merge params))))
 
                 (str/starts-with? header "application/json")
-                (with-open [^InputStream is (rreq/body request)]
+                (with-open [^InputStream is (yrq/body request)]
                   (let [params (json/decode is json-mapper)]
                     (-> request
                         (assoc :body-params params)
@@ -62,36 +65,37 @@
                 :else
                 request)))
 
-          (handle-error [cause]
+          (handle-error [raise cause]
             (cond
               (instance? RuntimeException cause)
               (if-let [cause (ex-cause cause)]
-                (handle-error cause)
-                (throw cause))
+                (handle-error raise cause)
+                (raise cause))
 
               (instance? RequestTooBigException cause)
-              (ex/raise :type :validation
-                        :code :request-body-too-large
-                        :hint (ex-message cause))
+              (raise (ex/error :type :validation
+                               :code :request-body-too-large
+                               :hint (ex-message cause)))
+
 
               (or (instance? JsonEOFException cause)
                   (instance? JsonParseException cause)
                   (instance? MismatchedInputException cause))
-              (ex/raise :type :validation
-                        :code :malformed-json
-                        :hint (ex-message cause)
-                        :cause cause)
+              (raise (ex/error :type :validation
+                               :code :malformed-json
+                               :hint (ex-message cause)
+                               :cause cause))
 
               :else
-              (throw cause)))]
+              (raise cause)))]
 
-    (fn [request]
-      (if (= (rreq/method request) :post)
+    (fn [request respond raise]
+      (if (= (yrq/method request) :post)
         (let [request (ex/try! (process-request request))]
           (if (ex/exception? request)
-            (handle-error request)
-            (handler request)))
-        (handler request)))))
+            (handle-error raise request)
+            (handler request respond raise)))
+        (handler request respond raise)))))
 
 (def parse-request
   {:name ::parse-request
@@ -109,7 +113,7 @@
 (defn wrap-format-response
   [handler]
   (letfn [(transit-streamable-body [data opts]
-            (reify rres/StreamableResponseBody
+            (reify yrs/StreamableResponseBody
               (-write-body-to-stream [_ _ output-stream]
                 (try
                   (with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
@@ -124,7 +128,7 @@
                     (.close ^OutputStream output-stream))))))
 
           (json-streamable-body [data]
-            (reify rres/StreamableResponseBody
+            (reify yrs/StreamableResponseBody
               (-write-body-to-stream [_ _ output-stream]
                 (try
                   (with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
@@ -139,24 +143,24 @@
                     (.close ^OutputStream output-stream))))))
 
           (format-response-with-json [response _]
-            (let [body (::rres/body response)]
+            (let [body (::yrs/body response)]
               (if (or (boolean? body) (coll? body))
                 (-> response
-                    (update ::rres/headers assoc "content-type" "application/json")
-                    (assoc ::rres/body (json-streamable-body body)))
+                    (update ::yrs/headers assoc "content-type" "application/json")
+                    (assoc ::yrs/body (json-streamable-body body)))
                 response)))
 
           (format-response-with-transit [response request]
-            (let [body (::rres/body response)]
+            (let [body (::yrs/body response)]
               (if (or (boolean? body) (coll? body))
-                (let [qs   (rreq/query request)
+                (let [qs   (yrq/query request)
                       opts (if (or (contains? cf/flags :transit-readable-response)
                                    (str/includes? qs "transit_verbose"))
                              {:type :json-verbose}
                              {:type :json})]
                   (-> response
-                      (update ::rres/headers assoc "content-type" "application/transit+json")
-                      (assoc ::rres/body (transit-streamable-body body opts))))
+                      (update ::yrs/headers assoc "content-type" "application/transit+json")
+                      (assoc ::yrs/body (transit-streamable-body body opts))))
                 response)))
 
           (format-from-params [{:keys [query-params] :as request}]
@@ -165,7 +169,7 @@
 
           (format-response [response request]
             (let [accept (or (format-from-params request)
-                             (rreq/get-header request "accept"))]
+                             (yrq/get-header request "accept"))]
               (cond
                 (or (= accept "application/transit+json")
                     (str/includes? accept "application/transit+json"))
@@ -182,9 +186,11 @@
             (cond-> response
               (map? response) (format-response request)))]
 
-    (fn [request]
-      (let [response (handler request)]
-        (process-response response request)))))
+    (fn [request respond raise]
+      (handler request
+               (fn [response]
+                 (respond (process-response response request)))
+               raise))))
 
 (def format-response
   {:name ::format-response
@@ -192,11 +198,12 @@
 
 (defn wrap-errors
   [handler on-error]
-  (fn [request]
-    (try
-      (handler request)
-      (catch Throwable cause
-        (on-error cause request)))))
+  (fn [request respond raise]
+    (handler request respond (fn [cause]
+                               (try
+                                 (respond (on-error cause request))
+                                 (catch Throwable cause
+                                   (raise cause)))))))
 
 (def errors
   {:name ::errors
@@ -214,11 +221,11 @@
 (defn wrap-cors
   [handler]
   (fn [request]
-    (let [response (if (= (rreq/method request) :options)
-                     {::rres/status 200}
+    (let [response (if (= (yrq/method request) :options)
+                     {::yrs/status 200}
                      (handler request))
-          origin   (rreq/get-header request "origin")]
-      (update response ::rres/headers with-cors-headers origin))))
+          origin   (yrq/get-header request "origin")]
+      (update response ::yrs/headers with-cors-headers origin))))
 
 (def cors
   {:name ::cors
@@ -232,8 +239,18 @@
    (fn [data _]
      (when-let [allowed (:allowed-methods data)]
        (fn [handler]
-         (fn [request]
-           (let [method (rreq/method request)]
+         (fn [request respond raise]
+           (let [method (yrq/method request)]
              (if (contains? allowed method)
-               (handler request)
-               {::rres/status 405}))))))})
+               (handler request respond raise)
+               (respond {::yrs/status 405})))))))})
+
+(def with-dispatch
+  {:name ::with-dispatch
+   :compile
+   (fn [& _]
+     (fn [handler executor]
+       (let [executor (px/resolve-executor executor)]
+         (fn [request respond raise]
+           (->> (px/submit! executor (partial handler request))
+                (p/fnly (pu/handler respond raise)))))))})

backend/src/app/http/session.clj

@@ -20,7 +20,6 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [ring.request :as rreq]
    [yetti.request :as yrq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -143,7 +142,7 @@
   (us/assert! ::us/uuid profile-id)
 
   (fn [request response]
-    (let [uagent  (rreq/get-header request "user-agent")
+    (let [uagent  (yrq/get-header request "user-agent")
           params  {:profile-id profile-id
                    :user-agent uagent
                    :created-at (dt/now)}
@@ -210,8 +209,9 @@
                 (l/trace :hint "exception on decoding malformed token" :cause cause)
                 request)))]
 
-    (fn [request]
-      (handler (handle-request request)))))
+    (fn [request respond raise]
+      (let [request (handle-request request)]
+        (handler request respond raise)))))
 
 (defn- wrap-authz
   [handler {:keys [::manager]}]
@@ -221,15 +221,12 @@
           request  (cond-> request
                      (some? session)
                      (assoc ::profile-id (:profile-id session)
-                            ::id (:id session)))
-          response (handler request)]
+                            ::id (:id session)))]
 
-      (if (renew-session? session)
-        (let [session (update! manager session)]
-          (-> response
-              (assign-auth-token-cookie session)
-              (assign-authenticated-cookie session)))
-        response))))
+      (cond-> (handler request)
+        (renew-session? session)
+        (-> (assign-auth-token-cookie session)
+            (assign-authenticated-cookie session))))))
 
 (def soft-auth
   {:name ::soft-auth

backend/src/app/http/sse.clj

@@ -1,86 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.sse
-  "SSE (server sent events) helpers"
-  (:refer-clojure :exclude [tap])
-  (:require
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.logging :as l]
-   [app.common.transit :as t]
-   [app.http.errors :as errors]
-   [promesa.core :as p]
-   [promesa.exec :as px]
-   [promesa.exec.csp :as sp]
-   [promesa.util :as pu]
-   [ring.response :as rres])
-  (:import
-   java.io.OutputStream))
-
-(def ^:dynamic *channel* nil)
-
-(defn- write!
-  [^OutputStream output ^bytes  data]
-  (l/trc :hint "writting data" :data data :length (alength data))
-  (.write output data)
-  (.flush output))
-
-(defn- create-writer-loop
-  [^OutputStream output]
-  (try
-    (loop []
-      (when-let [event (sp/take! *channel*)]
-        (let [result (ex/try! (write! output event))]
-          (if (ex/exception? result)
-            (l/wrn :hint "unexpected exception on sse writer" :cause result)
-            (recur)))))
-    (finally
-      (pu/close! output))))
-
-(defn- encode
-  [[name data]]
-  (try
-    (let [data (with-out-str
-                 (println "event:" (d/name name))
-                 (println "data:" (t/encode-str data {:type :json-verbose}))
-                 (println))]
-      (.getBytes data "UTF-8"))
-    (catch Throwable cause
-      (l/err :hint "unexpected error on encoding value on sse stream"
-             :cause cause)
-      nil)))
-
-;; ---- PUBLIC API
-
-(def default-headers
-  {"Content-Type" "text/event-stream;charset=UTF-8"
-   "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
-   "Pragma" "no-cache"})
-
-(defn tap
-  ([data] (tap "event" data))
-  ([name data]
-   (when-let [channel *channel*]
-     (sp/put! channel [name data])
-     nil)))
-
-(defn response
-  [handler & {:keys [buf] :or {buf 32} :as opts}]
-  (fn [request]
-    {::rres/headers default-headers
-     ::rres/status 200
-     ::rres/body (reify rres/StreamableResponseBody
-                   (-write-body-to-stream [_ _ output]
-                     (binding [*channel* (sp/chan :buf buf :xf (keep encode))]
-                       (let [writer (px/run! :virtual (partial create-writer-loop output))]
-                         (try
-                           (tap "end" (handler))
-                           (catch Throwable cause
-                             (tap "error" (errors/handle' cause request)))
-                           (finally
-                             (sp/close! *channel*)
-                             (p/await! writer)))))))}))

backend/src/app/http/websocket.clj

@@ -10,7 +10,7 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
-   [app.common.schema :as sm]
+   [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.http.session :as session]
@@ -21,7 +21,6 @@
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.exec.csp :as sp]
-   [ring.websocket :as rws]
    [yetti.websocket :as yws]))
 
 (def recv-labels
@@ -117,21 +116,21 @@
               :profile-id profile-id
               :session-id session-id}]
 
-    ;; Close profile subscription if exists
-    (when-let [ch (:channel psub)]
-      (sp/close! ch)
-      (mbus/purge! msgbus [ch]))
+      ;; Close profile subscription if exists
+      (when-let [ch (:channel psub)]
+        (sp/close! ch)
+        (mbus/purge! msgbus [ch]))
 
-    ;; Close team subscription if exists
-    (when-let [ch (:channel tsub)]
-      (sp/close! ch)
-      (mbus/purge! msgbus [ch]))
+      ;; Close team subscription if exists
+      (when-let [ch (:channel tsub)]
+        (sp/close! ch)
+        (mbus/purge! msgbus [ch]))
 
-    ;; Close file subscription if exists
-    (when-let [{:keys [topic channel]} fsub]
-      (sp/close! channel)
-      (mbus/purge! msgbus [channel])
-      (mbus/pub! msgbus :topic topic :message msg))))
+      ;; Close file subscription if exists
+      (when-let [{:keys [topic channel]} fsub]
+        (sp/close! channel)
+        (mbus/purge! msgbus [channel])
+        (mbus/pub! msgbus :topic topic :message msg))))
 
 (defmethod handle-message :subscribe-team
   [{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id]} {:keys [team-id] :as params}]
@@ -179,7 +178,7 @@
           (let [message {:type :presence
                          :file-id file-id
                          :session-id session-id
-                         :profile-id profile-id}]
+                           :profile-id profile-id}]
             (mbus/pub! msgbus
                        :topic file-id
                        :message message)))
@@ -278,23 +277,19 @@
             :inc 1)
   message)
 
-(def ^:private schema:params
-  (sm/define
-    [:map {:title "params"}
-     [:session-id ::sm/uuid]]))
+
+(s/def ::session-id ::us/uuid)
+(s/def ::handler-params
+  (s/keys :req-un [::session-id]))
 
 (defn- http-handler
   [cfg {:keys [params ::session/profile-id] :as request}]
-  (let [{:keys [session-id]} (sm/conform! schema:params params)]
+  (let [{:keys [session-id]} (us/conform ::handler-params params)]
     (cond
       (not profile-id)
       (ex/raise :type :authentication
                 :hint "Authentication required.")
 
-      ;; WORKAROUND: we use the adapter specific predicate for
-      ;; performance reasons; for now, the ring default impl for
-      ;; `upgrade-request?` parses all requests headers before perform
-      ;; any checking.
       (not (yws/upgrade-request? request))
       (ex/raise :type :validation
                 :code :websocket-request-expected
@@ -303,13 +298,14 @@
       :else
       (do
         (l/trace :hint "websocket request" :profile-id profile-id :session-id session-id)
-        {::rws/listener (ws/listener request
-                                     ::ws/on-rcv-message (partial on-rcv-message cfg)
-                                     ::ws/on-snd-message (partial on-snd-message cfg)
-                                     ::ws/on-connect (partial on-connect cfg)
-                                     ::ws/handler (partial handle-message cfg)
-                                     ::profile-id profile-id
-                                     ::session-id session-id)}))))
+        (->> (ws/handler
+              ::ws/on-rcv-message (partial on-rcv-message cfg)
+              ::ws/on-snd-message (partial on-snd-message cfg)
+              ::ws/on-connect (partial on-connect cfg)
+              ::ws/handler (partial handle-message cfg)
+              ::profile-id profile-id
+              ::session-id session-id)
+             (yws/upgrade request))))))
 
 (defmethod ig/pre-init-spec ::routes [_]
   (s/keys :req [::mbus/msgbus
@@ -322,4 +318,5 @@
 (defmethod ig/init-key ::routes
   [_ cfg]
   ["/ws/notifications" {:middleware [[session/authz cfg]]
-                        :handler (partial http-handler cfg)}])
+                        :handler (partial http-handler cfg)
+                        :allowed-methods #{:get}}])

backend/src/app/loggers/audit.clj

@@ -33,7 +33,7 @@
    [integrant.core :as ig]
    [lambdaisland.uri :as u]
    [promesa.exec :as px]
-   [ring.request :as rreq]))
+   [yetti.request :as yrq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -41,9 +41,9 @@
 
 (defn parse-client-ip
   [request]
-  (or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
-      (rreq/get-header request "x-real-ip")
-      (some-> (rreq/remote-addr request) str)))
+  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
+      (yrq/get-header request "x-real-ip")
+      (some-> (yrq/remote-addr request) str)))
 
 (defn extract-utm-params
   "Extracts additional data from params and namespace them under

backend/src/app/loggers/database.clj

@@ -39,40 +39,33 @@
 (defn record->report
   [{:keys [::l/context ::l/message ::l/props ::l/logger ::l/level ::l/cause] :as record}]
   (us/assert! ::l/record record)
-  (if (or (instance? java.util.concurrent.CompletionException cause)
-          (instance? java.util.concurrent.ExecutionException cause))
-    (-> record
-        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
-        (assoc ::l/cause (ex-cause cause))
-        (record->report))
 
-    (let [data (ex-data cause)
-          ctx  (-> context
-                   (assoc :tenant (cf/get :tenant))
-                   (assoc :host (cf/get :host))
-                   (assoc :public-uri (cf/get :public-uri))
-                   (assoc :logger/name logger)
-                   (assoc :logger/level level)
-                   (dissoc :request/params :value :params :data))]
-      (merge
-       {:context (-> (into (sorted-map) ctx)
-                     (pp/pprint-str :length 50))
-        :props   (pp/pprint-str props :length 50)
-        :hint    (or (ex-message cause) @message)
-        :trace   (or (::trace record)
-                     (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))}
+  (let [data (ex-data cause)
+        ctx  (-> context
+                 (assoc :tenant (cf/get :tenant))
+                 (assoc :host (cf/get :host))
+                 (assoc :public-uri (cf/get :public-uri))
+                 (assoc :logger/name logger)
+                 (assoc :logger/level level)
+                 (dissoc :request/params :value :params :data))]
+    (merge
+     {:context (-> (into (sorted-map) ctx)
+                   (pp/pprint-str :width 200 :length 50 :level 10))
+      :props   (pp/pprint-str props :width 200 :length 50)
+      :hint    (or (ex-message cause) @message)
+      :trace   (ex/format-throwable cause :data? false :explain? false :header? false :summary? false)}
 
-       (when-let [params (or (:request/params context) (:params context))]
-         {:params (pp/pprint-str params :length 30 :level 12)})
+     (when-let [params (or (:request/params context) (:params context))]
+       {:params (pp/pprint-str params :width 200 :length 50 :level 10)})
 
-       (when-let [value (:value context)]
-         {:value (pp/pprint-str value :length 30 :level 12)})
+     (when-let [value (:value context)]
+       {:value (pp/pprint-str value :width 200 :length 50 :level 10)})
 
-       (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-         {:data (pp/pprint-str data :length 30 :level 12)})
+     (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
+       {:data (pp/pprint-str data :width 200)})
 
-       (when-let [explain (ex/explain data :length 30 :level 12)]
-         {:explain explain})))))
+     (when-let [explain (ex/explain data {:level 10 :length 50})]
+       {:explain explain}))))
 
 (defn error-record?
   [{:keys [::l/level ::l/cause]}]
@@ -96,11 +89,11 @@
 
 (defmethod ig/init-key ::reporter
   [_ cfg]
-  (let [input (sp/chan :buf (sp/sliding-buffer 64)
+  (let [input (sp/chan :buf (sp/sliding-buffer 32)
                        :xf  (filter error-record?))]
     (add-watch l/log-record ::reporter #(sp/put! input %4))
 
-    (px/thread {:name "penpot/database-reporter"}
+    (px/thread {:name "penpot/database-reporter" :virtual true}
       (l/info :hint "initializing database error persistence")
       (try
         (loop []

backend/src/app/loggers/webhooks.clj

@@ -182,4 +182,5 @@
     "invalid-uri"
 
     (instance? java.net.http.HttpConnectTimeoutException cause)
-    "timeout"))
+    "timeout"
+    ))

backend/src/app/main.clj

@@ -10,7 +10,6 @@
    [app.auth.oidc :as-alias oidc]
    [app.auth.oidc.providers :as-alias oidc.providers]
    [app.common.logging :as l]
-   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.email :as-alias email]
@@ -37,12 +36,8 @@
    [app.storage.s3 :as-alias sto.s3]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [cider.nrepl :refer [cider-nrepl-handler]]
-   [clojure.test :as test]
-   [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [nrepl.server :as nrepl]
    [promesa.exec :as px])
   (:gen-class))
 
@@ -160,6 +155,12 @@
    {::mdef/name "penpot_executors_running_threads"
     ::mdef/help "Current number of threads with state RUNNING."
     ::mdef/labels ["name"]
+    ::mdef/type :gauge}
+
+   :executors-queued-submissions
+   {::mdef/name "penpot_executors_queued_submissions"
+    ::mdef/help "Current number of queued submissions."
+    ::mdef/labels ["name"]
     ::mdef/type :gauge}})
 
 (def system-config
@@ -174,12 +175,13 @@
 
    ;; Default thread pool for IO operations
    ::wrk/executor
-   {}
+   {::wrk/parallelism (cf/get :default-executor-parallelism
+                              (+ 3 (* (px/get-available-processors) 3)))}
 
    ::wrk/monitor
    {::mtx/metrics  (ig/ref ::mtx/metrics)
-    ::wrk/executor (ig/ref ::wrk/executor)
-    ::wrk/name     "default"}
+    ::wrk/name     "default"
+    ::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -210,7 +212,7 @@
    {::db/pool (ig/ref ::db/pool)}
 
    ::http.client/client
-   {}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    ::session/manager
    {::db/pool (ig/ref ::db/pool)}
@@ -221,12 +223,14 @@
    ::http.awsns/routes
    {::props              (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
-    ::http.client/client (ig/ref ::http.client/client)}
+    ::http.client/client (ig/ref ::http.client/client)
+    ::wrk/executor       (ig/ref ::wrk/executor)}
 
    ::http/server
    {::http/port                    (cf/get :http-server-port)
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
+    ::wrk/executor                 (ig/ref ::wrk/executor)
     ::http/io-threads              (cf/get :http-server-io-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
     ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
@@ -280,11 +284,11 @@
     ::http.ws/routes     (ig/ref ::http.ws/routes)
     ::http.awsns/routes  (ig/ref ::http.awsns/routes)}
 
-   ::http.debug/routes
+   :app.http.debug/routes
    {::db/pool         (ig/ref ::db/pool)
+    ::wrk/executor    (ig/ref ::wrk/executor)
     ::session/manager (ig/ref ::session/manager)
-    ::sto/storage     (ig/ref ::sto/storage)
-    ::props           (ig/ref ::setup/props)}
+    ::sto/storage     (ig/ref ::sto/storage)}
 
    ::http.ws/routes
    {::db/pool         (ig/ref ::db/pool)
@@ -296,10 +300,12 @@
    {::http.assets/path  (cf/get :assets-path)
     ::http.assets/cache-max-age (dt/duration {:hours 24})
     ::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
-    ::sto/storage  (ig/ref ::sto/storage)}
+    ::sto/storage  (ig/ref ::sto/storage)
+    ::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.rpc/climit
-   {::mtx/metrics  (ig/ref ::mtx/metrics)}
+   {::mtx/metrics  (ig/ref ::mtx/metrics)
+    ::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.rpc/rlimit
    {::wrk/executor (ig/ref ::wrk/executor)}
@@ -314,14 +320,14 @@
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
     ::rds/redis          (ig/ref ::rds/redis)
-    ::csvg/optimizer     (ig/ref ::csvg/optimizer)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
     ::setup/templates    (ig/ref ::setup/templates)
     ::props              (ig/ref ::setup/props)
 
-    :pool                (ig/ref ::db/pool)}
+    :pool                (ig/ref ::db/pool)
+    }
 
    :app.rpc.doc/routes
    {:methods (ig/ref :app.rpc/methods)}
@@ -329,6 +335,7 @@
    :app.rpc/routes
    {::rpc/methods     (ig/ref :app.rpc/methods)
     ::db/pool         (ig/ref ::db/pool)
+    ::wrk/executor    (ig/ref ::wrk/executor)
     ::session/manager (ig/ref ::session/manager)
     ::props           (ig/ref ::setup/props)}
 
@@ -403,9 +410,6 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
-   ::csvg/optimizer
-   {}
-
    ::audit.tasks/archive
    {::props              (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
@@ -430,18 +434,20 @@
 
    ::sto/storage
    {::db/pool      (ig/ref ::db/pool)
+    ::wrk/executor (ig/ref ::wrk/executor)
     ::sto/backends
     {:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
      :assets-fs (ig/ref [::assets :app.storage.fs/backend])}}
 
    [::assets :app.storage.s3/backend]
-   {::sto.s3/region     (cf/get :storage-assets-s3-region)
-    ::sto.s3/endpoint   (cf/get :storage-assets-s3-endpoint)
-    ::sto.s3/bucket     (cf/get :storage-assets-s3-bucket)
-    ::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)}
+   {::sto.s3/region   (cf/get :storage-assets-s3-region)
+    ::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
+    ::sto.s3/bucket   (cf/get :storage-assets-s3-bucket)
+    ::wrk/executor    (ig/ref ::wrk/executor)}
 
    [::assets :app.storage.fs/backend]
-   {::sto.fs/directory (cf/get :storage-assets-fs-directory)}})
+   {::sto.fs/directory (cf/get :storage-assets-fs-directory)}
+   })
 
 
 (def worker-config
@@ -515,65 +521,22 @@
                                    (merge worker-config))
                                  (ig/prep)
                                  (ig/init))))
-  (l/inf :hint "welcome to penpot"
-         :flags (str/join "," (map name cf/flags))
-         :worker? (contains? cf/flags :backend-worker)
-         :version (:full cf/version)))
+  (l/info :hint "welcome to penpot"
+          :flags (str/join "," (map name cf/flags))
+          :worker? (contains? cf/flags :backend-worker)
+          :version (:full cf/version)))
 
 (defn stop
   []
   (alter-var-root #'system (fn [sys]
                              (when sys (ig/halt! sys))
                              nil)))
-(defn restart
-  []
-  (stop)
-  (repl/refresh :after 'app.main/start))
-
-(defn restart-all
-  []
-  (stop)
-  (repl/refresh-all :after 'app.main/start))
-
-(defmacro run-bench
-  [& exprs]
-  `(do
-     (require 'criterium.core)
-     (criterium.core/with-progress-reporting (crit/quick-bench (do ~@exprs) :verbose))))
-
-(defn run-tests
-  ([] (run-tests #"^backend-tests.*-test$"))
-  ([o]
-   (repl/refresh)
-   (cond
-     (instance? java.util.regex.Pattern o)
-     (test/run-all-tests o)
-
-     (symbol? o)
-     (if-let [sns (namespace o)]
-       (do (require (symbol sns))
-           (test/test-vars [(resolve o)]))
-       (test/test-ns o)))))
-
-(repl/disable-reload! (find-ns 'integrant.core))
 
 (defn -main
   [& _args]
   (try
-    (let [p (promise)]
-      (when (contains? cf/flags :nrepl-server)
-        (l/inf :hint "start nrepl server" :port 6064)
-        (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
-
-      (start)
-      (deref p))
+    (start)
     (catch Throwable cause
-      (binding [*out* *err*]
-        (println "==== ERROR ===="))
-      (.printStackTrace cause)
-      (when-let [cause' (ex-cause cause)]
-        (binding [*out* *err*]
-          (println "==== CAUSE ===="))
-        (.printStackTrace cause'))
-      (px/sleep 500)
+      (l/error :hint (ex-message cause)
+               :cause cause)
       (System/exit -1))))

backend/src/app/media.clj

@@ -14,11 +14,11 @@
    [app.common.schema.generators :as sg]
    [app.common.schema.openapi :as-alias oapi]
    [app.common.spec :as us]
-   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
+   [app.util.svg :as svg]
    [app.util.time :as dt]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
@@ -201,7 +201,7 @@
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file

backend/src/app/metrics.clj

@@ -94,7 +94,7 @@
         writer   (StringWriter.)]
     (TextFormat/write004 writer samples)
     {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
-     :body (.toString writer)}))
+              :body (.toString writer)}))
 
 
 

backend/src/app/migrations.clj

@@ -330,14 +330,7 @@
    {:name "0105-mod-server-error-report-table"
     :fn (mg/resource "app/migrations/sql/0105-mod-server-error-report-table.sql")}
 
-   {:name "0106-add-file-tagged-object-thumbnail-table"
-    :fn (mg/resource "app/migrations/sql/0106-add-file-tagged-object-thumbnail-table.sql")}
-
-   {:name "0106-mod-team-table"
-    :fn (mg/resource "app/migrations/sql/0106-mod-team-table.sql")}
-
-   {:name "0107-mod-file-tagged-object-thumbnail-table"
-    :fn (mg/resource "app/migrations/sql/0107-mod-file-tagged-object-thumbnail-table.sql")}])
+   ])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0106-add-file-tagged-object-thumbnail-table.sql

@@ -1,10 +0,0 @@
-CREATE TABLE file_tagged_object_thumbnail (
-  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
-  tag text DEFAULT 'frame',
-  object_id text NOT NULL,
-
-  media_id uuid NOT NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE,
-  created_at timestamptz NOT NULL DEFAULT now(),
-
-  PRIMARY KEY(file_id, tag, object_id)
-);

backend/src/app/migrations/sql/0106-mod-team-table.sql

@@ -1 +0,0 @@
-ALTER TABLE team ADD COLUMN features text[] NULL DEFAULT null;

backend/src/app/migrations/sql/0107-mod-file-tagged-object-thumbnail-table.sql

@@ -1,2 +0,0 @@
-CREATE INDEX file_tagged_object_thumbnail__media_id__idx
-    ON file_tagged_object_thumbnail (media_id);

backend/src/app/rpc.clj

@@ -30,11 +30,12 @@
    [app.storage :as-alias sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.core :as p]
-   [ring.request :as rreq]
-   [ring.response :as rres]))
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
 (s/def ::profile-id ::us/uuid)
 
@@ -57,23 +58,21 @@
 
 (defn- handle-response
   [request result]
-  (let [mdata    (meta result)
-        response (if (fn? result)
-                   (result request)
-                   (let [result (rph/unwrap result)]
-                     {::rres/status  (::http/status mdata 200)
-                      ::rres/headers (::http/headers mdata {})
-                      ::rres/body    result}))]
-    (-> response
-        (handle-response-transformation request mdata)
-        (handle-before-comple-hook mdata))))
+  (if (fn? result)
+    (result request)
+    (let [mdata (meta result)]
+      (-> {::yrs/status  (::http/status mdata 200)
+           ::yrs/headers (::http/headers mdata {})
+           ::yrs/body    (rph/unwrap result)}
+          (handle-response-transformation request mdata)
+          (handle-before-comple-hook mdata)))))
 
 (defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
   [methods {:keys [params path-params] :as request}]
   (let [type       (keyword (:type path-params))
-        etag       (rreq/get-header request "if-none-match")
+        etag       (yrq/get-header request "if-none-match")
         profile-id (or (::session/profile-id request)
                        (::actoken/profile-id request))
 
@@ -139,20 +138,17 @@
         (f cfg (us/conform spec params)))
       f)))
 
-;; TODO: integrate with sm/define
-
 (defn- wrap-params-validation
   [_ f mdata]
   (if-let [schema (::sm/params mdata)]
-    (let [schema   (if (sm/lazy-schema? schema)
-                     schema
-                     (sm/define schema))
-          validate (sm/validator schema)
-          explain  (sm/explainer schema)
-          decode   (sm/decoder schema)]
+    (let [schema  (sm/schema schema)
+          valid?  (sm/validator schema)
+          explain (sm/explainer schema)
+          decode  (sm/decoder schema sm/default-transformer)]
+
       (fn [cfg params]
         (let [params (decode params)]
-          (if (validate params)
+          (if (valid? params)
             (f cfg params)
             (ex/raise :type :validation
                       :code :params-validation
@@ -163,15 +159,13 @@
   [_ f mdata]
   (if (contains? cf/flags :rpc-output-validation)
     (or (when-let [schema (::sm/result mdata)]
-          (let [schema   (if (sm/lazy-schema? schema)
-                           schema
-                           (sm/define schema))
-                validate (sm/validator schema)
-                explain  (sm/explainer schema)]
+          (let [schema  (sm/schema schema)
+                valid?  (sm/validator schema)
+                explain (sm/explainer schema)]
             (fn [cfg params]
               (let [response (f cfg params)]
                 (when (map? response)
-                  (when-not (validate response)
+                  (when-not (valid? response)
                     (ex/raise :type :validation
                               :code :data-validation
                               ::sm/explain (explain response))))
@@ -243,7 +237,8 @@
                 ::ldap/provider
                 ::sto/storage
                 ::mtx/metrics
-                ::main/props]
+                ::main/props
+                ::wrk/executor]
           :opt [::climit
                 ::rlimit]
           :req-un [::db/pool]))
@@ -262,6 +257,7 @@
   (s/keys :req [::methods
                 ::db/pool
                 ::main/props
+                ::wrk/executor
                 ::session/manager]))
 
 (defmethod ig/init-key ::routes

backend/src/app/rpc/climit.clj

@@ -31,24 +31,19 @@
 
 (set! *warn-on-reflection* true)
 
-(defn- id->str
-  [id]
-  (-> (str id)
-      (subs 1)))
-
 (defn- create-bulkhead-cache
-  [config]
-  (letfn [(load-fn [[id skey]]
-            (when-let [config (get config id)]
-              (l/trc :hint "insert into cache" :id (id->str id) :key skey)
+  [{:keys [::wrk/executor]} config]
+  (letfn [(load-fn [key]
+            (let [config (get config (nth key 0))]
+              (l/trace :hint "insert into cache" :key key)
               (pbh/create :permits (or (:permits config) (:concurrency config))
                           :queue (or (:queue config) (:queue-size config))
                           :timeout (:timeout config)
-                          :type :semaphore)))
+                          :executor executor
+                          :type (:type config :semaphore))))
 
-          (on-remove [key _ cause]
-            (let [[id skey] key]
-              (l/trc :hint "evict from cache" :id (id->str id) :key skey :reason (str cause))))]
+          (on-remove [_ _ cause]
+            (l/trace :hint "evict from cache" :key key :reason (str cause)))]
 
     (cache/create :executor :same-thread
                   :on-remove on-remove
@@ -70,21 +65,22 @@
 
 (s/def ::path ::fs/path)
 (defmethod ig/pre-init-spec ::rpc/climit [_]
-  (s/keys :req [::mtx/metrics ::path]))
+  (s/keys :req [::wrk/executor ::mtx/metrics ::path]))
 
 (defmethod ig/init-key ::rpc/climit
-  [_ {:keys [::path ::mtx/metrics] :as cfg}]
+  [_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}]
   (when (contains? cf/flags :rpc-climit)
     (when-let [params (some->> path slurp edn/read-string)]
-      (l/inf :hint "initializing concurrency limit" :config (str path))
+      (l/info :hint "initializing concurrency limit" :config (str path))
       (us/verify! ::config params)
-      {::cache (create-bulkhead-cache params)
+      {::cache (create-bulkhead-cache cfg params)
        ::config params
+       ::wrk/executor executor
        ::mtx/metrics metrics})))
 
 (s/def ::cache cache/cache?)
 (s/def ::instance
-  (s/keys :req [::cache ::config]))
+  (s/keys :req [::cache ::config ::wrk/executor]))
 
 (s/def ::rpc/climit
   (s/nilable ::instance))
@@ -95,92 +91,114 @@
 
 (defn invoke!
   [cache metrics id key f]
-  (if-let [limiter (cache/get cache [id key])]
-    (let [tpoint  (dt/tpoint)
-          labels  (into-array String [(id->str id)])
-          wrapped (fn []
-                    (let [elapsed (tpoint)
-                          stats   (pbh/get-stats limiter)]
-                      (l/trc :hint "acquired"
-                             :id (id->str id)
-                             :key key
-                             :permits (:permits stats)
-                             :queue (:queue stats)
-                             :max-permits (:max-permits stats)
-                             :max-queue (:max-queue stats)
-                             :elapsed (dt/format-duration elapsed))
+  (let [limiter (cache/get cache [id key])
+        tpoint  (dt/tpoint)
+        labels  (into-array String [(name id)])
 
-                      (mtx/run! metrics
-                                :id :rpc-climit-timing
-                                :val (inst-ms elapsed)
-                                :labels labels)
-                      (try
-                        (f)
-                        (finally
-                          (let [elapsed (tpoint)]
-                            (l/trc :hint "finished"
-                                   :id (id->str id)
-                                   :key key
-                                   :permits (:permits stats)
-                                   :queue (:queue stats)
-                                   :max-permits (:max-permits stats)
-                                   :max-queue (:max-queue stats)
-                                   :elapsed (dt/format-duration elapsed)))))))
-          measure!
-          (fn [stats]
+        wrapped
+        (fn []
+          (let [elapsed (tpoint)
+                stats   (pbh/get-stats limiter)]
+            (l/trace :hint "executed"
+                     :id (name id)
+                     :key key
+                     :fnh (hash f)
+                     :permits (:permits stats)
+                     :queue (:queue stats)
+                     :max-permits (:max-permits stats)
+                     :max-queue (:max-queue stats)
+                     :elapsed (dt/format-duration elapsed))
             (mtx/run! metrics
-                      :id :rpc-climit-queue
-                      :val (:queue stats)
+                      :id :rpc-climit-timing
+                      :val (inst-ms elapsed)
                       :labels labels)
-            (mtx/run! metrics
-                      :id :rpc-climit-permits
-                      :val (:permits stats)
-                      :labels labels))]
+            (try
+              (f)
+              (finally
+                (let [elapsed (tpoint)]
+                  (l/trace :hint "finished"
+                           :id (name id)
+                           :key key
+                           :fnh (hash f)
+                           :permits (:permits stats)
+                           :queue (:queue stats)
+                           :max-permits (:max-permits stats)
+                           :max-queue (:max-queue stats)
+                           :elapsed (dt/format-duration elapsed)))))))
+        measure!
+        (fn [stats]
+          (mtx/run! metrics
+                    :id :rpc-climit-queue
+                    :val (:queue stats)
+                    :labels labels)
+          (mtx/run! metrics
+                    :id :rpc-climit-permits
+                    :val (:permits stats)
+                    :labels labels))]
 
-      (try
-        (let [stats (pbh/get-stats limiter)]
-          (measure! stats)
-          (l/trc :hint "enqueued"
-                 :id (id->str id)
+    (try
+      (let [stats (pbh/get-stats limiter)]
+        (measure! stats)
+        (l/trace :hint "enqueued"
+                 :id (name id)
                  :key key
+                 :fnh (hash f)
                  :permits (:permits stats)
                  :queue (:queue stats)
                  :max-permits (:max-permits stats)
                  :max-queue (:max-queue stats))
-          (pbh/invoke! limiter wrapped))
-        (catch ExceptionInfo cause
-          (let [{:keys [type code]} (ex-data cause)]
-            (if (= :bulkhead-error type)
-              (ex/raise :type :concurrency-limit
-                        :code code
-                        :hint "concurrency limit reached")
-              (throw cause))))
+        (pbh/invoke! limiter wrapped))
+      (catch ExceptionInfo cause
+        (let [{:keys [type code]} (ex-data cause)]
+          (if (= :bulkhead-error type)
+            (ex/raise :type :concurrency-limit
+                      :code code
+                      :hint "concurrency limit reached")
+            (throw cause))))
 
-        (finally
-          (measure! (pbh/get-stats limiter)))))
+      (finally
+        (measure! (pbh/get-stats limiter))))))
 
-    (do
-      (l/wrn :hint "unable to load limiter" :id (id->str id))
-      (f))))
-
-(defn configure
-  [{:keys [::rpc/climit]} id]
-  (us/assert! ::rpc/climit climit)
-  (assoc climit ::id id))
 
 (defn run!
-  "Run a function in context of climit.
-  Intended to be used in virtual threads."
-  ([{:keys [::id ::cache ::mtx/metrics]} f]
-   (if (and cache id)
-     (invoke! cache metrics id nil f)
-     (f)))
+  [{:keys [::id ::cache ::mtx/metrics]} f]
+  (if (and cache id)
+    (invoke! cache metrics id nil f)
+    (f)))
 
-  ([{:keys [::id ::cache ::mtx/metrics]} f executor]
-   (let [f #(p/await! (px/submit! executor f))]
-     (if (and cache id)
-       (invoke! cache metrics id nil f)
-       (f)))))
+(defn submit!
+  [{:keys [::id ::cache ::wrk/executor ::mtx/metrics]} f]
+  (let [f (partial px/submit! executor (px/wrap-bindings f))]
+    (if (and cache id)
+      (p/await! (invoke! cache metrics id nil f))
+      (p/await! (f)))))
+
+(defn configure
+  ([{:keys [::rpc/climit]} id]
+   (us/assert! ::rpc/climit climit)
+   (assoc climit ::id id))
+  ([{:keys [::rpc/climit]} id executor]
+   (us/assert! ::rpc/climit climit)
+   (-> climit
+       (assoc ::id id)
+       (assoc ::wrk/executor executor))))
+
+(defmacro with-dispatch!
+  "Dispatch blocking operation to a separated thread protected with the
+  specified concurrency limiter. If climit is not active, the function
+  will be scheduled to execute without concurrency monitoring."
+  [instance & body]
+  (if (vector? instance)
+    `(-> (app.rpc.climit/configure ~@instance)
+         (app.rpc.climit/run! (^:once fn* [] ~@body)))
+    `(run! ~instance (^:once fn* [] ~@body))))
+
+(defmacro with-dispatch
+  "Dispatch blocking operation to a separated thread protected with
+  the specified semaphore.
+  DEPRECATED"
+  [& params]
+  `(with-dispatch! ~@params))
 
 (def noop-fn (constantly nil))
 
@@ -189,19 +207,18 @@
   (if (and (some? climit) (some? id))
     (if-let [config (get-in climit [::config id])]
       (let [cache (::cache climit)]
-        (l/dbg :hint "instrumenting method"
-               :limit (id->str id)
-               :service-name (::sv/name mdata)
-               :timeout (:timeout config)
-               :permits (:permits config)
-               :queue (:queue config)
-               :keyed? (not= key-fn noop-fn))
-
+        (l/debug :hint "wrap: instrumenting method"
+                 :limit (name id)
+                 :service-name (::sv/name mdata)
+                 :timeout (:timeout config)
+                 :permits (:permits config)
+                 :queue (:queue config)
+                 :keyed? (some? key-fn))
         (fn [cfg params]
           (invoke! cache metrics id (key-fn params) (partial f cfg params))))
 
       (do
-        (l/wrn :hint "no config found for specified queue" :id (id->str id))
+        (l/warn :hint "no config found for specified queue" :id id)
         f))
 
     f))

backend/src/app/rpc/commands/audit.clj

@@ -64,7 +64,7 @@
    [:events [:vector schema:event]]])
 
 (sv/defmethod ::push-audit-events
-  {::climit/id :submit-audit-events/by-profile
+  {::climit/id :submit-audit-events-by-profile
    ::climit/key-fn ::rpc/profile-id
    ::sm/params schema:push-audit-events
    ::audit/skip true

backend/src/app/rpc/commands/auth.clj

@@ -10,7 +10,6 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.uuid :as uuid]
@@ -292,12 +291,9 @@
 
 (defn create-profile-rels!
   [conn {:keys [id] :as profile}]
-  (let [features (cfeat/get-enabled-features cf/flags)
-        team     (teams/create-team conn
-                                    {:profile-id id
-                                     :name "Default"
-                                     :features features
-                                     :is-default true})]
+  (let [team (teams/create-team conn {:profile-id id
+                                      :name "Default"
+                                      :is-default true})]
     (-> (db/update! conn :profile
                     {:default-team-id (:id team)
                      :default-project-id  (:default-project-id team)}

backend/src/app/rpc/commands/binfile.clj

@@ -8,47 +8,39 @@
   (:refer-clojure :exclude [assert])
   (:require
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.defaults :as cfd]
-   [app.common.files.migrations :as fmg]
-   [app.common.files.validate :as fval]
+   [app.common.files.features :as ffeat]
+   [app.common.files.migrations :as pmg]
    [app.common.fressian :as fres]
    [app.common.logging :as l]
-   [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.components-v2 :as feat.compv2]
-   [app.features.fdata :as feat.fdata]
-   [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.blob :as blob]
+   [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as-alias wrk]
-   [clojure.set :as set]
    [clojure.spec.alpha :as s]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
    [datoteka.io :as io]
-   [promesa.core :as p]
-   [promesa.util :as pu]
-   [ring.response :as rres]
-   [yetti.adapter :as yt])
+   [yetti.adapter :as yt]
+   [yetti.response :as yrs])
   (:import
    com.github.luben.zstd.ZstdInputStream
    com.github.luben.zstd.ZstdOutputStream
@@ -305,25 +297,29 @@
 
 (defn- get-files
   [cfg ids]
-  (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (let [sql (str "SELECT id FROM file "
-                                " WHERE id = ANY(?) ")
-                       ids (db/create-array conn "uuid" ids)]
-                   (->> (db/exec! conn [sql ids])
-                        (into [] (map :id))
-                        (not-empty))))))
+  (letfn [(get-files* [{:keys [::db/conn]}]
+            (let [sql (str "SELECT id FROM file "
+                           " WHERE id = ANY(?) ")
+                  ids (db/create-array conn "uuid" ids)]
+              (->> (db/exec! conn [sql ids])
+                   (into [] (map :id))
+                   (not-empty))))]
+
+    (db/run! cfg get-files*)))
 
 (defn- get-file
   [cfg file-id]
-  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                 (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                   (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false})
-                           (files/decode-row)
-                           (update :data feat.fdata/process-pointers deref))))))
+  (letfn [(get-file* [{:keys [::db/conn]}]
+            (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
+              (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false})
+                      (files/decode-row)
+                      (files/process-pointers deref))))]
+
+    (db/run! cfg get-file*)))
 
 (defn- get-file-media
   [{:keys [::db/pool]} {:keys [data id] :as file}]
-  (pu/with-open [conn (db/open pool)]
+  (dm/with-open [conn (db/open pool)]
     (let [ids (app.tasks.file-gc/collect-used-media data)
           ids (db/create-array conn "uuid" ids)
           sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")]
@@ -335,14 +331,6 @@
       (->> (db/exec! conn [sql ids])
            (mapv #(assoc % :file-id id))))))
 
-(defn- get-file-thumbnails
-  "Return all file thumbnails for a given file."
-  [{:keys [::db/pool]} id]
-  (pu/with-open [conn (db/open pool)]
-    (let [sql "SELECT * FROM file_tagged_object_thumbnail WHERE file_id = ?"]
-      (->> (db/exec! conn [sql id])
-           (mapv #(dissoc % :file-id))))))
-
 (def ^:private storage-object-id-xf
   (comp
    (mapcat (juxt :media-id :thumbnail-id))
@@ -365,7 +353,7 @@
 
 (defn- get-libraries
   [{:keys [::db/pool]} ids]
-  (pu/with-open [conn (db/open pool)]
+  (dm/with-open [conn (db/open pool)]
     (let [ids (db/create-array conn "uuid" ids)]
       (map :id (db/exec! pool [sql:file-libraries ids])))))
 
@@ -377,7 +365,7 @@
                                 " WHERE flr.file_id = ANY(?)")]
                    (db/exec! conn [sql ids])))))
 
-(defn- create-or-update-file!
+(defn- create-or-update-file
   [conn params]
   (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
                  "VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
@@ -399,7 +387,6 @@
 (def ^:dynamic *options* nil)
 
 ;; --- EXPORT WRITER
-
 (defn- embed-file-assets
   [data cfg file-id]
   (letfn [(walk-map-form [form state]
@@ -484,19 +471,19 @@
 (defmethod write-export :default
   [{:keys [::output] :as options}]
   (write-header! output :v1)
-  (pu/with-open [output (zstd-output-stream output :level 12)
-                 output (io/data-output-stream output)]
-    (binding [*state* (volatile! {})]
-      (run! (fn [section]
-              (l/dbg :hint "write section" :section section ::l/sync? true)
-              (write-label! output section)
-              (let [options (-> options
-                                (assoc ::output output)
-                                (assoc ::section section))]
-                (binding [*options* options]
-                  (write-section options))))
+  (with-open [output (zstd-output-stream output :level 12)]
+    (with-open [output (io/data-output-stream output)]
+      (binding [*state* (volatile! {})]
+        (run! (fn [section]
+                (l/debug :hint "write section" :section section ::l/sync? true)
+                (write-label! output section)
+                (let [options (-> options
+                                  (assoc ::output output)
+                                  (assoc ::section section))]
+                  (binding [*options* options]
+                    (write-section options))))
 
-            [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
+              [:v1/metadata :v1/files :v1/rels :v1/sobjects])))))
 
 (defmethod write-section :v1/metadata
   [{:keys [::output ::file-ids ::include-libraries?] :as cfg}]
@@ -517,48 +504,37 @@
   (vswap! *state* assoc :sids [])
 
   (doseq [file-id (-> *state* deref :files)]
-    (let [detach?    (and (not embed-assets?) (not include-libraries?))
-          thumbnails (get-file-thumbnails cfg file-id)
-          file       (cond-> (get-file cfg file-id)
-                       detach?
-                       (-> (ctf/detach-external-references file-id)
-                           (dissoc :libraries))
+    (let [detach? (and (not embed-assets?) (not include-libraries?))
+          file  (cond-> (get-file cfg file-id)
+                  detach?
+                  (-> (ctf/detach-external-references file-id)
+                   (dissoc :libraries))
+                  embed-assets?
+                  (update :data embed-file-assets cfg file-id))
 
-                       embed-assets?
-                       (update :data embed-file-assets cfg file-id)
+          media (get-file-media cfg file)]
 
-                       :always
-                       (assoc :thumbnails thumbnails))
-
-          media      (get-file-media cfg file)]
-
-      (l/dbg :hint "write penpot file"
-             :id (str file-id)
-             :name (:name file)
-             :thumbnails (count thumbnails)
-             :features (:features file)
-             :media (count media)
-             ::l/sync? true)
+      (l/debug :hint "write penpot file"
+               :id file-id
+               :name (:name file)
+               :media (count media)
+               ::l/sync? true)
 
       (doseq [item media]
-        (l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true))
-
-      (doseq [item thumbnails]
-        (l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true))
+        (l/debug :hint "write penpot file media object" :id (:id item) ::l/sync? true))
 
       (doto output
         (write-obj! file)
         (write-obj! media))
 
-      (vswap! *state* update :sids into storage-object-id-xf media)
-      (vswap! *state* update :sids into storage-object-id-xf thumbnails))))
+      (vswap! *state* update :sids into storage-object-id-xf media))))
 
 (defmethod write-section :v1/rels
   [{:keys [::output ::include-libraries?] :as cfg}]
   (let [ids  (-> *state* deref :files)
         rels (when include-libraries?
                (get-library-relations cfg ids))]
-    (l/dbg :hint "found rels" :total (count rels) ::l/sync? true)
+    (l/debug :hint "found rels" :total (count rels) ::l/sync? true)
     (write-obj! output rels)))
 
 (defmethod write-section :v1/sobjects
@@ -566,22 +542,21 @@
   (let [sids    (-> *state* deref :sids)
         storage (media/configure-assets-storage storage)]
 
-    (l/dbg :hint "found sobjects"
-           :items (count sids)
-           ::l/sync? true)
+    (l/debug :hint "found sobjects"
+             :items (count sids)
+             ::l/sync? true)
 
     ;; Write all collected storage objects
     (write-obj! output sids)
 
     (doseq [id sids]
       (let [{:keys [size] :as obj} (sto/get-object storage id)]
-        (l/dbg :hint "write sobject" :id (str id) ::l/sync? true)
-
+        (l/debug :hint "write sobject" :id id ::l/sync? true)
         (doto output
           (write-uuid! id)
           (write-obj! (meta obj)))
 
-        (pu/with-open [stream (sto/get-object-data storage obj)]
+        (with-open [^InputStream stream (sto/get-object-data storage obj)]
           (let [written (write-stream! output stream size)]
             (when (not= written size)
               (ex/raise :type :validation
@@ -598,16 +573,15 @@
 (defmulti read-import ::version)
 (defmulti read-section ::section)
 
-(s/def ::profile-id ::us/uuid)
 (s/def ::project-id ::us/uuid)
 (s/def ::input io/input-stream?)
 (s/def ::overwrite? (s/nilable ::us/boolean))
+(s/def ::migrate? (s/nilable ::us/boolean))
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
 
-;; FIXME: replace with schema
 (s/def ::read-import-options
-  (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
-          :opt [::overwrite? ::ignore-index-errors?]))
+  (s/keys :req [::db/pool ::sto/storage ::project-id ::input]
+          :opt [::overwrite? ::migrate? ::ignore-index-errors?]))
 
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
@@ -617,6 +591,9 @@
   `::overwrite?`: if true, instead of creating new files and remapping id references,
   it reuses all ids and updates existing objects; defaults to `false`.
 
+  `::migrate?`: if true, applies the migration before persisting the
+  file data; defaults to `false`.
+
   `::ignore-index-errors?`: if true, do not fail on index lookup errors, can
   happen with broken files; defaults to: `false`.
   "
@@ -626,117 +603,53 @@
   (let [version (read-header! input)]
     (read-import (assoc options ::version version ::timestamp timestamp))))
 
-(defn- read-import-v1
-  [{:keys [::db/conn ::project-id ::profile-id ::input] :as options}]
-  (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
-  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-
-  (pu/with-open [input (zstd-input-stream input)
-                 input (io/data-input-stream input)]
-    (binding [*state* (volatile! {:media [] :index {}})]
-      (let [team      (teams/get-team conn
-                                      :profile-id profile-id
-                                      :project-id project-id)
-
-            validate? (contains? cf/flags :file-validation)
-            features  (cfeat/get-team-enabled-features cf/flags team)]
-
-        (sse/tap {:type :import-progress
-                  :section :read-import})
-
-        ;; Process all sections
-        (run! (fn [section]
-                (l/dbg :hint "reading section" :section section ::l/sync? true)
-                (assert-read-label! input section)
-                (let [options (-> options
-                                  (assoc ::enabled-features features)
-                                  (assoc ::section section)
-                                  (assoc ::input input))]
-                  (binding [*options* options]
-                    (sse/tap {:type :import-progress
-                              :section section})
-                    (read-section options))))
-              [:v1/metadata :v1/files :v1/rels :v1/sobjects])
-
-        ;; Run all pending migrations
-        (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
-          (case feature
-            "components/v2"
-            (feat.compv2/migrate-file! options file-id
-                                       :validate? validate?
-                                       :throw-on-validate? true)
-
-            "fdata/shape-data-type"
-            nil
-
-            (ex/raise :type :internal
-                      :code :no-migration-defined
-                      :hint (str/ffmt "no migation for feature '%' on file importation" feature)
-                      :feature feature)))
-
-        ;; Knowing that the ids of the created files are in index,
-        ;; just lookup them and return it as a set
-        (let [files (-> *state* deref :files)]
-          (into #{} (keep #(get-in @*state* [:index %])) files))))))
-
 (defmethod read-import :v1
-  [options]
-  (db/tx-run! options read-import-v1))
+  [{:keys [::db/pool ::input] :as options}]
+  (with-open [input (zstd-input-stream input)]
+    (with-open [input (io/data-input-stream input)]
+      (db/with-atomic [conn pool]
+        (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
+        (binding [*state* (volatile! {:media [] :index {}})]
+          (run! (fn [section]
+                  (l/debug :hint "reading section" :section section ::l/sync? true)
+                  (assert-read-label! input section)
+                  (let [options (-> options
+                                    (assoc ::section section)
+                                    (assoc ::input input)
+                                    (assoc ::db/conn conn))]
+                    (binding [*options* options]
+                      (read-section options))))
+                [:v1/metadata :v1/files :v1/rels :v1/sobjects])
+
+          ;; Knowing that the ids of the created files are in
+          ;; index, just lookup them and return it as a set
+          (let [files (-> *state* deref :files)]
+            (into #{} (keep #(get-in @*state* [:index %])) files)))))))
 
 (defmethod read-section :v1/metadata
   [{:keys [::input]}]
   (let [{:keys [version files]} (read-obj! input)]
-    (l/dbg :hint "metadata readed"
-           :version (:full version)
-           :files (mapv str files)
-           ::l/sync? true)
+    (l/debug :hint "metadata readed" :version (:full version) :files files ::l/sync? true)
     (vswap! *state* update :index update-index files)
     (vswap! *state* assoc :version version :files files)))
 
-(defn- get-remaped-thumbnails
-  [thumbnails file-id]
-  (mapv (fn [thumbnail]
-          (-> thumbnail
-              (assoc :file-id file-id)
-              (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/")))))
-        thumbnails))
-
-(defn- process-file
-  [{:keys [id] :as file}]
-  (-> file
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (assoc :id id)
-                          (dissoc :recent-colors)
-                          (cond-> (> (:version fdata) cfd/version)
-                            (assoc :version cfd/version))
-                          ;; FIXME: We're temporarily activating all
-                          ;; migrations because a problem in the
-                          ;; environments messed up with the version
-                          ;; numbers When this problem is fixed delete
-                          ;; the following line
-                          (cond-> (> (:version fdata) 22)
-                            (assoc :version 22)))))
-      (fmg/migrate-file)
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (update :pages-index relink-shapes)
-                          (update :components relink-shapes)
-                          (update :media relink-media)
-                          (d/without-nils))))))
-
+(defn- postprocess-file
+  [data]
+  (let [omap-wrap ffeat/*wrap-with-objects-map-fn*
+        pmap-wrap ffeat/*wrap-with-pointer-map-fn*]
+    (-> data
+        (update :pages-index update-vals #(update % :objects omap-wrap))
+        (update :pages-index update-vals pmap-wrap)
+        (update :components update-vals #(d/update-when % :objects omap-wrap))
+        (update :components pmap-wrap))))
 
 (defmethod read-section :v1/files
-  [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}]
-
+  [{:keys [::db/conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
   (doseq [expected-file-id (-> *state* deref :files)]
-    (let [file       (read-obj! input)
-          media      (read-obj! input)
-
-          file-id    (:id file)
-          file-id'   (lookup-index file-id)
-
-          thumbnails (:thumbnails file)]
+    (let [file     (read-obj! input)
+          media'   (read-obj! input)
+          file-id  (:id file)
+          features (files/get-default-features)]
 
       (when (not= file-id expected-file-id)
         (ex/raise :type :validation
@@ -745,84 +658,54 @@
                   :expected-id expected-file-id
                   :hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
 
-      (l/dbg :hint "processing file"
-             :id (str file-id)
-             :features (:features file)
-             :version (-> file :data :version)
-             :media (count media)
-             :thumbnails (count thumbnails)
-             ::l/sync? true)
+      ;; Update index using with media
+      (l/dbg :hint "update index with media" ::l/sync? true)
+      (vswap! *state* update :index update-index (map :id media'))
 
-      (when (seq thumbnails)
-        (let [thumbnails (get-remaped-thumbnails thumbnails file-id')]
-          (l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true)
-          (vswap! *state* update :thumbnails (fnil into []) thumbnails)))
+      ;; Store file media for later insertion
+      (l/dbg :hint "update media references" ::l/sync? true)
+      (vswap! *state* update :media into (map #(update % :id lookup-index)) media')
 
-      (when (seq media)
-        ;; Update index with media
-        (l/dbg :hint "update index with media" :total (count media) ::l/sync? true)
-        (vswap! *state* update :index update-index (map :id media))
+      (binding [ffeat/*current* features
+                ffeat/*wrap-with-objects-map-fn* (if (features "storage/objects-map") omap/wrap identity)
+                ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)
+                pmap/*tracked* (atom {})]
 
-        ;; Store file media for later insertion
-        (l/dbg :hint "update media references" ::l/sync? true)
-        (vswap! *state* update :media into (map #(update % :id lookup-index)) media))
+        (l/dbg :hint "processing file"
+               :id file-id
+               :features features
+               :version (-> file :data :version)
+               ::l/sync? true)
 
-      (let [file (-> file
-                     (assoc :id file-id')
-                     (process-file))
+        (let [file-id' (lookup-index file-id)
+              data     (-> (:data file)
+                           (assoc :id file-id')
+                           (cond-> migrate? (pmg/migrate-data))
+                           (update :pages-index relink-shapes)
+                           (update :components relink-shapes)
+                           (update :media relink-media)
+                           (postprocess-file))
 
-            ;; All features that are enabled and requires explicit migration are
-            ;; added to the state for a posterior migration step.
-            _    (doseq [feature (-> enabled-features
-                                     (set/difference cfeat/no-migration-features)
-                                     (set/difference (:features file)))]
-                   (vswap! *state* update :pending-to-migrate (fnil conj []) [feature file-id']))
+              params  {:id file-id'
+                       :project-id project-id
+                       :features (db/create-array conn "text" features)
+                       :name (:name file)
+                       :revn (:revn file)
+                       :is-shared (:is-shared file)
+                       :data (blob/encode data)
+                       :created-at timestamp
+                       :modified-at timestamp}]
 
-            file (-> file
-                     (assoc :project-id project-id)
-                     (assoc :created-at timestamp)
-                     (assoc :modified-at timestamp)
-                     (dissoc :thumbnails)
-                     (update :features
-                             (fn [features]
-                               (let [features (cfeat/check-supported-features! features)]
-                                 (-> enabled-features
-                                     (set/difference cfeat/frontend-only-features)
-                                     (set/union features))))))
-
-            _    (when (contains? cf/flags :file-schema-validation)
-                   (fval/validate-file-schema! file))
-
-            _    (when (contains? cf/flags :soft-file-schema-validation)
-                   (let [result (ex/try! (fval/validate-file-schema! file))]
-                     (when (ex/exception? result)
-                       (l/error :hint "file schema validation error" :cause result))))
-
-            file (if (contains? (:features file) "fdata/objects-map")
-                   (feat.fdata/enable-objects-map file)
-                   file)
-
-            file (if (contains? (:features file) "fdata/pointer-map")
-                   (binding [pmap/*tracked* (pmap/create-tracked)]
-                     (let [file (feat.fdata/enable-pointer-map file)]
-                       (feat.fdata/persist-pointers! system file-id')
-                       file))
-                   file)
-
-            file (-> file
-                     (update :features #(db/create-array conn "text" %))
-                     (update :data blob/encode))]
-
-          (l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
+          (l/debug :hint "create file" :id file-id' ::l/sync? true)
 
           (if overwrite?
-            (create-or-update-file! conn file)
-            (db/insert! conn :file file))
+            (create-or-update-file conn params)
+            (db/insert! conn :file params))
+
+          (files/persist-pointers! conn file-id')
 
           (when overwrite?
-            (db/delete! conn :file-thumbnail {:file-id file-id'}))
-
-          file-id'))))
+            (db/delete! conn :file-thumbnail {:file-id file-id'})))))))
 
 (defmethod read-section :v1/rels
   [{:keys [::db/conn ::input ::timestamp]}]
@@ -837,10 +720,10 @@
 
         (if (contains? ids library-file-id)
           (do
-            (l/dbg :hint "create file library link"
-                   :file-id (:file-id rel)
-                   :lib-id (:library-file-id rel)
-                   ::l/sync? true)
+            (l/debug :hint "create file library link"
+                     :file-id (:file-id rel)
+                     :lib-id (:library-file-id rel)
+                     ::l/sync? true)
             (db/insert! conn :file-library-rel rel))
 
           (l/warn :hint "ignoring file library link"
@@ -849,10 +732,9 @@
                   ::l/sync? true))))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::sto/storage ::db/conn ::input ::overwrite? ::timestamp]}]
+  [{:keys [::sto/storage ::db/conn ::input ::overwrite?]}]
   (let [storage (media/configure-assets-storage storage)
-        ids     (read-obj! input)
-        thumb?  (into #{} (map :media-id) (:thumbnails @*state*))]
+        ids     (read-obj! input)]
 
     (doseq [expected-storage-id ids]
       (let [id    (read-uuid! input)
@@ -863,57 +745,38 @@
                     :code :inconsistent-penpot-file
                     :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
 
-        (l/dbg :hint "readed storage object" :id (str id) ::l/sync? true)
+        (l/debug :hint "readed storage object" :id id ::l/sync? true)
 
         (let [[size resource] (read-stream! input)
               hash            (sto/calculate-hash resource)
               content         (-> (sto/content resource size)
                                   (sto/wrap-with-hash hash))
-
               params          (-> mdata
-                                  (assoc ::sto/content content)
                                   (assoc ::sto/deduplicate? true)
-                                  (assoc ::sto/touched-at timestamp))
-
-              params          (if (thumb? id)
-                                (assoc params :bucket "file-object-thumbnail")
-                                (assoc params :bucket "file-media-object"))
+                                  (assoc ::sto/content content)
+                                  (assoc ::sto/touched-at (dt/now))
+                                  (assoc :bucket "file-media-object"))
 
               sobject         (sto/put-object! storage params)]
 
-          (l/dbg :hint "persisted storage object"
-                 :old-id (str id)
-                 :new-id (str (:id sobject))
-                 :is-thumbnail (boolean (thumb? id))
-                 ::l/sync? true)
-
+          (l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/sync? true)
           (vswap! *state* update :index assoc id (:id sobject)))))
 
     (doseq [item (:media @*state*)]
-      (l/dbg :hint "inserting file media object"
-             :id (str (:id item))
-             :file-id (str (:file-id item))
-             ::l/sync? true)
+      (l/debug :hint "inserting file media object"
+               :id (:id item)
+               :file-id (:file-id item)
+               ::l/sync? true)
 
       (let [file-id (lookup-index (:file-id item))]
         (if (= file-id (:file-id item))
-          (l/warn :hint "ignoring file media object" :file-id (str file-id) ::l/sync? true)
+          (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/sync? true)
           (db/insert! conn :file-media-object
                       (-> item
                           (assoc :file-id file-id)
                           (d/update-when :media-id lookup-index)
                           (d/update-when :thumbnail-id lookup-index))
-                      {::db/on-conflict-do-nothing? overwrite?}))))
-
-    (doseq [item (:thumbnails @*state*)]
-      (let [item (update item :media-id lookup-index)]
-        (l/dbg :hint "inserting file object thumbnail"
-               :file-id (str (:file-id item))
-               :media-id (str (:media-id item))
-               :object-id (:object-id item)
-               ::l/sync? true)
-        (db/insert! conn :file-tagged-object-thumbnail item
-                    {::db/on-conflict-do-nothing? overwrite?})))))
+                      {:on-conflict-do-nothing overwrite?}))))))
 
 (defn- lookup-index
   [id]
@@ -1008,8 +871,8 @@
         ab (volatile! false)
         cs (volatile! nil)]
     (try
-      (l/info :hint "start exportation" :export-id (str id))
-      (pu/with-open [output (io/output-stream output)]
+      (l/info :hint "start exportation" :export-id id)
+      (dm/with-open [output (io/output-stream output)]
         (binding [*position* (atom 0)]
           (write-export! (assoc cfg ::output output))))
 
@@ -1024,7 +887,7 @@
         (throw cause))
 
       (finally
-        (l/info :hint "exportation finished" :export-id (str id)
+        (l/info :hint "exportation finished" :export-id id
                 :elapsed (str (inst-ms (tp)) "ms")
                 :aborted @ab
                 :cause @cs)))))
@@ -1032,7 +895,7 @@
 (defn export-to-tmpfile!
   [cfg]
   (let [path (tmp/tempfile :prefix "penpot.export.")]
-    (pu/with-open [output (io/output-stream path)]
+    (dm/with-open [output (io/output-stream path)]
       (export! cfg output)
       path)))
 
@@ -1041,10 +904,10 @@
   (let [id (uuid/next)
         tp (dt/tpoint)
         cs (volatile! nil)]
-    (l/info :hint "import: started" :id (str id))
+    (l/info :hint "import: started" :import-id id)
     (try
       (binding [*position* (atom 0)]
-        (pu/with-open [input (io/input-stream input)]
+        (dm/with-open [input (io/input-stream input)]
           (read-import! (assoc cfg ::input input))))
 
       (catch Throwable cause
@@ -1053,78 +916,61 @@
 
       (finally
         (l/info :hint "import: terminated"
-                :id (str id)
+                :import-id id
                 :elapsed (dt/format-duration (tp))
-                :error? (some? @cs))))))
-
+                :error? (some? @cs)
+                :cause @cs
+                )))))
 
 ;; --- Command: export-binfile
 
-(def ^:private
-  schema:export-binfile
-  (sm/define
-    [:map {:title "export-binfile"}
-     [:file-id ::sm/uuid]
-     [:include-libraries? :boolean]
-     [:embed-assets? :boolean]]))
+(s/def ::file-id ::us/uuid)
+(s/def ::include-libraries? ::us/boolean)
+(s/def ::embed-assets? ::us/boolean)
+
+(s/def ::export-binfile
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::include-libraries? ::embed-assets?]))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
-   ::webhooks/event? true
-   ::sm/result schema:export-binfile}
+   ::webhooks/event? true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries? embed-assets?] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (fn [_]
-    {::rres/status 200
-     ::rres/headers {"content-type" "application/octet-stream"}
-     ::rres/body (reify rres/StreamableResponseBody
-                   (-write-body-to-stream [_ _ output-stream]
-                     (-> cfg
-                         (assoc ::file-ids [file-id])
-                         (assoc ::embed-assets? embed-assets?)
-                         (assoc ::include-libraries? include-libraries?)
-                         (export! output-stream))))}))
+  (let [body (reify yrs/StreamableResponseBody
+               (-write-body-to-stream [_ _ output-stream]
+                 (-> cfg
+                     (assoc ::file-ids [file-id])
+                     (assoc ::embed-assets? embed-assets?)
+                     (assoc ::include-libraries? include-libraries?)
+                     (export! output-stream))))]
 
+    (fn [_]
+      {::yrs/status 200
+       ::yrs/body body
+       ::yrs/headers {"content-type" "application/octet-stream"}})))
 
-;; --- Command: import-binfile
-
-(def ^:private
-  schema:import-binfile
-  (sm/define
-    [:map {:title "import-binfile"}
-     [:project-id ::sm/uuid]
-     [:file ::media/upload]]))
-
-(declare ^:private import-binfile)
+(s/def ::file ::media/upload)
+(s/def ::import-binfile
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::project-id ::file]))
 
 (sv/defmethod ::import-binfile
   "Import a penpot file in a binary format."
   {::doc/added "1.15"
-   ::webhooks/event? true
-   ::sse/stream? true
-   ::sm/params schema:import-binfile}
+   ::webhooks/event? true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id file] :as params}]
-  (projects/check-read-permissions! pool profile-id project-id)
-  (let [params (-> cfg
-                   (assoc ::input (:path file))
-                   (assoc ::project-id project-id)
-                   (assoc ::profile-id profile-id)
-                   (assoc ::ignore-index-errors? true))]
-    (with-meta
-      (sse/response #(import-binfile params))
-      {::audit/props {:file nil}})))
+  (db/with-atomic [conn pool]
+    (projects/check-read-permissions! conn profile-id project-id)
+    (let [ids (import! (assoc cfg
+                              ::input (:path file)
+                              ::project-id project-id
+                              ::ignore-index-errors? true))]
 
-(defn- import-binfile
-  [{:keys [::wrk/executor ::project-id] :as params}]
-  (db/tx-run! params
-              (fn [{:keys [::db/conn] :as params}]
-                ;; NOTE: the importation process performs some operations that
-                ;; are not very friendly with virtual threads, and for avoid
-                ;; unexpected blocking of other concurrent operations we
-                ;; dispatch that operation to a dedicated executor.
-                (let [result (p/thread-call executor (partial import! params))]
-                  (db/update! conn :project
-                              {:modified-at (dt/now)}
-                              {:id project-id})
-                  (deref result)))))
+      (db/update! conn :project
+                  {:modified-at (dt/now)}
+                  {:id project-id})
+
+      (rph/with-meta ids
+        {::audit/props {:file nil :file-ids ids}}))))

backend/src/app/rpc/commands/comments.clj

@@ -12,7 +12,6 @@
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -44,17 +43,15 @@
 
 (defn- get-file
   "A specialized version of get-file for comments module."
-  [{:keys [::db/conn] :as cfg} file-id page-id]
-  (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id])
-                                           (files/decode-row))]
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+  [conn file-id page-id]
+  (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
+    (if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id]) (files/decode-row))]
       (-> file
           (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
-          (assoc :page-id page-id)))
-
-    (ex/raise :type :not-found
-              :code :object-not-found
-              :hint "file not found")))
+          (assoc :page-id page-id))
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "file not found"))))
 
 (defn- get-comment-thread
   [conn thread-id & {:as opts}]
@@ -291,37 +288,38 @@
 (sv/defmethod ::create-comment-thread
   {::doc/added "1.15"
    ::webhooks/event? true}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (files/check-comment-permissions! conn profile-id file-id share-id)
-                (let [{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
+  [{:keys [::db/pool] :as cfg}
+   {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
 
-                  (run! (partial quotes/check-quote! conn)
-                        (list {::quotes/id ::quotes/comment-threads-per-file
-                               ::quotes/profile-id profile-id
-                               ::quotes/team-id team-id
-                               ::quotes/project-id project-id
-                               ::quotes/file-id file-id}
-                              {::quotes/id ::quotes/comments-per-file
-                               ::quotes/profile-id profile-id
-                               ::quotes/team-id team-id
-                               ::quotes/project-id project-id
-                               ::quotes/file-id file-id}))
+  (db/with-atomic [conn pool]
+    (let [{:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
+      (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (rtry/with-retry {::rtry/when rtry/conflict-exception?
-                                    ::rtry/max-retries 3
-                                    ::rtry/label "create-comment-thread"
-                                    ::db/conn conn}
-                    (create-comment-thread conn
-                                           {:created-at request-at
-                                            :profile-id profile-id
-                                            :file-id file-id
-                                            :page-id page-id
-                                            :page-name page-name
-                                            :position position
-                                            :content content
-                                            :frame-id frame-id}))))))
+      (run! (partial quotes/check-quote! conn)
+            (list {::quotes/id ::quotes/comment-threads-per-file
+                   ::quotes/profile-id profile-id
+                   ::quotes/team-id team-id
+                   ::quotes/project-id project-id
+                   ::quotes/file-id file-id}
+                  {::quotes/id ::quotes/comments-per-file
+                   ::quotes/profile-id profile-id
+                   ::quotes/team-id team-id
+                   ::quotes/project-id project-id
+                   ::quotes/file-id file-id}))
+
+      (rtry/with-retry {::rtry/when rtry/conflict-exception?
+                        ::rtry/max-retries 3
+                        ::rtry/label "create-comment-thread"
+                        ::db/conn conn}
+        (create-comment-thread conn
+                               {:created-at request-at
+                                :profile-id profile-id
+                                :file-id file-id
+                                :page-id page-id
+                                :page-name page-name
+                                :position position
+                                :content content
+                                :frame-id frame-id})))))
 
 
 (defn- create-comment-thread
@@ -404,7 +402,8 @@
 
 ;; --- COMMAND: Add Comment
 
-(declare ^:private get-comment-thread)
+(declare get-comment-thread)
+(declare create-comment)
 
 (s/def ::create-comment
   (s/keys :req [::rpc/profile-id]
@@ -414,51 +413,49 @@
 (sv/defmethod ::create-comment
   {::doc/added "1.15"
    ::webhooks/event? true}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
-                      {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content] :as params}]
+  (db/with-atomic [conn pool]
+    (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
+          {:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
 
-                  (files/check-comment-permissions! conn profile-id (:id file) share-id)
-                  (quotes/check-quote! conn
-                                       {::quotes/id ::quotes/comments-per-file
-                                        ::quotes/profile-id profile-id
-                                        ::quotes/team-id team-id
-                                        ::quotes/project-id project-id
-                                        ::quotes/file-id (:id file)})
+      (files/check-comment-permissions! conn profile-id (:id file) share-id)
+      (quotes/check-quote! conn
+                           {::quotes/id ::quotes/comments-per-file
+                            ::quotes/profile-id profile-id
+                            ::quotes/team-id team-id
+                            ::quotes/project-id project-id
+                            ::quotes/file-id (:id file)})
 
-                  ;; Update the page-name cached attribute on comment thread table.
-                  (when (not= page-name (:page-name thread))
-                    (db/update! conn :comment-thread
-                                {:page-name page-name}
-                                {:id thread-id}))
+      ;; Update the page-name cached attribute on comment thread table.
+      (when (not= page-name (:page-name thread))
+        (db/update! conn :comment-thread
+                    {:page-name page-name}
+                    {:id thread-id}))
 
-                  (let [comment (db/insert! conn :comment
-                                            {:id (uuid/next)
-                                             :created-at request-at
-                                             :modified-at request-at
-                                             :thread-id thread-id
-                                             :owner-id profile-id
-                                             :content content})
-                        props    {:file-id file-id
-                                  :share-id nil}]
+      (let [comment (db/insert! conn :comment
+                                {:id (uuid/next)
+                                 :created-at request-at
+                                 :modified-at request-at
+                                 :thread-id thread-id
+                                 :owner-id profile-id
+                                 :content content})
+            props    {:file-id file-id
+                      :share-id nil}]
 
-                    ;; Update thread modified-at attribute and assoc the current
-                    ;; profile to the participant set.
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :participants (-> (:participants thread #{})
-                                                   (conj profile-id)
-                                                   (db/tjson))}
-                                {:id thread-id})
+        ;; Update thread modified-at attribute and assoc the current
+        ;; profile to the participant set.
+        (db/update! conn :comment-thread
+                    {:modified-at request-at
+                     :participants (-> (:participants thread #{})
+                                       (conj profile-id)
+                                       (db/tjson))}
+                    {:id thread-id})
 
-                    ;; Update the current profile status in relation to the
-                    ;; current thread.
-                    (upsert-comment-thread-status! conn profile-id thread-id request-at)
-
-                    (vary-meta comment assoc ::audit/props props))))))
+        ;; Update the current profile status in relation to the
+        ;; current thread.
+        (upsert-comment-thread-status! conn profile-id thread-id request-at)
 
+        (vary-meta comment assoc ::audit/props props)))))
 
 ;; --- COMMAND: Update Comment
 
@@ -469,31 +466,29 @@
 
 (sv/defmethod ::update-comment
   {::doc/added "1.15"}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content] :as params}]
+  (db/with-atomic [conn pool]
+    (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true)
+          {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
 
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true)
-                      {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
+      (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (files/check-comment-permissions! conn profile-id file-id share-id)
+      ;; Don't allow edit comments to not owners
+      (when-not (= owner-id profile-id)
+        (ex/raise :type :validation
+                  :code :not-allowed))
 
-                  ;; Don't allow edit comments to not owners
-                  (when-not (= owner-id profile-id)
-                    (ex/raise :type :validation
-                              :code :not-allowed))
+      (let [{:keys [page-name] :as file} (get-file conn file-id page-id)]
+        (db/update! conn :comment
+                    {:content content
+                     :modified-at request-at}
+                    {:id id})
 
-                  (let [{:keys [page-name] :as file} (get-file cfg file-id page-id)]
-                    (db/update! conn :comment
-                                {:content content
-                                 :modified-at request-at}
-                                {:id id})
-
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :page-name page-name}
-                                {:id thread-id})
-                    nil)))))
+        (db/update! conn :comment-thread
+                    {:modified-at request-at
+                     :page-name page-name}
+                    {:id thread-id})
+        nil))))
 
 ;; --- COMMAND: Delete Comment Thread
 
@@ -504,7 +499,7 @@
 
 (sv/defmethod ::delete-comment-thread
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}]
   (db/with-atomic [conn pool]
     (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
@@ -544,12 +539,12 @@
 
 (sv/defmethod ::update-comment-thread-position
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id position frame-id share-id] :as params}]
   (db/with-atomic [conn pool]
     (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
       (db/update! conn :comment-thread
-                  {:modified-at request-at
+                  {:modified-at (::rpc/request-at params)
                    :position (db/pgpoint position)
                    :frame-id frame-id}
                   {:id (:id thread)})
@@ -564,12 +559,12 @@
 
 (sv/defmethod ::update-comment-thread-frame
   {::doc/added "1.15"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id frame-id share-id] :as params}]
   (db/with-atomic [conn pool]
     (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
       (files/check-comment-permissions! conn profile-id file-id share-id)
       (db/update! conn :comment-thread
-                  {:modified-at request-at
+                  {:modified-at (::rpc/request-at params)
                    :frame-id frame-id}
                   {:id id})
       nil)))

backend/src/app/rpc/commands/files.clj

@@ -9,18 +9,16 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.helpers :as cfh]
-   [app.common.files.migrations :as fmg]
-   [app.common.logging :as l]
+   [app.common.files.migrations :as pmg]
+   [app.common.pages.helpers :as cph]
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as-alias smdj]
+   [app.common.schema.generators :as sg]
    [app.common.spec :as us]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -34,6 +32,7 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
+   [clojure.set :as set]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]))
 
@@ -44,6 +43,23 @@
   (when media-id
     (str (cf/get :public-uri) "/assets/by-id/" media-id)))
 
+(def supported-features
+  #{"storage/objects-map"
+    "storage/pointer-map"
+    "internal/shape-record"
+    "internal/geom-record"
+    "components/v2"})
+
+(defn get-default-features
+  []
+  (cond-> #{"internal/shape-record"
+            "internal/geom-record"}
+    (contains? cf/flags :fdata-storage-pointer-map)
+    (conj "storage/pointer-map")
+
+    (contains? cf/flags :fdata-storage-objects-map)
+    (conj "storage/objects-map")))
+
 ;; --- SPECS
 
 (s/def ::features ::us/set-of-strings)
@@ -69,22 +85,6 @@
       changes  (assoc :changes (blob/decode changes))
       data     (assoc :data (blob/decode data)))))
 
-(defn check-version!
-  [{:keys [data] :as file}]
-  (dm/assert!
-   "expect data to be decoded"
-   (map? data))
-
-  (let [version (:version data 0)]
-    (when (> version fmg/version)
-      (ex/raise :type :restriction
-                :code :file-version-not-supported
-                :hint "file version is greated that the maximum"
-                :file-version version
-                :max-version fmg/version))
-
-    file))
-
 ;; --- FILE PERMISSIONS
 
 (def ^:private sql:file-permissions
@@ -181,94 +181,194 @@
                 :code :object-not-found
                 :hint "not found"))))
 
+;; --- HELPERS
+
+(defn get-team-id
+  [conn project-id]
+  (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FEATURES: pointer-map
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn check-features-compatibility!
+  "Function responsible to check if provided features are supported by
+  the current backend"
+  [features]
+  (let [not-supported (set/difference features supported-features)]
+    (when (seq not-supported)
+      (ex/raise :type :restriction
+                :code :features-not-supported
+                :feature (first not-supported)
+                :hint (format "features %s not supported" (str/join "," (map name not-supported)))))
+    features))
+
+(defn load-pointer
+  [conn file-id id]
+  (let [row (db/get conn :file-data-fragment
+                    {:id id :file-id file-id}
+                    {:columns [:content]
+                     ::db/check-deleted? false})]
+    (blob/decode (:content row))))
+
+(defn- load-all-pointers!
+  [{:keys [data] :as file}]
+  (doseq [[_id page] (:pages-index data)]
+    (when (pmap/pointer-map? page)
+      (pmap/load! page)))
+  (doseq [[_id component] (:components data)]
+    (when (pmap/pointer-map? component)
+      (pmap/load! component)))
+  file)
+
+(defn persist-pointers!
+  [conn file-id]
+  (doseq [[id item] @pmap/*tracked*]
+    (when (pmap/modified? item)
+      (let [content (-> item deref blob/encode)]
+        (db/insert! conn :file-data-fragment
+                    {:id id
+                     :file-id file-id
+                     :content content})))))
+
+(defn process-pointers
+  [file update-fn]
+  (update file :data (fn resolve-fn [data]
+                       (cond-> data
+                         (contains? data :pages-index)
+                         (update :pages-index resolve-fn)
+
+                         :always
+                         (update-vals (fn [val]
+                                        (if (pmap/pointer-map? val)
+                                          (update-fn val)
+                                          val)))))))
+
+
+(defn get-all-pointer-ids
+  "Given a file, return all pointer ids used in the data."
+  [fdata]
+  (->> (concat (vals fdata)
+               (vals (:pages-index fdata)))
+       (into #{} (comp (filter pmap/pointer-map?)
+                       (map pmap/get-id)))))
+
+;; FIXME: file locking
+(defn- process-components-v2-feature
+  "A special case handling of the components/v2 feature."
+  [{:keys [features data] :as file}]
+  (let [data     (ctf/migrate-to-components-v2 data)
+        features (conj features "components/v2")]
+    (-> file
+        (assoc ::pmg/migrated true)
+        (assoc :features features)
+        (assoc :data data))))
+
+(defn handle-file-features!
+  [{:keys [features] :as file} client-features]
+
+  ;; Check features compatibility between the currently supported features on
+  ;; the current backend instance and the file retrieved from the database
+  (check-features-compatibility! features)
+
+  (cond-> file
+    (and (contains? features "components/v2")
+         (not (contains? client-features "components/v2")))
+    (as-> file (ex/raise :type :restriction
+                         :code :feature-mismatch
+                         :feature "components/v2"
+                         :hint "file has 'components/v2' feature enabled but frontend didn't specifies it"
+                         :file-id (:id file)))
+
+    ;; This operation is needed because the components migration generates a new
+    ;; page with random id which is returned to the client; without persisting
+    ;; the migration this can cause that two simultaneous clients can have a
+    ;; different view of the file data and end persisting two pages with main
+    ;; components and breaking the whole file."
+    (and (contains? client-features "components/v2")
+         (not (contains? features "components/v2")))
+    (as-> file (process-components-v2-feature file))
+
+    ;; This operation is needed for backward comapatibility with frontends that
+    ;; does not support pointer-map resolution mechanism; this just resolves the
+    ;; pointers on backend and return a complete file.
+    (and (contains? features "storage/pointer-map")
+         (not (contains? client-features "storage/pointer-map")))
+    (process-pointers deref)))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; QUERY COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; --- COMMAND QUERY: get-file (by id)
 
+(def schema:features
+  [:schema
+   {:title "FileFeatures"
+    ::smdj/inline true
+    :gen/gen (sg/subseq supported-features)}
+   ::sm/set-of-strings])
+
 (def schema:file
-  (sm/define
-    [:map {:title "File"}
-     [:id ::sm/uuid]
-     [:features ::cfeat/features]
-     [:has-media-trimmed :boolean]
-     [:comment-thread-seqn {:min 0} :int]
-     [:name :string]
-     [:revn {:min 0} :int]
-     [:modified-at ::dt/instant]
-     [:is-shared :boolean]
-     [:project-id ::sm/uuid]
-     [:created-at ::dt/instant]
-     [:data {:optional true} :any]]))
+  [:map {:title "File"}
+   [:id ::sm/uuid]
+   [:features schema:features]
+   [:has-media-trimmed :boolean]
+   [:comment-thread-seqn {:min 0} :int]
+   [:name :string]
+   [:revn {:min 0} :int]
+   [:modified-at ::dt/instant]
+   [:is-shared :boolean]
+   [:project-id ::sm/uuid]
+   [:created-at ::dt/instant]
+   [:data {:optional true} :any]])
 
 (def schema:permissions-mixin
-  (sm/define
-    [:map {:title "PermissionsMixin"}
-     [:permissions ::perms/permissions]]))
+  [:map {:title "PermissionsMixin"}
+   [:permissions ::perms/permissions]])
 
 (def schema:file-with-permissions
-  (sm/define
-    [:merge {:title "FileWithPermissions"}
-     schema:file
-     schema:permissions-mixin]))
+  [:merge {:title "FileWithPermissions"}
+   schema:file
+   schema:permissions-mixin])
 
-(def ^:private
-  schema:get-file
-  (sm/define
-    [:map {:title "get-file"}
-     [:features {:optional true} ::cfeat/features]
-     [:id ::sm/uuid]
-     [:project-id {:optional true} ::sm/uuid]]))
-
-(defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-            pmap/*tracked* (pmap/create-tracked)]
-    (let [file (fmg/migrate-file file)]
-
-      ;; NOTE: when file is migrated, we break the rule of no perform
-      ;; mutations on get operations and update the file with all
-      ;; migrations applied
-      ;;
-      ;; NOTE: the following code will not work on read-only mode, it
-      ;; is a known issue; we keep is not implemented until we really
-      ;; need this
-      (when (fmg/migrated? file)
-        (db/update! conn :file
-                    {:data (blob/encode (:data file))
-                     :features (db/create-array conn "text" (:features file))}
-                    {:id id}
-                    {::db/return-keys? false})
-
-        (when (contains? (:features file) "fdata/pointer-map")
-          (feat.fdata/persist-pointers! cfg id)))
-
-      file)))
+(def schema:get-file
+  [:map {:title "get-file"}
+   [:features {:optional true} schema:features]
+   [:id ::sm/uuid]
+   [:project-id {:optional true} ::sm/uuid]])
 
 (defn get-file
-  [{:keys [::db/conn] :as cfg} id & {:keys [project-id
-                                            migrate?
-                                            include-deleted?
-                                            lock-for-update?]
-                                     :or {include-deleted? false
-                                          lock-for-update? false
-                                          migrate? true}}]
-  (dm/assert!
-   "expected cfg with valid connection"
-   (db/connection-map? cfg))
+  ([conn id client-features]
+   (get-file conn id client-features nil))
+  ([conn id client-features project-id]
+   ;; here we check if client requested features are supported
+   (check-features-compatibility! client-features)
+   (binding [pmap/*load-fn* (partial load-pointer conn id)
+             pmap/*tracked* (atom {})]
 
-  (let [params (merge {:id id}
-                      (when (some? project-id)
-                        {:project-id project-id}))
-        file   (-> (db/get conn :file params
-                           {::db/check-deleted? (not include-deleted?)
-                            ::db/remove-deleted? (not include-deleted?)
-                            ::db/for-update? lock-for-update?})
-                   (decode-row))]
-    (if migrate?
-      (migrate-file cfg file)
-      file)))
+     (let [params (merge {:id id}
+                         (when (some? project-id)
+                           {:project-id project-id}))
+
+           file   (-> (db/get conn :file params)
+                      (decode-row)
+                      (pmg/migrate-file))
+
+           file   (handle-file-features! file client-features)]
+
+       ;; NOTE: when file is migrated, we break the rule of no perform
+       ;; mutations on get operations and update the file with all
+       ;; migrations applied
+       (when (pmg/migrated? file)
+         (let [features (db/create-array conn "text" (:features file))]
+           (db/update! conn :file
+                       {:data (blob/encode (:data file))
+                        :features features}
+                       {:id id})
+           (persist-pointers! conn id)))
+
+       file))))
 
 (defn get-minimal-file
   [{:keys [::db/pool] :as cfg} id]
@@ -285,33 +385,14 @@
    ::cond/key-fn get-file-etag
    ::sm/params schema:get-file
    ::sm/result schema:file-with-permissions}
-  [cfg {:keys [::rpc/profile-id id project-id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (let [perms (get-permissions conn profile-id id)]
-                      (check-read-permissions! perms)
-                      (let [team (teams/get-team conn
-                                                 :profile-id profile-id
-                                                 :project-id project-id
-                                                 :file-id id)
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id features project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [perms (get-permissions conn profile-id id)]
+      (check-read-permissions! perms)
+      (let [file (-> (get-file conn id features project-id)
+                     (assoc :permissions perms))]
+        (vary-meta file assoc ::cond/key (get-file-etag params file))))))
 
-                            file (-> (get-file cfg id :project-id project-id)
-                                     (assoc :permissions perms)
-                                     (check-version!))
-
-                            _    (-> (cfeat/get-team-enabled-features cf/flags team)
-                                     (cfeat/check-client-features! (:features params))
-                                     (cfeat/check-file-features! (:features file) (:features params)))
-
-                            ;; This operation is needed for backward comapatibility with frontends that
-                            ;; does not support pointer-map resolution mechanism; this just resolves the
-                            ;; pointers on backend and return a complete file.
-                            file (if (and (contains? (:features file) "fdata/pointer-map")
-                                          (not (contains? (:features params) "fdata/pointer-map")))
-                                   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                                     (update file :data feat.fdata/process-pointers deref))
-                                   file)]
-
-                        (vary-meta file assoc ::cond/key (get-file-etag params file)))))))
 
 ;; --- COMMAND QUERY: get-file-fragment (by id)
 
@@ -334,11 +415,11 @@
           (update :content blob/decode)))
 
 (sv/defmethod ::get-file-fragment
-  "Retrieve a file fragment by its ID. Only authenticated users."
+  "Retrieve a file by its ID. Only authenticated users."
   {::doc/added "1.17"
    ::sm/params schema:get-file-fragment
    ::sm/result schema:file-fragment}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id fragment-id share-id]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id fragment-id share-id] }]
   (dm/with-open [conn (db/open pool)]
     (let [perms (get-permissions conn profile-id file-id share-id)]
       (check-read-permissions! perms)
@@ -389,6 +470,7 @@
     (projects/check-read-permissions! conn profile-id project-id)
     (get-project-files conn project-id)))
 
+
 ;; --- COMMAND QUERY: has-file-libraries
 
 (declare get-has-file-libraries)
@@ -427,17 +509,10 @@
   "Given the page data and the object-id returns the page data with all
   other not needed objects removed from the `:objects` data
   structure."
-  [page id-or-ids]
-  (update page :objects (fn [objects]
-                          (reduce (fn [result object-id]
-                                    (->> (cfh/get-children-with-self objects object-id)
-                                         (filter some?)
-                                         (d/index-by :id)
-                                         (merge result)))
-                                  {}
-                                  (if (uuid? id-or-ids)
-                                    [id-or-ids]
-                                    id-or-ids)))))
+  [{:keys [objects] :as page} object-id]
+  (let [objects (->> (cph/get-children-with-self objects object-id)
+                     (filter some?))]
+    (assoc page :objects (d/index-by :id objects))))
 
 (defn- prune-thumbnails
   "Given the page data, removes the `:thumbnail` prop from all
@@ -446,42 +521,30 @@
   (update page :objects update-vals #(dissoc % :thumbnail)))
 
 (defn get-page
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id] :as params}]
-
+  [conn {:keys [file-id page-id object-id features]}]
   (when (and (uuid? object-id)
              (not (uuid? page-id)))
     (ex/raise :type :validation
               :code :params-validation
               :hint "page-id is required when object-id is provided"))
 
-  (let [team (teams/get-team conn
-                             :profile-id profile-id
-                             :file-id file-id)
-
-        file (get-file cfg file-id)
-
-        _    (-> (cfeat/get-team-enabled-features cf/flags team)
-                 (cfeat/check-client-features! (:features params))
-                 (cfeat/check-file-features! (:features file) (:features params)))
-
-        page (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-               (let [page-id (or page-id (-> file :data :pages first))
-                     page    (dm/get-in file [:data :pages-index page-id])]
-                 (if (pmap/pointer-map? page)
+  (let [file     (get-file conn file-id features)
+        page-id  (or page-id (-> file :data :pages first))
+        page     (dm/get-in file [:data :pages-index page-id])
+        page     (if (pmap/pointer-map? page)
                    (deref page)
-                   page)))]
-
+                   page)]
     (cond-> (prune-thumbnails page)
-      (some? object-id)
+      (uuid? object-id)
       (prune-objects object-id))))
 
 (def schema:get-page
-  [:map {:title "get-page"}
+  [:map {:title "GetPage"}
    [:file-id ::sm/uuid]
    [:page-id {:optional true} ::sm/uuid]
    [:share-id {:optional true} ::sm/uuid]
-   [:object-id {:optional true} [:or ::sm/uuid ::sm/coll-of-uuid]]
-   [:features {:optional true} ::cfeat/features]])
+   [:object-id {:optional true} ::sm/uuid]
+   [:features {:optional true} schema:features]])
 
 (sv/defmethod ::get-page
   "Retrieves the page data from file and returns it. If no page-id is
@@ -492,15 +555,15 @@
   If you specify the object-id, the page-id parameter becomes
   mandatory.
 
-  Mainly used for rendering purposes on the exporter. It does not
-  accepts client features."
+  Mainly used for rendering purposes."
   {::doc/added "1.17"
    ::sm/params schema:get-page}
-  [cfg {:keys [::rpc/profile-id file-id share-id] :as params}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (check-read-permissions! conn profile-id file-id share-id)
-                (get-page cfg (assoc params :profile-id profile-id)))))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id share-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (let [perms (get-permissions conn profile-id file-id share-id)]
+      (check-read-permissions! perms)
+      (binding [pmap/*load-fn* (partial load-pointer conn file-id)]
+        (get-page conn params)))))
 
 ;; --- COMMAND QUERY: get-team-shared-files
 
@@ -523,50 +586,51 @@
       and p.team_id = ?
     order by f.modified_at desc")
 
-
-(defn- get-library-summary
-  [cfg {:keys [id data] :as file}]
+(defn get-team-shared-files
+  [conn team-id]
   (letfn [(assets-sample [assets limit]
-            (let [sorted-assets (->> (vals assets)
-                                     (sort-by #(str/lower (:name %))))]
-              {:count (count sorted-assets)
-               :sample (into [] (take limit sorted-assets))}))]
+           (let [sorted-assets (->> (vals assets)
+                                    (sort-by #(str/lower (:name %))))]
+             {:count (count sorted-assets)
+              :sample (into [] (take limit sorted-assets))}))
 
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-      (let [load-objects      (fn [component]
-                                (ctf/load-component-objects data component))
-            components-sample (-> (assets-sample (ctkl/components data) 4)
-                                  (update :sample #(mapv load-objects %)))]
-        {:components components-sample
-         :media (assets-sample (:media data) 3)
-         :colors (assets-sample (:colors data) 3)
-         :typographies (assets-sample (:typographies data) 3)}))))
+          (library-summary [{:keys [id data] :as file}]
+            (binding [pmap/*load-fn* (partial load-pointer conn id)]
+              (let [load-objects (fn [component]
+                                   (binding [pmap/*load-fn* (partial load-pointer conn id)]
+                                     (ctf/load-component-objects data component)))
+                    components-sample (-> (assets-sample (ctkl/components data) 4)
+                                          (update :sample
+                                                  #(map load-objects %)))]
+                {:components components-sample
+                 :media (assets-sample (:media data) 3)
+                 :colors (assets-sample (:colors data) 3)
+                 :typographies (assets-sample (:typographies data) 3)})))]
 
-(defn- get-team-shared-files
-  [{:keys [::db/conn] :as cfg} {:keys [team-id profile-id]}]
-  (teams/check-read-permissions! conn profile-id team-id)
-  (->> (db/exec! conn [sql:team-shared-files team-id])
-       (into #{} (comp
-                  (map decode-row)
-                  (map (fn [row]
-                         (if-let [media-id (:media-id row)]
-                           (-> row
-                               (dissoc :media-id)
-                               (assoc :thumbnail-uri (resolve-public-uri media-id)))
-                           (dissoc row :media-id))))
-                  (map #(assoc % :library-summary (get-library-summary cfg %)))
-                  (map #(dissoc % :data))))))
+    (->> (db/exec! conn [sql:team-shared-files team-id])
+         (into #{} (comp
+                    (map decode-row)
+                    (map (fn [row]
+                           (if-let [media-id (:media-id row)]
+                             (-> row
+                                 (dissoc :media-id)
+                                 (assoc :thumbnail-uri (resolve-public-uri media-id)))
+                             (dissoc row :media-id))))
+                    (map #(assoc % :library-summary (library-summary %)))
+                    (map #(dissoc % :data)))))))
 
-(def ^:private schema:get-team-shared-files
-  [:map {:title "get-team-shared-files"}
-   [:team-id ::sm/uuid]])
+(s/def ::get-team-shared-files
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id]))
 
 (sv/defmethod ::get-team-shared-files
   "Get all file (libraries) for the specified team."
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-shared-files}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg get-team-shared-files (assoc params :profile-id profile-id)))
+  {::doc/added "1.17"}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
+  (dm/with-open [conn (db/open pool)]
+    (teams/check-read-permissions! conn profile-id team-id)
+    (get-team-shared-files conn team-id)))
+
 
 ;; --- COMMAND QUERY: get-file-libraries
 
@@ -598,20 +662,17 @@
   [conn file-id]
   (into []
         (comp
-         ;; FIXME: :is-indirect set to false to all rows looks
-         ;; completly useless
          (map #(assoc % :is-indirect false))
          (map decode-row))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
-(def ^:private schema:get-file-libraries
-  [:map {:title "get-file-libraries"}
-   [:file-id ::sm/uuid]])
+(s/def ::get-file-libraries
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id]))
 
 (sv/defmethod ::get-file-libraries
   "Get libraries used by the specified file."
-  {::doc/added "1.17"
-   ::sm/params schema:get-file-libraries}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id file-id)
@@ -632,14 +693,12 @@
   [conn file-id]
   (db/exec! conn [sql:library-using-files file-id]))
 
-(def ^:private schema:get-library-file-references
-  [:map {:title "get-library-file-references"}
-   [:file-id ::sm/uuid]])
+(s/def ::get-library-file-references
+  (s/keys :req [::rpc/profile-id] :req-un [::file-id]))
 
 (sv/defmethod ::get-library-file-references
   "Returns all the file references that use specified file (library) id."
-  {::doc/added "1.17"
-   ::sm/params schema:get-library-file-references}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id file-id)
@@ -679,13 +738,12 @@
                      (assoc :thumbnail-uri (resolve-public-uri media-id)))
                  (dissoc row :media-id))))))
 
-(def ^:private schema:get-team-recent-files
-  [:map {:title "get-team-recent-files"}
-   [:team-id ::sm/uuid]])
+(s/def ::get-team-recent-files
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id]))
 
 (sv/defmethod ::get-team-recent-files
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-recent-files}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (teams/check-read-permissions! conn profile-id team-id)
@@ -694,34 +752,20 @@
 
 ;; --- COMMAND QUERY: get-file-summary
 
-(defn- get-file-summary
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}]
-  (check-read-permissions! conn profile-id id)
-  (let [team (teams/get-team conn
-                             :profile-id profile-id
-                             :project-id project-id
-                             :file-id id)
-
-        file (get-file cfg id :project-id project-id)]
-
-    (-> (cfeat/get-team-enabled-features cf/flags team)
-        (cfeat/check-client-features! (:features params))
-        (cfeat/check-file-features! (:features file) (:features params)))
-
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+(sv/defmethod ::get-file-summary
+  "Retrieve a file summary by its ID. Only authenticated users."
+  {::doc/added "1.20"
+   ::sm/params schema:get-file}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id features project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-read-permissions! conn profile-id id)
+    (let [file (get-file conn id features project-id)]
       {:name             (:name file)
        :components-count (count (ctkl/components-seq (:data file)))
        :graphics-count   (count (get-in file [:data :media] []))
        :colors-count     (count (get-in file [:data :colors] []))
        :typography-count (count (get-in file [:data :typographies] []))})))
 
-(sv/defmethod ::get-file-summary
-  "Retrieve a file summary by its ID. Only authenticated users."
-  {::doc/added "1.20"
-   ::sm/params schema:get-file}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg get-file-summary (assoc params :profile-id profile-id)))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -771,6 +815,16 @@
 
 ;; --- MUTATION COMMAND: set-file-shared
 
+(defn- unlink-files!
+  [conn {:keys [id]}]
+  (db/delete! conn :file-library-rel {:library-file-id id}))
+
+(defn- set-file-shared!
+  [conn {:keys [id is-shared] :as params}]
+  (db/update! conn :file
+              {:is-shared is-shared}
+              {:id id}))
+
 (def sql:get-referenced-files
   "SELECT f.id
      FROM file_library_rel AS flr
@@ -779,172 +833,81 @@
       AND (f.deleted_at IS NULL OR f.deleted_at > now())
     ORDER BY f.created_at ASC;")
 
-(defn- absorb-library-by-file!
-  [cfg ldata file-id]
-
-  (dm/assert!
-   "expected cfg with valid connection"
-   (db/connection-map? cfg))
-
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)
-            pmap/*tracked* (pmap/create-tracked)]
-    (let [file (-> (get-file cfg file-id
-                             :include-deleted? true
-                             :lock-for-update? true)
-                   (update :data ctf/absorb-assets ldata))]
-
-      (l/trc :hint "library absorbed"
-             :library-id (str (:id ldata))
-             :file-id (str file-id))
-
-      (db/update! cfg :file
-                  {:revn (inc (:revn file))
-                   :data (blob/encode (:data file))
-                   :modified-at (dt/now)}
-                  {:id file-id})
-
-      (feat.fdata/persist-pointers! cfg file-id))))
-
 (defn- absorb-library!
   "Find all files using a shared library, and absorb all library assets
   into the file local libraries"
-  [cfg {:keys [id] :as library}]
+  [conn {:keys [id] :as library}]
+  (let [ldata (binding [pmap/*load-fn* (partial load-pointer conn id)]
+                (-> library decode-row (process-pointers deref) pmg/migrate-file :data))
+        rows  (db/exec! conn [sql:get-referenced-files id])]
+    (doseq [file-id (map :id rows)]
+      (binding [pmap/*load-fn* (partial load-pointer conn file-id)
+                pmap/*tracked* (atom {})]
+        (let [file (-> (db/get-by-id conn :file file-id
+                                     ::db/check-deleted? false
+                                     ::db/remove-deleted? false)
+                       (decode-row)
+                       (load-all-pointers!)
+                       (pmg/migrate-file))
+              data (ctf/absorb-assets (:data file) ldata)]
+          (db/update! conn :file
+                      {:revn (inc (:revn file))
+                       :data (blob/encode data)
+                       :modified-at (dt/now)}
+                      {:id file-id})
+          (persist-pointers! conn file-id))))))
 
-  (dm/assert!
-   "expected cfg with valid connection"
-   (db/connection-map? cfg))
-
-  (let [ldata (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                (-> library :data (feat.fdata/process-pointers deref)))
-        ids   (->> (db/exec! cfg [sql:get-referenced-files id])
-                   (map :id))]
-
-    (l/trc :hint "absorbing library"
-           :library-id (str id)
-           :files (str/join "," (map str ids)))
-
-    (run! (partial absorb-library-by-file! cfg ldata) ids)))
-
-(defn- set-file-shared
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
-  (check-edition-permissions! conn profile-id id)
-  (let [file (db/get-by-id conn :file id {:columns [:id :name :is-shared]})
-        file (cond
-               (and (true? (:is-shared file))
-                    (false? (:is-shared params)))
-               ;; When we disable shared flag on an already shared
-               ;; file, we need to perform more complex operation,
-               ;; so in this case we retrieve the complete file and
-               ;; perform all required validations.
-               (let [file (-> (get-file cfg id :lock-for-update? true)
-                              (check-version!)
-                              (assoc :is-shared false))
-                     team (teams/get-team conn
-                                          :profile-id profile-id
-                                          :project-id (:project-id file))]
-
-                 (-> (cfeat/get-team-enabled-features cf/flags team)
-                     (cfeat/check-client-features! (:features params))
-                     (cfeat/check-file-features! (:features file)))
-
-                 (absorb-library! cfg file)
-
-                 (db/delete! conn :file-library-rel {:library-file-id id})
-                 (db/update! conn :file
-                             {:is-shared false}
-                             {:id id})
-                 file)
-
-               (and (false? (:is-shared file))
-                    (true? (:is-shared params)))
-               (let [file (assoc file :is-shared true)]
-                 (db/update! conn :file
-                             {:is-shared true}
-                             {:id id}
-                             ::db/return-keys? false)
-
-                 file)
-
-               :else
-               (ex/raise :type :validation
-                         :code :invalid-shared-state
-                         :hint "unexpected state found"
-                         :params-is-shared (:is-shared params)
-                         :file-is-shared (:is-shared file)))]
-
-    (rph/with-meta
-      (select-keys file [:id :name :is-shared])
-      {::audit/props {:name (:name file)
-                      :project-id (:project-id file)
-                      :is-shared (:is-shared file)}})))
-
-(def ^:private
-  schema:set-file-shared
-  (sm/define
-    [:map {:title "set-file-shared"}
-     [:id ::sm/uuid]
-     [:is-shared :boolean]]))
+(def ^:private schema:set-file-shared
+  [:map {:title "set-file-shared"}
+   [:id ::sm/uuid]
+   [:is-shared :boolean]])
 
 (sv/defmethod ::set-file-shared
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:set-file-shared}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg set-file-shared (assoc params :profile-id profile-id)))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id is-shared] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id id)
+    (let [file (set-file-shared! conn params)]
+      (when-not is-shared
+        (absorb-library! conn file)
+        (unlink-files! conn file))
+
+      (rph/with-meta
+        (select-keys file [:id :name :is-shared])
+        {::audit/props {:name (:name file)
+                        :project-id (:project-id file)
+                        :is-shared (:is-shared file)}}))))
 
 ;; --- MUTATION COMMAND: delete-file
 
 (defn- mark-file-deleted!
-  [conn file-id]
+  [conn {:keys [id]}]
   (db/update! conn :file
               {:deleted-at (dt/now)}
-              {:id file-id}
-              {::db/columns [:id :name :is-shared :project-id :created-at :modified-at]}))
+              {:id id}))
 
-(def ^:private
-  schema:delete-file
-  (sm/define
-    [:map {:title "delete-file"}
-     [:id ::sm/uuid]]))
-
-(defn- delete-file
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
-  (check-edition-permissions! conn profile-id id)
-  (let [file (mark-file-deleted! conn id)]
-
-    ;; NOTE: when a file is a shared library, then we proceed to load
-    ;; the whole file, proceed with feature checking and properly execute
-    ;; the absorb-library procedure
-    (when (:is-shared file)
-      (let [file (-> (get-file cfg id
-                               :lock-for-update? true
-                               :include-deleted? true)
-                     (check-version!))
-
-            team (teams/get-team conn
-                                 :profile-id profile-id
-                                 :project-id (:project-id file))]
-
-
-
-        (-> (cfeat/get-team-enabled-features cf/flags team)
-            (cfeat/check-client-features! (:features params))
-            (cfeat/check-file-features! (:features file)))
-
-        (absorb-library! cfg file)))
-
-    (rph/with-meta (rph/wrap)
-      {::audit/props {:project-id (:project-id file)
-                      :name (:name file)
-                      :created-at (:created-at file)
-                      :modified-at (:modified-at file)}})))
+(def ^:private schema:delete-file
+  [:map {:title "delete-file"}
+   [:id ::sm/uuid]])
 
 (sv/defmethod ::delete-file
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:delete-file}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg delete-file (assoc params :profile-id profile-id)))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (check-edition-permissions! conn profile-id id)
+    (let [file (mark-file-deleted! conn params)]
+      (when (:is-shared file)
+        (absorb-library! conn file))
+
+      (rph/with-meta (rph/wrap)
+        {::audit/props {:project-id (:project-id file)
+                        :name (:name file)
+                        :created-at (:created-at file)
+                        :modified-at (:modified-at file)}}))))
 
 ;; --- MUTATION COMMAND: link-file-to-library
 
@@ -957,17 +920,13 @@
   [conn {:keys [file-id library-id] :as params}]
   (db/exec-one! conn [sql:link-file-to-library file-id library-id]))
 
-(def ^:private
-  schema:link-file-to-library
-  (sm/define
-    [:map {:title "link-file-to-library"}
-     [:file-id ::sm/uuid]
-     [:library-id ::sm/uuid]]))
+(s/def ::link-file-to-library
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::library-id]))
 
 (sv/defmethod ::link-file-to-library
   {::doc/added "1.17"
-   ::webhooks/event? true
-   ::sm/params schema:link-file-to-library}
+   ::webhooks/event? true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id library-id] :as params}]
   (when (= file-id library-id)
     (ex/raise :type :validation
@@ -986,15 +945,13 @@
               {:file-id file-id
                :library-file-id library-id}))
 
-(def ^:private schema:unlink-file-to-library
-  [:map {:title "unlink-file-to-library"}
-   [:file-id ::sm/uuid]
-   [:library-id ::sm/uuid]])
+(s/def ::unlink-file-from-library
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::library-id]))
 
 (sv/defmethod ::unlink-file-from-library
   {::doc/added "1.17"
-   ::webhooks/event? true
-   ::sm/params schema:unlink-file-to-library}
+   ::webhooks/event? true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
     (check-edition-permissions! conn profile-id file-id)
@@ -1010,15 +967,15 @@
               {:file-id file-id
                :library-file-id library-id}))
 
-(def ^:private schema:update-file-library-sync-status
-  [:map {:title "update-file-library-sync-status"}
-   [:file-id ::sm/uuid]
-   [:library-id ::sm/uuid]])
+(s/def ::update-file-library-sync-status
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::library-id]))
+
+;; TODO: improve naming
 
 (sv/defmethod ::update-file-library-sync-status
   "Update the synchronization status of a file->library link"
-  {::doc/added "1.17"
-   ::sm/params schema:update-file-library-sync-status}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
     (check-edition-permissions! conn profile-id file-id)

backend/src/app/rpc/commands/files_create.clj

@@ -7,20 +7,15 @@
 (ns app.rpc.commands.files-create
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.features :as cfeat]
-   [app.common.schema :as sm]
+   [app.common.files.features :as ffeat]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.permissions :as perms]
    [app.rpc.quotes :as quotes]
@@ -29,7 +24,7 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.set :as set]))
+   [clojure.spec.alpha :as s]))
 
 (defn create-file-role!
   [conn {:keys [file-id profile-id role]}]
@@ -39,34 +34,27 @@
          (db/insert! conn :file-profile-rel))))
 
 (defn create-file
-  [{:keys [::db/conn] :as cfg}
-   {:keys [id name project-id is-shared revn
-           modified-at deleted-at create-page
-           ignore-sync-until features]
-    :or {is-shared false revn 0 create-page true}
-    :as params}]
-
-  (dm/assert!
-   "expected a valid connection"
-   (db/connection? conn))
+  [conn {:keys [id name project-id is-shared revn
+                modified-at deleted-at create-page
+                ignore-sync-until features]
+         :or {is-shared false revn 0 create-page true}
+         :as params}]
 
   (let [id       (or id (uuid/next))
+        features (->> features
+                      (into (files/get-default-features))
+                      (files/check-features-compatibility!))
 
-        pointers (pmap/create-tracked)
-        pmap?    (contains? features "fdata/pointer-map")
-        omap?    (contains? features "fdata/objects-map")
-
+        pointers (atom {})
         data     (binding [pmap/*tracked* pointers
-                           cfeat/*current* features
-                           cfeat/*wrap-with-objects-map-fn* (if omap? omap/wrap identity)
-                           cfeat/*wrap-with-pointer-map-fn* (if pmap? pmap/wrap identity)]
+                           ffeat/*current* features
+                           ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity)
+                           ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)]
                    (if create-page
                      (ctf/make-file-data id)
                      (ctf/make-file-data id nil)))
 
-        features (->> (set/difference features cfeat/frontend-only-features)
-                      (db/create-array conn "text"))
-
+        features (db/create-array conn "text" features)
         file     (db/insert! conn :file
                              (d/without-nils
                               {:id id
@@ -81,69 +69,40 @@
                                :deleted-at deleted-at}))]
 
     (binding [pmap/*tracked* pointers]
-      (feat.fdata/persist-pointers! cfg id))
+      (files/persist-pointers! conn id))
 
     (->> (assoc params :file-id id :role :owner)
          (create-file-role! conn))
 
     (db/update! conn :project
-                {:modified-at (dt/now)}
-                {:id project-id})
+      {:modified-at (dt/now)}
+      {:id project-id})
 
     (files/decode-row file)))
 
-(def ^:private schema:create-file
-  [:map {:title "create-file"}
-   [:name :string]
-   [:project-id ::sm/uuid]
-   [:id {:optional true} ::sm/uuid]
-   [:is-shared {:optional true} :boolean]
-   [:features {:optional true} ::cfeat/features]])
+(s/def ::create-file
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::files/name
+                   ::files/project-id]
+          :opt-un [::files/id
+                   ::files/is-shared
+                   ::files/features]))
 
 (sv/defmethod ::create-file
   {::doc/added "1.17"
    ::doc/module :files
-   ::webhooks/event? true
-   ::sm/params schema:create-file}
-  [cfg {:keys [::rpc/profile-id project-id] :as params}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (projects/check-edition-permissions! conn profile-id project-id)
-                (let [team     (teams/get-team conn
-                                               :profile-id profile-id
-                                               :project-id project-id)
-                      team-id  (:id team)
+   ::webhooks/event? true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (projects/check-edition-permissions! conn profile-id project-id)
+    (let [team-id (files/get-team-id conn project-id)
+          params  (assoc params :profile-id profile-id)]
 
-                      ;; When we create files, we only need to respect the team
-                      ;; features, because some features can be enabled
-                      ;; globally, but the team is still not migrated properly.
-                      features (-> (cfeat/get-team-enabled-features cf/flags team)
-                                   (cfeat/check-client-features! (:features params)))
+      (run! (partial quotes/check-quote! conn)
+            (list {::quotes/id ::quotes/files-per-project
+                   ::quotes/team-id team-id
+                   ::quotes/profile-id profile-id
+                   ::quotes/project-id project-id}))
 
-                      ;; We also include all no migration features declared by
-                      ;; client; that enables the ability to enable a runtime
-                      ;; feature on frontend and make it permanent on file
-                      features (-> (:features params #{})
-                                   (set/intersection cfeat/no-migration-features)
-                                   (set/union features))
-
-                      params   (-> params
-                                   (assoc :profile-id profile-id)
-                                   (assoc :features features))]
-
-                  (run! (partial quotes/check-quote! conn)
-                        (list {::quotes/id ::quotes/files-per-project
-                               ::quotes/team-id team-id
-                               ::quotes/profile-id profile-id
-                               ::quotes/project-id project-id}))
-
-                  ;; When newly computed features does not match exactly with
-                  ;; the features defined on team row, we update it.
-                  (when (not= features (:features team))
-                    (let [features (db/create-array conn "text" features)]
-                      (db/update! conn :team
-                                  {:features features}
-                                  {:id team-id})))
-
-                  (-> (create-file cfg params)
-                      (vary-meta assoc ::audit/props {:team-id team-id}))))))
+      (-> (create-file conn params)
+          (vary-meta assoc ::audit/props {:team-id team-id})))))

backend/src/app/rpc/commands/files_snapshot.clj

@@ -74,8 +74,8 @@
                :file-id file-id)
 
       (db/update! conn :file
-                  {:data (:data snapshot)}
-                  {:id file-id})
+                    {:data (:data snapshot)}
+                    {:id file-id})
 
       ;; clean object thumbnails
       (let [sql (str "delete from file_object_thumbnail "

backend/src/app/rpc/commands/files_temp.clj

@@ -7,26 +7,21 @@
 (ns app.rpc.commands.files-temp
   (:require
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.changes :as fch]
+   [app.common.pages :as cp]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-create :as files.create]
+   [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.files-update :as-alias files.update]
    [app.rpc.commands.projects :as projects]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.util.blob :as blob]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [clojure.set :as set]
    [clojure.spec.alpha :as s]))
 
-
 ;; --- MUTATION COMMAND: create-temp-file
 
 (s/def ::create-page ::us/boolean)
@@ -43,35 +38,25 @@
 (sv/defmethod ::create-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [cfg {:keys [::rpc/profile-id project-id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (projects/check-edition-permissions! conn profile-id project-id)
-                    (let [team     (teams/get-team conn
-                                                   :profile-id profile-id
-                                                   :project-id project-id)
-
-                          ;; When we create files, we only need to respect the team
-                          ;; features, because some features can be enabled
-                          ;; globally, but the team is still not migrated properly.
-                          features (-> (cfeat/get-team-enabled-features cf/flags team)
-                                       (cfeat/check-client-features! (:features params)))
-
-                          ;; We also include all no migration features declared by
-                          ;; client; that enables the ability to enable a runtime
-                          ;; feature on frontend and make it permanent on file
-                          features (-> (:features params #{})
-                                       (set/intersection cfeat/no-migration-features)
-                                       (set/union features))
-
-                          params   (-> params
-                                       (assoc :profile-id profile-id)
-                                       (assoc :deleted-at (dt/in-future {:days 1}))
-                                       (assoc :features features))]
-
-                      (files.create/create-file cfg params)))))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
+  (db/with-atomic [conn pool]
+    (projects/check-edition-permissions! conn profile-id project-id)
+    (create-file conn (assoc params :profile-id profile-id :deleted-at (dt/in-future {:days 1})))))
 
 ;; --- MUTATION COMMAND: update-temp-file
 
+(defn update-temp-file
+  [conn {:keys [profile-id session-id id revn changes] :as params}]
+  (db/insert! conn :file-change
+              {:id (uuid/next)
+               :session-id session-id
+               :profile-id profile-id
+               :created-at (dt/now)
+               :file-id id
+               :revn revn
+               :data nil
+               :changes (blob/encode changes)}))
+
 (s/def ::update-temp-file
   (s/keys :req [::rpc/profile-id]
           :req-un [::files.update/changes
@@ -82,18 +67,10 @@
 (sv/defmethod ::update-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (db/insert! conn :file-change
-                                {:id (uuid/next)
-                                 :session-id session-id
-                                 :profile-id profile-id
-                                 :created-at (dt/now)
-                                 :file-id id
-                                 :revn revn
-                                 :data nil
-                                 :changes (blob/encode changes)})
-                    nil)))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (update-temp-file conn (assoc params :profile-id profile-id))
+    nil))
 
 ;; --- MUTATION COMMAND: persist-temp-file
 
@@ -113,7 +90,7 @@
     (let [data
           (->> revs
                (mapcat #(->> % :changes blob/decode))
-               (fch/process-changes (blob/decode (:data file))))]
+               (cp/process-changes (blob/decode (:data file))))]
       (db/update! conn :file
                   {:deleted-at nil
                    :revn revn
@@ -128,7 +105,7 @@
 (sv/defmethod ::persist-temp-file
   {::doc/added "1.17"
    ::doc/module :files}
-  [cfg {:keys [::rpc/profile-id id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (files/check-edition-permissions! conn profile-id id)
-                    (persist-temp-file conn params))))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id id)
+    (persist-temp-file conn params)))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -8,25 +8,22 @@
   (:require
    [app.common.data :as d]
    [app.common.data.macros :as dm]
-   [app.common.features :as cfeat]
-   [app.common.files.helpers :as cfh]
-   [app.common.files.migrations :as fmg]
+   [app.common.exceptions :as ex]
    [app.common.geom.shapes :as gsh]
+   [app.common.pages.helpers :as cph]
    [app.common.schema :as sm]
-   [app.common.thumbnails :as thc]
+   [app.common.spec :as us]
    [app.common.types.shape-tree :as ctt]
-   [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
+   [app.db.sql :as sql]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
    [app.rpc :as-alias rpc]
-   [app.rpc.climit :as-alias climit]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
    [app.storage :as sto]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
@@ -41,60 +38,88 @@
 
 ;; --- COMMAND QUERY: get-file-object-thumbnails
 
-(defn- get-object-thumbnails-by-tag
-  [conn file-id tag]
-  (let [sql (str/concat
-             "select object_id, media_id, tag "
-             "  from file_tagged_object_thumbnail"
-             " where file_id=? and tag=?")
-        res (db/exec! conn [sql file-id tag])]
-    (->> res
-         (d/index-by :object-id (fn [row]
-                                  (files/resolve-public-uri (:media-id row))))
-         (d/without-nils))))
-
 (defn- get-object-thumbnails
   ([conn file-id]
    (let [sql (str/concat
-              "select object_id, media_id, tag "
-              "  from file_tagged_object_thumbnail"
+              "select object_id, data, media_id "
+              "  from file_object_thumbnail"
               " where file_id=?")
          res (db/exec! conn [sql file-id])]
      (->> res
           (d/index-by :object-id (fn [row]
-                                   (files/resolve-public-uri (:media-id row))))
+                                   (or (some-> row :media-id files/resolve-public-uri)
+                                       (:data row))))
           (d/without-nils))))
 
   ([conn file-id object-ids]
    (let [sql (str/concat
-              "select object_id, media_id, tag "
-              "  from file_tagged_object_thumbnail"
+              "select object_id, data, media_id "
+              "  from file_object_thumbnail"
               " where file_id=? and object_id = ANY(?)")
          ids (db/create-array conn "text" (seq object-ids))
          res (db/exec! conn [sql file-id ids])]
-
-     (->> res
-          (d/index-by :object-id (fn [row]
-                                   (files/resolve-public-uri (:media-id row))))
-          (d/without-nils)))))
+     (d/index-by :object-id
+                 (fn [row]
+                   (or (some-> row :media-id files/resolve-public-uri)
+                       (:data row)))
+                 res))))
 
 (sv/defmethod ::get-file-object-thumbnails
   "Retrieve a file object thumbnails."
   {::doc/added "1.17"
    ::doc/module :files
    ::sm/params [:map {:title "get-file-object-thumbnails"}
-                [:file-id ::sm/uuid]
-                [:tag {:optional true} :string]]
+                [:file-id ::sm/uuid]]
    ::sm/result [:map-of :string :string]
    ::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
    ::cond/reuse-key? true
    ::cond/key-fn files/get-file-etag}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id tag] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (dm/with-open [conn (db/open pool)]
     (files/check-read-permissions! conn profile-id file-id)
-    (if tag
-      (get-object-thumbnails-by-tag conn file-id tag)
-      (get-object-thumbnails conn file-id))))
+    (get-object-thumbnails conn file-id)))
+
+;; --- COMMAND QUERY: get-file-thumbnail
+
+(defn get-file-thumbnail
+  [conn file-id revn]
+  (let [sql (sql/select :file-thumbnail
+                        (cond-> {:file-id file-id}
+                          revn (assoc :revn revn))
+                        {:limit 1
+                         :order-by [[:revn :desc]]})
+        row (db/exec-one! conn sql)]
+
+    (when-not row
+      (ex/raise :type :not-found
+                :code :file-thumbnail-not-found))
+
+    (when-not (:data row)
+      (ex/raise :type :not-found
+                :code :file-thumbnail-not-found))
+
+    {:data (:data row)
+     :props (some-> (:props row) db/decode-transit-pgobject)
+     :revn (:revn row)
+     :file-id (:file-id row)}))
+
+(s/def ::revn ::us/integer)
+(s/def ::file-id ::us/uuid)
+
+(s/def ::get-file-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id]
+          :opt-un [::revn]))
+
+(sv/defmethod ::get-file-thumbnail
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::doc/deprecated "1.19"}
+  [{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
+  (dm/with-open [conn (db/open pool)]
+    (files/check-read-permissions! conn profile-id file-id)
+    (-> (get-file-thumbnail conn file-id revn)
+        (rph/with-http-cache long-cache-duration))))
 
 ;; --- COMMAND QUERY: get-file-data-for-thumbnail
 
@@ -102,11 +127,23 @@
 ;; loading all pages into memory for find the frame set for thumbnail.
 
 (defn get-file-data-for-thumbnail
-  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
+  [conn {:keys [data id] :as file}]
   (letfn [;; function responsible on finding the frame marked to be
           ;; used as thumbnail; the returned frame always have
           ;; the :page-id set to the page that it belongs.
-          (get-thumbnail-frame [{:keys [data]}]
+
+          (get-thumbnail-frame [data]
+            ;; NOTE: this is a hack for avoid perform blocking
+            ;; operation inside the for loop, clojure lazy-seq uses
+            ;; synchronized blocks that does not plays well with
+            ;; virtual threads, so we need to perform the load
+            ;; operation first. This operation forces all pointer maps
+            ;; load into the memory.
+            (->> (-> data :pages-index vals)
+                 (filter pmap/pointer-map?)
+                 (run! pmap/load!))
+
+            ;; Then proceed to find the frame set for thumbnail
             (d/seek #(or (:use-for-thumbnail %)
                          (:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21)
                     (for [page  (-> data :pages-index vals)
@@ -117,24 +154,24 @@
           ;; all unneeded shapes if a concrete frame is provided. If no
           ;; frame, the objects is returned untouched.
           (filter-objects [objects frame-id]
-            (d/index-by :id (cfh/get-children-with-self objects frame-id)))
+            (d/index-by :id (cph/get-children-with-self objects frame-id)))
 
           ;; function responsible of assoc available thumbnails
           ;; to frames and remove all children shapes from objects if
           ;; thumbnails is available
           (assoc-thumbnails [objects page-id thumbnails]
             (loop [objects objects
-                   frames  (filter cfh/frame-shape? (vals objects))]
+                   frames  (filter cph/frame-shape? (vals objects))]
 
               (if-let [frame  (-> frames first)]
                 (let [frame-id  (:id frame)
-                      object-id (thc/fmt-object-id (:id file) page-id frame-id "frame")
+                      object-id (str page-id frame-id)
                       frame     (if-let [thumb (get thumbnails object-id)]
                                   (assoc frame :thumbnail thumb :shapes [])
                                   (dissoc frame :thumbnail))
 
                       children-ids
-                      (cfh/get-children-ids objects frame-id)
+                      (cph/get-children-ids objects frame-id)
 
                       bounds
                       (when (:show-content frame)
@@ -155,44 +192,41 @@
 
                 objects)))]
 
-    (let [frame     (get-thumbnail-frame file)
-          frame-id  (:id frame)
-          page-id   (or (:page-id frame)
-                        (-> data :pages first))
+    (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
+      (let [frame     (get-thumbnail-frame data)
+            frame-id  (:id frame)
+            page-id   (or (:page-id frame)
+                          (-> data :pages first))
 
-          page      (dm/get-in data [:pages-index page-id])
-          page      (cond-> page (pmap/pointer-map? page) deref)
-          frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
+            page      (dm/get-in data [:pages-index page-id])
+            page      (cond-> page (pmap/pointer-map? page) deref)
+            frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
 
-          obj-ids   (map #(thc/fmt-object-id (:id file) page-id % "frame") frame-ids)
-          thumbs    (get-object-thumbnails conn id obj-ids)]
+            obj-ids   (map #(str page-id %) frame-ids)
+            thumbs    (get-object-thumbnails conn id obj-ids)]
 
-      (cond-> page
-        ;; If we have frame, we need to specify it on the page level
-        ;; and remove the all other unrelated objects.
-        (some? frame-id)
-        (-> (assoc :thumbnail-frame-id frame-id)
-            (update :objects filter-objects frame-id))
+        (cond-> page
+          ;; If we have frame, we need to specify it on the page level
+          ;; and remove the all other unrelated objects.
+          (some? frame-id)
+          (-> (assoc :thumbnail-frame-id frame-id)
+              (update :objects filter-objects frame-id))
 
-        ;; Assoc the available thumbnails and prune not visible shapes
-        ;; for avoid transfer unnecessary data.
-        :always
-        (update :objects assoc-thumbnails page-id thumbs)))))
+          ;; Assoc the available thumbnails and prune not visible shapes
+          ;; for avoid transfer unnecessary data.
+          :always
+          (update :objects assoc-thumbnails page-id thumbs))))))
 
-(def ^:private
-  schema:get-file-data-for-thumbnail
-  (sm/define
-    [:map {:title "get-file-data-for-thumbnail"}
-     [:file-id ::sm/uuid]
-     [:features {:optional true} ::cfeat/features]]))
+(def ^:private schema:get-file-data-for-thumbnail
+  [:map {:title "get-file-data-for-thumbnail"}
+   [:file-id ::sm/uuid]
+   [:features {:optional true} files/schema:features]])
 
-(def ^:private
-  schema:partial-file
-  (sm/define
-    [:map {:title "PartialFile"}
-     [:id ::sm/uuid]
-     [:revn {:min 0} :int]
-     [:page :any]]))
+(def ^:private schema:partial-file
+  [:map {:title "PartialFile"}
+   [:id ::sm/uuid]
+   [:revn {:min 0} :int]
+   [:page :any]])
 
 (sv/defmethod ::get-file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
@@ -201,42 +235,68 @@
    ::doc/module :files
    ::sm/params schema:get-file-data-for-thumbnail
    ::sm/result schema:partial-file}
-  [cfg {:keys [::rpc/profile-id file-id] :as params}]
-  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                 (files/check-read-permissions! conn profile-id file-id)
-
-                 (let [team     (teams/get-team conn
-                                                :profile-id profile-id
-                                                :file-id file-id)
-
-                       file     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                                  (-> (files/get-file cfg file-id :migrate? false)
-                                      (update :data feat.fdata/process-pointers deref)
-                                      (fmg/migrate-file)))]
-
-                   (-> (cfeat/get-team-enabled-features cf/flags team)
-                       (cfeat/check-client-features! (:features params))
-                       (cfeat/check-file-features! (:features file) (:features params)))
-
-                   {:file-id file-id
-                    :revn (:revn file)
-                    :page (get-file-data-for-thumbnail cfg file)}))))
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
+  (dm/with-open [conn (db/open pool)]
+    (files/check-read-permissions! conn profile-id file-id)
+    ;; NOTE: we force here the "storage/pointer-map" feature, because
+    ;; it used internally only and is independent if user supports it
+    ;; or not.
+    (let [feat (into #{"storage/pointer-map"} features)
+          file (files/get-file conn file-id feat)]
+      {:file-id file-id
+       :revn (:revn file)
+       :page (get-file-data-for-thumbnail conn file)})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; --- MUTATION COMMAND: upsert-file-object-thumbnail
+
+(def sql:upsert-object-thumbnail
+  "insert into file_object_thumbnail(file_id, object_id, data)
+   values (?, ?, ?)
+       on conflict(file_id, object_id) do
+          update set data = ?;")
+
+(defn upsert-file-object-thumbnail!
+  [conn {:keys [file-id object-id data]}]
+  (if data
+    (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
+    (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
+
+(s/def ::data (s/nilable ::us/string))
+(s/def ::object-id ::us/string)
+
+(s/def ::upsert-file-object-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::object-id]
+          :opt-un [::data]))
+
+(sv/defmethod ::upsert-file-object-thumbnail
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::doc/deprecated "1.19"
+   ::audit/skip true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+
+    (when-not (db/read-only? conn)
+      (upsert-file-object-thumbnail! conn params)
+      nil)))
+
+
 ;; --- MUTATION COMMAND: create-file-object-thumbnail
 
 (def ^:private sql:create-object-thumbnail
-  "insert into file_tagged_object_thumbnail(file_id, object_id, media_id, tag)
-   values (?, ?, ?, ?)
-       on conflict(file_id, tag, object_id) do
-          update set media_id = ?
-   returning *;")
+  "insert into file_object_thumbnail(file_id, object_id, media_id)
+   values (?, ?, ?)
+       on conflict(file_id, object_id) do
+          update set media_id = ?;")
 
 (defn- create-file-object-thumbnail!
-  [{:keys [::db/conn ::sto/storage]} file-id object-id media tag]
+  [{:keys [::db/conn ::sto/storage]} file-id object-id media]
 
   (let [path  (:path media)
         mtype (:mtype media)
@@ -245,30 +305,27 @@
                   (sto/wrap-with-hash hash))
         media (sto/put-object! storage
                                {::sto/content data
-                                ::sto/deduplicate? true
-                                ::sto/touched-at (dt/now)
+                                ::sto/deduplicate? false
                                 :content-type mtype
                                 :bucket "file-object-thumbnail"})]
 
     (db/exec-one! conn [sql:create-object-thumbnail file-id object-id
-                        (:id media) tag (:id media)])))
+                        (:id media) (:id media)])))
+
 
 (def schema:create-file-object-thumbnail
   [:map {:title "create-file-object-thumbnail"}
    [:file-id ::sm/uuid]
    [:object-id :string]
-   [:media ::media/upload]
-   [:tag {:optional true} :string]])
+   [:media ::media/upload]])
 
 (sv/defmethod ::create-file-object-thumbnail
-  {::doc/added "1.19"
+  {:doc/added "1.19"
    ::doc/module :files
-   ::climit/id :file-thumbnail-ops
-   ::climit/key-fn ::rpc/profile-id
    ::audit/skip true
    ::sm/params schema:create-file-object-thumbnail}
 
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media tag]}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media]}]
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id file-id)
     (media/validate-media-type! media)
@@ -278,19 +335,21 @@
       (-> cfg
           (update ::sto/storage media/configure-assets-storage)
           (assoc ::db/conn conn)
-          (create-file-object-thumbnail! file-id object-id media (or tag "frame"))))))
+          (create-file-object-thumbnail! file-id object-id media))
+      nil)))
 
 ;; --- MUTATION COMMAND: delete-file-object-thumbnail
 
 (defn- delete-file-object-thumbnail!
   [{:keys [::db/conn ::sto/storage]} file-id object-id]
-  (when-let [{:keys [media-id]} (db/get* conn :file-tagged-object-thumbnail
+  (when-let [{:keys [media-id]} (db/get* conn :file-object-thumbnail
                                          {:file-id file-id
                                           :object-id object-id}
                                          {::db/for-update? true})]
+    (when media-id
+      (sto/del-object! storage media-id))
 
-    (sto/touch-object! storage media-id)
-    (db/delete! conn :file-tagged-object-thumbnail
+    (db/delete! conn :file-object-thumbnail
                 {:file-id file-id
                  :object-id object-id})
     nil))
@@ -300,10 +359,8 @@
           :req-un [::file-id ::object-id]))
 
 (sv/defmethod ::delete-file-object-thumbnail
-  {::doc/added "1.19"
+  {:doc/added "1.19"
    ::doc/module :files
-   ::climit/id :file-thumbnail-ops
-   ::climit/key-fn ::rpc/profile-id
    ::audit/skip true}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}]
 
@@ -317,6 +374,41 @@
           (delete-file-object-thumbnail! file-id object-id))
       nil)))
 
+;; --- MUTATION COMMAND: upsert-file-thumbnail
+
+(def ^:private sql:upsert-file-thumbnail
+  "insert into file_thumbnail (file_id, revn, data, props)
+   values (?, ?, ?, ?::jsonb)
+       on conflict(file_id, revn) do
+          update set data = ?, props=?, updated_at=now();")
+
+(defn- upsert-file-thumbnail!
+  [conn {:keys [file-id revn data props]}]
+  (let [props (db/tjson (or props {}))]
+    (db/exec-one! conn [sql:upsert-file-thumbnail
+                        file-id revn data props data props])))
+
+(s/def ::revn ::us/integer)
+(s/def ::props map?)
+
+(s/def ::upsert-file-thumbnail
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id ::revn ::props ::data]))
+
+(sv/defmethod ::upsert-file-thumbnail
+  "Creates or updates the file thumbnail. Mainly used for paint the
+  grid thumbnails."
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::doc/deprecated "1.19"
+   ::audit/skip true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+    (when-not (db/read-only? conn)
+      (upsert-file-thumbnail! conn params)
+      nil)))
+
 ;; --- MUTATION COMMAND: create-file-thumbnail
 
 (def ^:private sql:create-file-thumbnail
@@ -352,12 +444,11 @@
   {::doc/added "1.19"
    ::doc/module :files
    ::audit/skip true
-   ::climit/id :file-thumbnail-ops
-   ::climit/key-fn ::rpc/profile-id
    ::sm/params [:map {:title "create-file-thumbnail"}
                 [:file-id ::sm/uuid]
                 [:revn :int]
-                [:media ::media/upload]]}
+                [:media ::media/upload]]
+   }
 
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (db/with-atomic [conn pool]

backend/src/app/rpc/commands/files_update.clj

@@ -6,19 +6,18 @@
 
 (ns app.rpc.commands.files-update
   (:require
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.changes :as cpc]
-   [app.common.files.migrations :as fmg]
-   [app.common.files.validate :as val]
+   [app.common.files.features :as ffeat]
+   [app.common.files.migrations :as pmg]
    [app.common.logging :as l]
+   [app.common.pages :as cp]
+   [app.common.pages.changes :as cpc]
    [app.common.schema :as sm]
+   [app.common.schema.generators :as smg]
+   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
-   [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
    [app.metrics :as mtx]
@@ -26,45 +25,46 @@
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as climit]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [app.util.time :as dt]
-   [app.worker :as-alias wrk]
-   [clojure.set :as set]))
+   [app.util.time :as dt]))
 
 ;; --- SCHEMA
 
-(def ^:private
-  schema:update-file
-  (sm/define
-    [:map {:title "update-file"}
-     [:id ::sm/uuid]
-     [:session-id ::sm/uuid]
-     [:revn {:min 0} :int]
-     [:features {:optional true} ::cfeat/features]
-     [:changes {:optional true} [:vector ::cpc/change]]
-     [:changes-with-metadata {:optional true}
-      [:vector [:map
-                [:changes [:vector ::cpc/change]]
-                [:hint-origin {:optional true} :keyword]
-                [:hint-events {:optional true} [:vector :string]]]]]
-     [:skip-validate {:optional true} :boolean]]))
+(sm/def! ::changes
+  [:vector ::cpc/change])
 
-(def ^:private
-  schema:update-file-result
-  (sm/define
-    [:vector {:title "update-file-result"}
-     [:map
-      [:changes [:vector ::cpc/change]]
-      [:file-id ::sm/uuid]
-      [:id ::sm/uuid]
-      [:revn {:min 0} :int]
-      [:session-id ::sm/uuid]]]))
+(sm/def! ::change-with-metadata
+  [:map {:title "ChangeWithMetadata"}
+   [:changes ::changes]
+   [:hint-origin {:optional true} :keyword]
+   [:hint-events {:optional true} [:vector :string]]])
+
+(sm/def! ::update-file-params
+  [:map {:title "UpdateFileParams"}
+   [:id ::sm/uuid]
+   [:session-id ::sm/uuid]
+   [:revn {:min 0} :int]
+   [:features {:optional true
+               :gen/max 3
+               :gen/gen (smg/subseq files/supported-features)}
+    ::sm/set-of-strings]
+   [:changes {:optional true} ::changes]
+   [:changes-with-metadata {:optional true}
+    [:vector ::change-with-metadata]]])
+
+(sm/def! ::update-file-result
+  [:vector {:title "UpdateFileResults"}
+   [:map {:title "UpdateFileResult"}
+    [:changes ::changes]
+    [:file-id ::sm/uuid]
+    [:id ::sm/uuid]
+    [:revn {:min 0} :int]
+    [:session-id ::sm/uuid]]])
 
 ;; --- HELPERS
 
@@ -74,7 +74,7 @@
 (def ^:private library-change-types
   #{:add-color :mod-color :del-color
     :add-media :mod-media :del-media
-    :add-component :mod-component :del-component :restore-component
+    :add-component :mod-component :del-component
     :add-typography :mod-typography :del-typography})
 
 (def ^:private file-change-types
@@ -106,18 +106,18 @@
 
 (defn- wrap-with-pointer-map-context
   [f]
-  (fn [cfg {:keys [id] :as file}]
-    (binding [pmap/*tracked* (pmap/create-tracked)
-              pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-              cfeat/*wrap-with-pointer-map-fn* pmap/wrap]
+  (fn [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+    (binding [pmap/*tracked* (atom {})
+              pmap/*load-fn* (partial files/load-pointer conn id)
+              ffeat/*wrap-with-pointer-map-fn* pmap/wrap]
       (let [result (f cfg file)]
-        (feat.fdata/persist-pointers! cfg id)
+        (files/persist-pointers! conn id)
         result))))
 
 (defn- wrap-with-objects-map-context
   [f]
   (fn [cfg file]
-    (binding [cfeat/*wrap-with-objects-map-fn* omap/wrap]
+    (binding [ffeat/*wrap-with-objects-map-fn* omap/wrap]
       (f cfg file))))
 
 (declare get-lagged-changes)
@@ -132,94 +132,74 @@
 ;; database.
 
 (sv/defmethod ::update-file
-  {::climit/id :update-file/by-profile
-   ::climit/key-fn ::rpc/profile-id
+  {::climit/id :update-file-by-id
+   ::climit/key-fn :id
    ::webhooks/event? true
    ::webhooks/batch-timeout (dt/duration "2m")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
 
-   ::sm/params schema:update-file
-   ::sm/result schema:update-file-result
+   ::sm/params ::update-file-params
+   ::sm/result ::update-file-result
+
    ::doc/module :files
    ::doc/added "1.17"}
-  [cfg {:keys [::rpc/profile-id id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (files/check-edition-permissions! conn profile-id id)
-                    (db/xact-lock! conn id)
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id id)
+    (db/xact-lock! conn id)
 
-                    (let [file     (get-file conn id)
-                          team     (teams/get-team conn
-                                                   :profile-id profile-id
-                                                   :team-id (:team-id file))
-
-                          features (-> (cfeat/get-team-enabled-features cf/flags team)
-                                       (cfeat/check-client-features! (:features params))
-                                       (cfeat/check-file-features! (:features file) (:features params)))
-
-                          params   (assoc params
-                                          :profile-id profile-id
-                                          :features features
-                                          :team team
-                                          :file file)
-
-                          tpoint   (dt/tpoint)]
-
-                      ;; When newly computed features does not match exactly with
-                      ;; the features defined on team row, we update it.
-                      (when (not= features (:features team))
-                        (let [features (db/create-array conn "text" features)]
-                          (db/update! conn :team
-                                      {:features features}
-                                      {:id (:id team)})))
-
-                      (binding [l/*context* (some-> (meta params)
-                                                    (get :app.http/request)
-                                                    (errors/request->context))]
-                        (-> (update-file cfg params)
-                            (rph/with-defer #(let [elapsed (tpoint)]
-                                               (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
+    (let [cfg    (assoc cfg ::db/conn conn)
+          params (assoc params :profile-id profile-id)
+          tpoint (dt/tpoint)]
+      (-> (update-file cfg params)
+          (rph/with-defer #(let [elapsed (tpoint)]
+                             (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))
 
 (defn update-file
-  [{:keys [::db/conn ::mtx/metrics] :as cfg}
-   {:keys [id file features changes changes-with-metadata] :as params}]
-  (let [features  (-> features
-                      (set/difference cfeat/frontend-only-features)
-                      (set/union (:features file)))
+  [{:keys [::db/conn ::mtx/metrics] :as cfg} {:keys [profile-id id changes changes-with-metadata] :as params}]
+  (let [file     (get-file conn id)
+        features (->> (concat (:features file)
+                              (:features params))
+                      (into (files/get-default-features))
+                      (files/check-features-compatibility!))]
 
-        update-fn (cond-> update-file*
-                    (contains? features "fdata/pointer-map")
-                    (wrap-with-pointer-map-context)
+    (files/check-edition-permissions! conn profile-id (:id file))
 
-                    (contains? features "fdata/objects-map")
-                    (wrap-with-objects-map-context))
+    (binding [ffeat/*current*  features
+              ffeat/*previous* (:features file)]
 
-        changes   (if changes-with-metadata
-                    (->> changes-with-metadata (mapcat :changes) vec)
-                    (vec changes))]
+      (let [update-fn (cond-> update-file*
+                        (contains? features "storage/pointer-map")
+                        (wrap-with-pointer-map-context)
 
-    (when (> (:revn params)
-             (:revn file))
-      (ex/raise :type :validation
-                :code :revn-conflict
-                :hint "The incoming revision number is greater that stored version."
-                :context {:incoming-revn (:revn params)
-                          :stored-revn (:revn file)}))
+                        (contains? features "storage/objects-map")
+                        (wrap-with-objects-map-context))
 
-    (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+            file      (assoc file :features features)
+            changes   (if changes-with-metadata
+                        (->> changes-with-metadata (mapcat :changes) vec)
+                        (vec changes))
 
-    (when (not= features (:features file))
-      (let [features (db/create-array conn "text" features)]
-        (db/update! conn :file
-                    {:features features}
-                    {:id id})))
+            params    (-> params
+                          (assoc :file file)
+                          (assoc :changes changes)
+                          (assoc ::created-at (dt/now)))]
 
-    (binding [cfeat/*current*  features
-              cfeat/*previous* (:features file)]
-      (let [file   (assoc file :features features)
-            params (-> params
-                       (assoc :file file)
-                       (assoc :changes changes)
-                       (assoc ::created-at (dt/now)))]
+        (when (> (:revn params)
+                 (:revn file))
+          (ex/raise :type :validation
+                    :code :revn-conflict
+                    :hint "The incoming revision number is greater that stored version."
+                    :context {:incoming-revn (:revn params)
+                              :stored-revn (:revn file)}))
+
+        (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
+        (when (not= features (:features file))
+          (let [features (db/create-array conn "text" features)]
+            (db/update! conn :file
+                        {:features features}
+                        {:id id})))
 
         (-> (update-fn cfg params)
             (vary-meta assoc ::audit/replace-props
@@ -230,15 +210,12 @@
                         :team-id    (:team-id file)}))))))
 
 (defn- update-file*
-  [{:keys [::db/conn ::wrk/executor] :as cfg}
-   {:keys [profile-id file changes session-id ::created-at skip-validate] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file changes session-id ::created-at] :as params}]
   (let [;; Process the file data in the CLIMIT context; scheduling it
         ;; to be executed on a separated executor for avoid to do the
         ;; CPU intensive operation on vthread.
-
-        update-fdata-fn (partial update-file-data cfg file changes skip-validate)
-        file            (-> (climit/configure cfg :update-file/global)
-                            (climit/run! update-fdata-fn executor))]
+        file (-> (climit/configure cfg :update-file)
+                 (climit/submit! (partial update-file-data file changes)))]
 
     (db/insert! conn :file-change
                 {:id (uuid/next)
@@ -271,73 +248,24 @@
       ;; Retrieve and return lagged data
       (get-lagged-changes conn params))))
 
-(defn- soft-validate-file-schema!
-  [file]
-  (try
-    (val/validate-file-schema! file)
-    (catch Throwable cause
-      (l/error :hint "file schema validation error" :cause cause))))
-
-(defn- soft-validate-file!
-  [file libs]
-  (try
-    (val/validate-file! file libs)
-    (catch Throwable cause
-      (l/error :hint "file validation error"
-               :cause cause))))
-
 (defn- update-file-data
-  [{:keys [::db/conn] :as cfg} file changes skip-validate]
-  (let [file (update file :data (fn [data]
-                                  (-> data
-                                      (blob/decode)
-                                      (assoc :id (:id file))
-                                      (fmg/migrate-data)
-                                      (d/without-nils))))
+  [file changes]
+  (-> file
+      (update :revn inc)
+      (update :data (fn [data]
+                      (cond-> data
+                        :always
+                        (-> (blob/decode)
+                            (assoc :id (:id file))
+                            (pmg/migrate-data))
 
-        ;; WARNING: this ruins performance; maybe we need to find
-        ;; some other way to do general validation
-        libs (when (and (or (contains? cf/flags :file-validation)
-                            (contains? cf/flags :soft-file-validation))
-                        (not skip-validate))
-               (->> (files/get-file-libraries conn (:id file))
-                    (into [file] (map (fn [{:keys [id]}]
-                                        (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-                                                  pmap/*tracked* nil]
-                                          (-> (files/get-file cfg id :migrate? false)
-                                              (feat.fdata/process-pointers deref) ; ensure all pointers resolved
-                                              (fmg/migrate-file))))))
-                    (d/index-by :id)))
+                        (and (contains? ffeat/*current* "components/v2")
+                             (not (contains? ffeat/*previous* "components/v2")))
+                        (ctf/migrate-to-components-v2)
 
-        file (-> (files/check-version! file)
-                 (update :revn inc)
-                 (update :data cpc/process-changes changes))]
-
-    (when (contains? cf/flags :soft-file-validation)
-      (soft-validate-file! file libs))
-
-    (when (contains? cf/flags :soft-file-schema-validation)
-      (soft-validate-file-schema! file))
-
-    (when (and (contains? cf/flags :file-validation)
-               (not skip-validate))
-      (val/validate-file! file libs))
-
-    (when (and (contains? cf/flags :file-schema-validation)
-               (not skip-validate))
-      (val/validate-file-schema! file))
-
-    (cond-> file
-      (and (contains? cfeat/*current* "fdata/objects-map")
-           (not (contains? cfeat/*previous* "fdata/objects-map")))
-      (feat.fdata/enable-objects-map)
-
-      (and (contains? cfeat/*current* "fdata/pointer-map")
-           (not (contains? cfeat/*previous* "fdata/pointer-map")))
-      (feat.fdata/enable-pointer-map)
-
-      :always
-      (update :data blob/encode))))
+                        :always
+                        (-> (cp/process-changes changes)
+                            (blob/encode)))))))
 
 (defn- take-snapshot?
   "Defines the rule when file `data` snapshot should be saved."
@@ -366,7 +294,7 @@
        (vec)))
 
 (defn- send-notifications!
-  [cfg {:keys [file team changes session-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [file changes session-id] :as params}]
   (let [lchanges (filter library-change? changes)
         msgbus   (::mbus/msgbus cfg)]
 
@@ -380,12 +308,14 @@
                          :changes changes})
 
     (when (and (:is-shared file) (seq lchanges))
-      (mbus/pub! msgbus
-                 :topic (:id team)
-                 :message {:type :library-change
-                           :profile-id (:profile-id params)
-                           :file-id (:id file)
-                           :session-id session-id
-                           :revn (:revn file)
-                           :modified-at (dt/now)
-                           :changes lchanges}))))
+      (let [team-id (or (:team-id file)
+                        (files/get-team-id conn (:project-id file)))]
+        (mbus/pub! msgbus
+                   :topic team-id
+                   :message {:type :library-change
+                             :profile-id (:profile-id params)
+                             :file-id (:id file)
+                             :session-id session-id
+                             :revn (:revn file)
+                             :modified-at (dt/now)
+                             :changes lchanges})))))

backend/src/app/rpc/commands/fonts.clj

@@ -25,7 +25,6 @@
    [app.storage :as sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]))
 
 (def valid-weight #{100 200 300 400 500 600 700 800 900 950})
@@ -81,8 +80,8 @@
             perms   (files/get-permissions conn profile-id file-id share-id)]
         (files/check-read-permissions! perms)
         (db/query conn :team-font-variant
-                  {:team-id (:team-id project)
-                   :deleted-at nil})))))
+          {:team-id (:team-id project)
+           :deleted-at nil})))))
 
 
 (declare create-font-variant)
@@ -157,11 +156,11 @@
                          :woff1-file-id (:id woff1)
                          :woff2-file-id (:id woff2)
                          :otf-file-id (:id otf)
-                         :ttf-file-id (:id ttf)}))]
+                         :ttf-file-id (:id ttf)}))
+          ]
 
-    (let [data   (-> (climit/configure cfg :process-font/global)
-                     (climit/run! (partial generate-missing! data)
-                                  (::wrk/executor cfg)))
+    (let [data   (-> (climit/configure cfg :process-font)
+                     (climit/submit! (partial generate-missing! data)))
           assets (persist-fonts-files! data)
           result (insert-font-variant! assets)]
       (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))

backend/src/app/rpc/commands/ldap.clj

@@ -78,13 +78,13 @@
                             ::audit/profile-id (:id profile)}))))))
 
 (defn- login-or-register
-  [cfg info]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (or (some->> (:email info)
-                             (profile/get-profile-by-email conn)
-                             (profile/decode-row))
-                    (->> (assoc info :is-active true :is-demo false)
-                         (auth/create-profile! conn)
-                         (auth/create-profile-rels! conn)
-                         (profile/strip-private-attrs))))))
+  [{:keys [::db/pool] :as cfg} info]
+  (db/with-atomic [conn pool]
+    (or (some->> (:email info)
+                 (profile/get-profile-by-email conn)
+                 (profile/decode-row))
+        (->> (assoc info :is-active true :is-demo false)
+             (auth/create-profile! conn)
+             (auth/create-profile-rels! conn)
+             (profile/strip-private-attrs)))))
+

backend/src/app/rpc/commands/management.clj

@@ -9,20 +9,17 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
    [app.common.files.migrations :as pmg]
    [app.common.schema :as sm]
+   [app.common.spec :as us]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
-   [app.http.sse :as sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.binfile :as binfile]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as proj]
-   [app.rpc.commands.teams :as teams]
+   [app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
    [app.rpc.doc :as-alias doc]
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
@@ -30,29 +27,32 @@
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as-alias wrk]
+   [clojure.spec.alpha :as s]
    [clojure.walk :as walk]
-   [promesa.core :as p]
    [promesa.exec :as px]))
 
 ;; --- COMMAND: Duplicate File
 
 (declare duplicate-file)
 
-(def ^:private
-  schema:duplicate-file
-  (sm/define
-    [:map {:title "duplicate-file"}
-     [:file-id ::sm/uuid]
-     [:name {:optional true} :string]]))
+(s/def ::id ::us/uuid)
+(s/def ::project-id ::us/uuid)
+(s/def ::file-id ::us/uuid)
+(s/def ::team-id ::us/uuid)
+(s/def ::name ::us/string)
+
+(s/def ::duplicate-file
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::file-id]
+          :opt-un [::name]))
 
 (sv/defmethod ::duplicate-file
   "Duplicate a single file in the same team."
   {::doc/added "1.16"
-   ::webhooks/event? true
-   ::sm/params schema:duplicate-file}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg duplicate-file (assoc params :profile-id profile-id)))
+   ::webhooks/event? true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (duplicate-file conn (assoc params :profile-id profile-id))))
 
 (defn- remap-id
   [item index key]
@@ -61,7 +61,7 @@
     (assoc key (get index (get item key) (get item key)))))
 
 (defn- process-file
-  [cfg index {:keys [id] :as file}]
+  [conn {:keys [id] :as file} index]
   (letfn [(process-form [form]
             (cond-> form
               ;; Relink library items
@@ -104,45 +104,35 @@
                                  (dissoc k))
                              res)))
                        media
-                       media))
+                       media))]
+    (-> file
+        (update :id #(get index %))
+        (update :data
+                (fn [data]
+                  (binding [pmap/*load-fn* (partial files/load-pointer conn id)
+                            pmap/*tracked* (atom {})]
+                    (let [file-id (get index id)
+                          data    (-> data
+                                      (blob/decode)
+                                      (assoc :id file-id)
+                                      (pmg/migrate-data)
+                                      (update :pages-index relink-shapes)
+                                      (update :components relink-shapes)
+                                      (update :media relink-media)
+                                      (d/without-nils)
+                                      (files/process-pointers pmap/clone)
+                                      (blob/encode))]
+                      (files/persist-pointers! conn file-id)
+                      data)))))))
 
-          (process-file [{:keys [id] :as file}]
-            (-> file
-                (update :data assoc :id id)
-                (update :data feat.fdata/process-pointers deref)
-                (pmg/migrate-file)
-                (update :data (fn [data]
-                                (-> data
-                                    (update :pages-index relink-shapes)
-                                    (update :components relink-shapes)
-                                    (update :media relink-media)
-                                    (d/without-nils))))))]
-
-    (let [new-id (get index id)
-          file   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                   (-> (assoc file :id new-id)
-                       (process-file)))
-
-          file (if (contains? (:features file) "fdata/objects-map")
-                 (feat.fdata/enable-objects-map file)
-                 file)
-
-          file (if (contains? (:features file) "fdata/pointer-map")
-                 (binding [pmap/*tracked* (pmap/create-tracked)]
-                   (let [file (feat.fdata/enable-pointer-map file)]
-                     (feat.fdata/persist-pointers! cfg (:id file))
-                     file))
-                 file)]
-      file)))
-
-(def sql:get-used-libraries
+(def sql:retrieve-used-libraries
   "select flr.*
      from file_library_rel as flr
     inner join file as l on (flr.library_file_id = l.id)
     where flr.file_id = ?
       and l.deleted_at is null")
 
-(def sql:get-used-media-objects
+(def sql:retrieve-used-media-objects
   "select fmo.*
      from file_media_object as fmo
     inner join storage_object as so on (fmo.media_id = so.id)
@@ -150,9 +140,9 @@
       and so.deleted_at is null")
 
 (defn duplicate-file*
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
-  (let [flibs    (or flibs (db/exec! conn [sql:get-used-libraries (:id file)]))
-        fmeds    (or fmeds (db/exec! conn [sql:get-used-media-objects (:id file)]))
+  [conn {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
+  (let [flibs    (or flibs (db/exec! conn [sql:retrieve-used-libraries (:id file)]))
+        fmeds    (or fmeds (db/exec! conn [sql:retrieve-used-media-objects (:id file)]))
 
         ;; memo uniform creation/modification date
         now      (dt/now)
@@ -193,61 +183,54 @@
                     (assoc :modified-at now)
                     (assoc :ignore-sync-until ignore))
 
-        file    (process-file cfg index file)]
-
-    (db/insert! conn :file
-                (-> file
-                    (update :features #(db/create-array conn "text" %))
-                    (update :data blob/encode))
-                {::db/return-keys? false})
+        file    (process-file conn file index)]
 
+    (db/insert! conn :file file)
     (db/insert! conn :file-profile-rel
                 {:file-id (:id file)
                  :profile-id profile-id
                  :is-owner true
                  :is-admin true
-                 :can-edit true}
-                {::db/return-keys? false})
+                 :can-edit true})
 
     (doseq [params flibs]
-      (db/insert! conn :file-library-rel params ::db/return-keys? false))
+      (db/insert! conn :file-library-rel params))
 
     (doseq [params fmeds]
-      (db/insert! conn :file-media-object params ::db/return-keys? false))
+      (db/insert! conn :file-media-object params))
 
     file))
 
 (defn duplicate-file
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id] :as params}]
-  (let [;; We don't touch the original file on duplication
-        file   (files/get-file cfg file-id :migrate? false)
+  [conn {:keys [profile-id file-id] :as params}]
+  (let [file   (db/get-by-id conn :file file-id)
         index  {file-id (uuid/next)}
         params (assoc params :index index :file file)]
     (proj/check-edition-permissions! conn profile-id (:project-id file))
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-    (duplicate-file* cfg params {:reset-shared-flag true})))
+    (-> (duplicate-file* conn params {:reset-shared-flag true})
+        (update :data blob/decode)
+        (update :features db/decode-pgarray #{}))))
 
 ;; --- COMMAND: Duplicate Project
 
 (declare duplicate-project)
 
-(def ^:private
-  schema:duplicate-project
-  (sm/define
-    [:map {:title "duplicate-project"}
-     [:project-id ::sm/uuid]
-     [:name {:optional true} :string]]))
+(s/def ::duplicate-project
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::project-id]
+          :opt-un [::name]))
 
 (sv/defmethod ::duplicate-project
   "Duplicate an entire project with all the files"
   {::doc/added "1.16"
-   ::webhooks/event? true
-   ::sm/params schema:duplicate-project}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg duplicate-project (assoc params :profile-id profile-id)))
+   ::webhooks/event? true}
+  [{:keys [::db/pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (duplicate-project conn (assoc params :profile-id (::rpc/profile-id params)))))
 
 (defn duplicate-project
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id project-id name] :as params}]
+  [conn {:keys [profile-id project-id name] :as params}]
 
   ;; Defer all constraints
   (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
@@ -256,9 +239,9 @@
                     (assoc :is-pinned false))
 
         files   (db/query conn :file
-                          {:project-id (:id project)
-                           :deleted-at nil}
-                          {:columns [:id]})
+                  {:project-id (:id project)
+                   :deleted-at nil}
+                  {:columns [:id]})
 
         project (cond-> project
                   (string? name)
@@ -272,8 +255,8 @@
 
     ;; create the duplicated project and assign the current profile as
     ;; a project owner
-    (teams/create-project conn project)
-    (teams/create-project-role conn profile-id (:id project) :owner)
+    (create-project conn project)
+    (create-project-role conn profile-id (:id project) :owner)
 
     ;; duplicate all files
     (let [index  (reduce #(assoc %1 (:id %2) (uuid/next)) {} files)
@@ -282,18 +265,18 @@
                      (assoc :project-id (:id project))
                      (assoc :index index))]
       (doseq [{:keys [id]} files]
-        (let [file   (files/get-file cfg id :migrate? false)
+        (let [file   (db/get-by-id conn :file id)
               params (assoc params :file file)
               opts   {:reset-shared-flag false}]
-          (duplicate-file* cfg params opts))))
+          (duplicate-file* conn params opts))))
 
     ;; return the created project
     project))
 
 ;; --- COMMAND: Move file
 
-(def sql:get-files
-  "select id, features, project_id from file where id = ANY(?)")
+(def sql:retrieve-files
+  "select id, project_id from file where id = ANY(?)")
 
 (def sql:move-files
   "update file set project_id = ? where id = ANY(?)")
@@ -314,20 +297,14 @@
       and rel.library_file_id = br.library_file_id")
 
 (defn move-files
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id ids project-id] :as params}]
+  [conn {:keys [profile-id ids project-id] :as params}]
 
   (let [fids    (db/create-array conn "uuid" ids)
-        files   (->> (db/exec! conn [sql:get-files fids])
-                     (map files/decode-row))
+        files   (db/exec! conn [sql:retrieve-files fids])
         source  (into #{} (map :project-id) files)
         pids    (->> (conj source project-id)
                      (db/create-array conn "uuid"))]
 
-    (when (contains? source project-id)
-      (ex/raise :type :validation
-                :code :cant-move-to-same-project
-                :hint "Unable to move a file to the same project"))
-
     ;; Check if we have permissions on the destination project
     (proj/check-edition-permissions! conn profile-id project-id)
 
@@ -335,15 +312,10 @@
     (doseq [project-id source]
       (proj/check-edition-permissions! conn profile-id project-id))
 
-    ;; Check the team compatibility
-    (let [orig-team (teams/get-team conn :profile-id profile-id :project-id (first source))
-          dest-team (teams/get-team conn :profile-id profile-id :project-id project-id)]
-      (cfeat/check-teams-compatibility! orig-team dest-team)
-
-      ;; Check if all pending to move files are compaib
-      (let [features (cfeat/get-team-enabled-features cf/flags dest-team)]
-        (doseq [file files]
-          (cfeat/check-file-features! features (:features file)))))
+    (when (contains? source project-id)
+      (ex/raise :type :validation
+                :code :cant-move-to-same-project
+                :hint "Unable to move a file to the same project"))
 
     ;; move all files to the project
     (db/exec-one! conn [sql:move-files project-id fids])
@@ -365,51 +337,36 @@
 
     nil))
 
-(def ^:private
-  schema:move-files
-  (sm/define
-    [:map {:title "move-files"}
-     [:ids ::sm/set-of-uuid]
-     [:project-id ::sm/uuid]]))
+(s/def ::ids (s/every ::us/uuid :kind set?))
+(s/def ::move-files
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::ids ::project-id]))
 
 (sv/defmethod ::move-files
   "Move a set of files from one project to other."
   {::doc/added "1.16"
-   ::webhooks/event? true
-   ::sm/params schema:move-files}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg #(move-files % (assoc params :profile-id profile-id))))
+   ::webhooks/event? true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (move-files conn (assoc params :profile-id profile-id))))
 
 ;; --- COMMAND: Move project
 
 (defn move-project
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id team-id project-id] :as params}]
+  [conn {:keys [profile-id team-id project-id] :as params}]
   (let [project (db/get-by-id conn :project project-id {:columns [:id :team-id]})
         pids    (->> (db/query conn :project {:team-id (:team-id project)} {:columns [:id]})
                      (map :id)
                      (db/create-array conn "uuid"))]
 
+    (teams/check-edition-permissions! conn profile-id (:team-id project))
+    (teams/check-edition-permissions! conn profile-id team-id)
+
     (when (= team-id (:team-id project))
       (ex/raise :type :validation
                 :code :cant-move-to-same-team
                 :hint "Unable to move a project to same team"))
 
-    (teams/check-edition-permissions! conn profile-id (:team-id project))
-    (teams/check-edition-permissions! conn profile-id team-id)
-
-    ;; Check the teams compatibility
-    (let [orig-team (teams/get-team conn :profile-id profile-id :team-id (:team-id project))
-          dest-team (teams/get-team conn :profile-id profile-id :team-id team-id)]
-      (cfeat/check-teams-compatibility! orig-team dest-team)
-
-      ;; Check if all pending to move files are compaib
-      (let [features (cfeat/get-team-enabled-features cf/flags dest-team)]
-        (doseq [file (->> (db/query conn :file
-                                    {:project-id project-id}
-                                    {:columns [:features]})
-                          (map files/decode-row))]
-          (cfeat/check-file-features! features (:features file)))))
-
     ;; move project to the destination team
     (db/update! conn :project
                 {:team-id team-id}
@@ -420,73 +377,67 @@
 
     nil))
 
-(def ^:private
-  schema:move-project
-  (sm/define
-    [:map {:title "move-project"}
-     [:team-id ::sm/uuid]
-     [:project-id ::sm/uuid]]))
+(s/def ::move-project
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id ::project-id]))
 
 (sv/defmethod ::move-project
-  "Move projects between teams"
+  "Move projects between teams."
   {::doc/added "1.16"
-   ::webhooks/event? true
-   ::sm/params schema:move-project}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg #(move-project % (assoc params :profile-id profile-id))))
+   ::webhooks/event? true}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (move-project conn (assoc params :profile-id profile-id))))
 
 ;; --- COMMAND: Clone Template
 
-(def ^:private
-  schema:clone-template
-  (sm/define
-    [:map {:title "clone-template"}
-     [:project-id ::sm/uuid]
-     [:template-id ::sm/word-string]]))
-
-(declare ^:private clone-template)
-
-(sv/defmethod ::clone-template
-  "Clone into the specified project the template by its id."
-  {::doc/added "1.16"
-   ::sse/stream? true
-   ::webhooks/event? true
-   ::sm/params schema:clone-template}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id template-id] :as params}]
-  (let [project   (db/get-by-id pool :project project-id {:columns [:id :team-id]})
-        _         (teams/check-edition-permissions! pool profile-id (:team-id project))
-        template  (tmpl/get-template-stream cfg template-id)
-        params    (-> cfg
-                      (assoc ::binfile/input template)
-                      (assoc ::binfile/project-id (:id project))
-                      (assoc ::binfile/profile-id profile-id)
-                      (assoc ::binfile/ignore-index-errors? true)
-                      (assoc ::binfile/migrate? true))]
+(defn- clone-template!
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id template-id project-id]}]
+  (let [template (tmpl/get-template-stream cfg template-id)
+        project  (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
 
     (when-not template
       (ex/raise :type :not-found
                 :code :template-not-found
                 :hint "template not found"))
 
-    (sse/response #(clone-template params))))
+    (teams/check-edition-permissions! conn profile-id (:team-id project))
 
-(defn- clone-template
-  [{:keys [::wrk/executor ::binfile/project-id] :as params}]
-  (db/tx-run! params
-              (fn [{:keys [::db/conn] :as params}]
-                ;; NOTE: the importation process performs some operations that
-                ;; are not very friendly with virtual threads, and for avoid
-                ;; unexpected blocking of other concurrent operations we
-                ;; dispatch that operation to a dedicated executor.
-                (let [result (p/thread-call executor (partial binfile/import! params))]
-                  (db/update! conn :project
-                              {:modified-at (dt/now)}
-                              {:id project-id})
+    (-> cfg
+        ;; FIXME: maybe reuse the conn instead of creating more
+        ;; connections in the import process?
+        (dissoc ::db/conn)
+        (assoc ::binfile/input template)
+        (assoc ::binfile/project-id (:id project))
+        (assoc ::binfile/ignore-index-errors? true)
+        (assoc ::binfile/migrate? true)
+        (binfile/import!))))
 
-                  (deref result)))))
+(def schema:clone-template
+  [:map {:title "clone-template"}
+   [:project-id ::sm/uuid]
+   [:template-id ::sm/word-string]])
+
+(sv/defmethod ::clone-template
+  "Clone into the specified project the template by its id."
+  {::doc/added "1.16"
+   ::webhooks/event? true
+   ::sm/params schema:clone-template}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (-> (assoc cfg ::db/conn conn)
+        (clone-template! (assoc params :profile-id profile-id)))))
 
 ;; --- COMMAND: Get list of builtin templates
 
+(s/def ::retrieve-list-of-builtin-templates any?)
+
+(sv/defmethod ::retrieve-list-of-builtin-templates
+  {::doc/added "1.10"
+   ::doc/deprecated "1.19"}
+  [cfg _params]
+  (mapv #(select-keys % [:id :name]) (::setup/templates cfg)))
+
 (sv/defmethod ::get-builtin-templates
   {::doc/added "1.19"}
   [cfg _params]

backend/src/app/rpc/commands/media.clj

@@ -23,7 +23,6 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.services :as sv]
-   [app.worker :as-alias wrk]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.io :as io]))
@@ -61,7 +60,7 @@
     (files/check-edition-permissions! pool profile-id file-id)
     (media/validate-media-type! content)
     (media/validate-media-size! content)
-    (let [object (db/run! cfg #(create-file-media-object % params))
+    (let [object (create-file-media-object cfg params)
           props  {:name (:name params)
                   :file-id file-id
                   :is-local (:is-local params)
@@ -143,17 +142,17 @@
       (assoc ::image (process-main-image info)))))
 
 (defn create-file-media-object
-  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
+  [{:keys [::sto/storage ::db/pool] :as cfg}
    {:keys [id file-id is-local name content]}]
 
-  (let [result (-> (climit/configure cfg :process-image/global)
-                   (climit/run! (partial process-image content) executor))
+  (let [result (-> (climit/configure cfg :process-image)
+                   (climit/submit! (partial process-image content)))
 
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]
 
-    (db/exec-one! conn [sql:create-file-media-object
+    (db/exec-one! pool [sql:create-file-media-object
                         (or id (uuid/next))
                         file-id is-local name
                         (:id image)
@@ -177,9 +176,9 @@
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
     (files/check-edition-permissions! pool profile-id file-id)
-    (db/run! cfg #(create-file-media-object-from-url % params))))
+    (create-file-media-object-from-url cfg params)))
 
-(defn download-image
+(defn- download-image
   [{:keys [::http/client]} uri]
   (letfn [(parse-and-validate [{:keys [headers] :as response}]
             (let [size     (some-> (get headers "content-length") d/parse-integer)
@@ -210,6 +209,7 @@
                                                  {:method :get :uri uri}
                                                  {:response-type :input-stream :sync? true})
           {:keys [size mtype]} (parse-and-validate response)
+
           path    (tmp/tempfile :prefix "penpot.media.download.")
           written (io/write-to-file! body path :size size)]
 
@@ -223,6 +223,7 @@
        :path path
        :mtype mtype})))
 
+
 (defn- create-file-media-object-from-url
   [cfg {:keys [url name] :as params}]
   (let [content (download-image cfg url)

backend/src/app/rpc/commands/profile.clj

@@ -8,6 +8,7 @@
   (:require
    [app.auth :as auth]
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.common.uuid :as uuid]
@@ -26,7 +27,6 @@
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as-alias wrk]
    [cuerdas.core :as str]))
 
 (declare check-profile-existence!)
@@ -37,30 +37,30 @@
 (declare strip-private-attrs)
 (declare verify-password)
 
-(def ^:private
-  schema:profile
-  (sm/define
-    [:map {:title "Profile"}
-     [:id ::sm/uuid]
-     [:fullname [::sm/word-string {:max 250}]]
-     [:email ::sm/email]
-     [:is-active {:optional true} :boolean]
-     [:is-blocked {:optional true} :boolean]
-     [:is-demo {:optional true} :boolean]
-     [:is-muted {:optional true} :boolean]
-     [:created-at {:optional true} ::sm/inst]
-     [:modified-at {:optional true} ::sm/inst]
-     [:default-project-id {:optional true} ::sm/uuid]
-     [:default-team-id {:optional true} ::sm/uuid]
-     [:props {:optional true}
-      [:map-of {:title "ProfileProps"} :keyword :any]]]))
+(def schema:profile
+  [:map {:title "Profile"}
+   [:id ::sm/uuid]
+   [:fullname [::sm/word-string {:max 250}]]
+   [:email ::sm/email]
+   [:is-active {:optional true} :boolean]
+   [:is-blocked {:optional true} :boolean]
+   [:is-demo {:optional true} :boolean]
+   [:is-muted {:optional true} :boolean]
+   [:created-at {:optional true} ::sm/inst]
+   [:modified-at {:optional true} ::sm/inst]
+   [:default-project-id {:optional true} ::sm/uuid]
+   [:default-team-id {:optional true} ::sm/uuid]
+   [:props {:optional true}
+    [:map-of {:title "ProfileProps"} :keyword :any]]])
+
+(def profile?
+  (sm/pred-fn schema:profile))
 
 ;; --- QUERY: Get profile (own)
 
 (sv/defmethod ::get-profile
   {::rpc/auth false
    ::doc/added "1.18"
-   ::sm/params [:map]
    ::sm/result schema:profile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id]}]
   ;; We need to return the anonymous profile object in two cases, when
@@ -81,13 +81,11 @@
 
 ;; --- MUTATION: Update Profile (own)
 
-(def ^:private
-  schema:update-profile
-  (sm/define
-    [:map {:title "update-profile"}
-     [:fullname [::sm/word-string {:max 250}]]
-     [:lang {:optional true} [:string {:max 5}]]
-     [:theme {:optional true} [:string {:max 250}]]]))
+(def schema:update-profile
+  [:map {:title "update-profile"}
+   [:fullname [::sm/word-string {:max 250}]]
+   [:lang {:optional true} [:string {:max 5}]]
+   [:theme {:optional true} [:string {:max 250}]]])
 
 (sv/defmethod ::update-profile
   {::doc/added "1.0"
@@ -95,6 +93,10 @@
    ::sm/result schema:profile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id fullname lang theme] :as params}]
 
+  (dm/assert!
+   "expected valid profile data"
+   (profile? params))
+
   (db/with-atomic [conn pool]
     ;; NOTE: we need to retrieve the profile independently if we use
     ;; it or not for explicit locking and avoid concurrent updates of
@@ -106,7 +108,8 @@
           profile (-> profile
                       (assoc :fullname fullname)
                       (assoc :lang lang)
-                      (assoc :theme theme))]
+                      (assoc :theme theme))
+          ]
 
       (db/update! conn :profile
                   {:fullname fullname
@@ -127,13 +130,11 @@
 (declare update-profile-password!)
 (declare invalidate-profile-session!)
 
-(def ^:private
-  schema:update-profile-password
-  (sm/define
-    [:map {:title "update-profile-password"}
-     [:password [::sm/word-string {:max 500}]]
-     ;; Social registered users don't have old-password
-     [:old-password {:optional true} [:maybe [::sm/word-string {:max 500}]]]]))
+(def schema:update-profile-password
+  [:map {:title "update-profile-password"}
+   [:password [::sm/word-string {:max 500}]]
+   ;; Social registered users don't have old-password
+   [:old-password {:optional true} [:maybe [::sm/word-string {:max 500}]]]])
 
 (sv/defmethod ::update-profile-password
   {:doc/added "1.0"
@@ -183,11 +184,9 @@
 (declare upload-photo)
 (declare update-profile-photo)
 
-(def ^:private
-  schema:update-profile-photo
-  (sm/define
-    [:map {:title "update-profile-photo"}
-     [:file ::media/upload]]))
+(def schema:update-profile-photo
+  [:map {:title "update-profile-photo"}
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-profile-photo
   {:doc/added "1.1"
@@ -238,9 +237,9 @@
      :content-type (:mtype thumb)}))
 
 (defn upload-photo
-  [{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file]}]
-  (let [params (-> (climit/configure cfg :process-image/global)
-                   (climit/run! (partial generate-thumbnail! file) executor))]
+  [{:keys [::sto/storage] :as cfg} {:keys [file]}]
+  (let [params (-> (climit/configure cfg :process-image)
+                   (climit/submit! (partial generate-thumbnail! file)))]
     (sto/put-object! storage params)))
 
 
@@ -249,11 +248,9 @@
 (declare ^:private request-email-change!)
 (declare ^:private change-email-immediately!)
 
-(def ^:private
-  schema:request-email-change
-  (sm/define
-    [:map {:title "request-email-change"}
-     [:email ::sm/email]]))
+(def schema:request-email-change
+  [:map {:title "request-email-change"}
+   [:email ::sm/email]])
 
 (sv/defmethod ::request-email-change
   {::doc/added "1.0"
@@ -318,11 +315,9 @@
 
 ;; --- MUTATION: Update Profile Props
 
-(def ^:private
-  schema:update-profile-props
-  (sm/define
-    [:map {:title "update-profile-props"}
-     [:props [:map-of :keyword :any]]]))
+(def schema:update-profile-props
+  [:map {:title "update-profile-props"}
+   [:props [:map-of :keyword :any]]])
 
 (sv/defmethod ::update-profile-props
   {::doc/added "1.0"
@@ -438,15 +433,13 @@
 (defn derive-password
   [cfg password]
   (when password
-    (-> (climit/configure cfg :derive-password/global)
-        (climit/run! (partial auth/derive-password password)
-                     (::wrk/executor cfg)))))
+    (-> (climit/configure cfg :derive-password)
+        (climit/submit! (partial auth/derive-password password)))))
 
 (defn verify-password
   [cfg password password-data]
-  (-> (climit/configure cfg :derive-password/global)
-      (climit/run! (partial auth/verify-password password password-data)
-                   (::wrk/executor cfg))))
+  (-> (climit/configure cfg :derive-password)
+      (climit/submit! (partial auth/verify-password password password-data))))
 
 (defn decode-row
   [{:keys [props] :as row}]

backend/src/app/rpc/commands/projects.clj

@@ -50,11 +50,11 @@
         is-owner (boolean (some :is-owner rows))
         is-admin (boolean (some :is-admin rows))
         can-edit (boolean (some :can-edit rows))]
-    (when (seq rows)
-      {:is-owner is-owner
-       :is-admin (or is-owner is-admin)
-       :can-edit (or is-owner is-admin can-edit)
-       :can-read true})))
+     (when (seq rows)
+       {:is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true})))
 
 (def has-edit-permissions?
   (perms/make-edition-predicate-fn get-permissions))

backend/src/app/rpc/commands/search.clj

@@ -9,7 +9,6 @@
    [app.common.spec :as us]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :refer [resolve-public-uri]]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
    [clojure.spec.alpha :as s]))
@@ -38,15 +37,12 @@
    )
    select distinct
           f.id,
-          f.revn,
           f.project_id,
           f.created_at,
           f.modified_at,
           f.name,
-          f.is_shared,
-          ft.media_id
+          f.is_shared
      from file as f
-     left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn)
     inner join projects as pr on (f.project_id = pr.id)
     where f.name ilike ('%' || ? || '%')
       and (f.deleted_at is null or f.deleted_at > now())
@@ -54,16 +50,10 @@
 
 (defn search-files
   [conn profile-id team-id search-term]
-  (->> (db/exec! conn [sql:search-files
-                       profile-id team-id
-                       profile-id team-id
-                       search-term])
-       (mapv (fn [row]
-               (if-let [media-id (:media-id row)]
-                 (-> row
-                     (dissoc :media-id)
-                     (assoc :thumbnail-uri (resolve-public-uri media-id)))
-                 (dissoc row :media-id))))))
+  (db/exec! conn [sql:search-files
+                  profile-id team-id
+                  profile-id team-id
+                  search-term]))
 
 (s/def ::team-id ::us/uuid)
 (s/def ::search-files ::us/string)

backend/src/app/rpc/commands/teams.clj

@@ -9,7 +9,6 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.spec :as us]
@@ -56,11 +55,11 @@
         is-owner (boolean (some :is-owner rows))
         is-admin (boolean (some :is-admin rows))
         can-edit (boolean (some :can-edit rows))]
-    (when (seq rows)
-      {:is-owner is-owner
-       :is-admin (or is-owner is-admin)
-       :can-edit (or is-owner is-admin can-edit)
-       :can-read true})))
+     (when (seq rows)
+       {:is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true})))
 
 (def has-admin-permissions?
   (perms/make-admin-predicate-fn get-permissions))
@@ -80,26 +79,22 @@
 (def check-read-permissions!
   (perms/make-check-fn has-read-permissions?))
 
-(defn decode-row
-  [{:keys [features] :as row}]
-  (cond-> row
-    (some? features) (assoc :features (db/decode-pgarray features #{}))))
-
 ;; --- Query: Teams
 
-(declare get-teams)
+(declare retrieve-teams)
 
-(def ^:private schema:get-teams
-  [:map {:title "get-teams"}])
+(def counter (volatile! 0))
+
+(s/def ::get-teams
+  (s/keys :req [::rpc/profile-id]))
 
 (sv/defmethod ::get-teams
-  {::doc/added "1.17"
-   ::sm/params schema:get-teams}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
   (dm/with-open [conn (db/open pool)]
-    (get-teams conn profile-id)))
+    (retrieve-teams conn profile-id)))
 
-(def sql:get-teams-with-permissions
+(def sql:teams
   "select t.*,
           tp.is_owner,
           tp.is_admin,
@@ -124,77 +119,37 @@
         (dissoc :is-owner :is-admin :can-edit)
         (assoc :permissions permissions))))
 
-(defn get-teams
+(defn retrieve-teams
   [conn profile-id]
   (let [profile (profile/get-profile conn profile-id)]
-    (->> (db/exec! conn [sql:get-teams-with-permissions (:default-team-id profile) profile-id])
-         (map decode-row)
-         (map process-permissions)
-         (vec))))
+    (->> (db/exec! conn [sql:teams (:default-team-id profile) profile-id])
+         (mapv process-permissions))))
 
 ;; --- Query: Team (by ID)
 
-(declare get-team)
+(declare retrieve-team)
 
-(def ^:private schema:get-team
-  [:and
-   [:map {:title "get-team"}
-    [:id {:optional true} ::sm/uuid]
-    [:file-id {:optional true} ::sm/uuid]]
-
-   [:fn (fn [params]
-          (or (contains? params :id)
-              (contains? params :file-id)))]])
+(s/def ::get-team
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::id]))
 
 (sv/defmethod ::get-team
-  {::doc/added "1.17"
-   ::sm/params schema:get-team}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id id file-id]}]
-  (get-team pool :profile-id profile-id :team-id id :file-id file-id))
+  {::doc/added "1.17"}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
+  (dm/with-open [conn (db/open pool)]
+    (retrieve-team conn profile-id id)))
 
-(defn get-team
-  [conn & {:keys [profile-id team-id project-id file-id] :as params}]
-
-  (dm/assert!
-   "connection or pool is mandatory"
-   (or (db/connection? conn)
-       (db/pool? conn)))
-
-  (dm/assert!
-   "profile-id is mandatory"
-   (uuid? profile-id))
-
-  (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
-        result (cond
-                 (some? team-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
-                                ") SELECT * FROM teams WHERE id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id team-id]))
-
-                 (some? project-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                " WHERE p.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id project-id]))
-
-                 (some? file-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                "  JOIN file AS f ON (f.project_id = p.id) "
-                                " WHERE f.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id file-id]))
-
-                 :else
-                 (throw (IllegalArgumentException. "invalid arguments")))]
+(defn retrieve-team
+  [conn profile-id team-id]
+  (let [profile (profile/get-profile conn profile-id)
+        sql     (str "WITH teams AS (" sql:teams ") SELECT * FROM teams WHERE id=?")
+        result  (db/exec-one! conn [sql (:default-team-id profile) profile-id team-id])]
 
     (when-not result
       (ex/raise :type :not-found
                 :code :team-does-not-exist))
-    (-> result
-        (decode-row)
-        (process-permissions))))
+
+    (process-permissions result)))
 
 ;; --- Query: Team Members
 
@@ -210,48 +165,44 @@
      join profile as p on (p.id = tp.profile_id)
     where tp.team_id = ?")
 
-(defn get-team-members
+(defn retrieve-team-members
   [conn team-id]
   (db/exec! conn [sql:team-members team-id]))
 
-(def ^:private schema:get-team-memebrs
-  [:map {:title "get-team-members"}
-   [:team-id ::sm/uuid]])
+(s/def ::team-id ::us/uuid)
+(s/def ::get-team-members
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id]))
 
 (sv/defmethod ::get-team-members
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-memebrs}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
-    (get-team-members conn team-id)))
+    (retrieve-team-members conn team-id)))
+
 
 ;; --- Query: Team Users
 
-(declare get-users)
-(declare get-team-for-file)
+(declare retrieve-users)
+(declare retrieve-team-for-file)
 
-(def ^:private schema:get-team-users
-  [:and {:title "get-team-users"}
-   [:map
-    [:team-id {:optional true} ::sm/uuid]
-    [:file-id {:optional true} ::sm/uuid]]
-   [:fn #(or (contains? % :team-id)
-             (contains? % :file-id))]])
+(s/def ::get-team-users
+  (s/and (s/keys :req [::rpc/profile-id]
+                 :opt-un [::team-id ::file-id])
+         #(or (:team-id %) (:file-id %))))
 
 (sv/defmethod ::get-team-users
-  "Get team users by team-id or by file-id"
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-users}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (if team-id
       (do
         (check-read-permissions! conn profile-id team-id)
-        (get-users conn team-id))
-      (let [{team-id :id} (get-team-for-file conn file-id)]
+        (retrieve-users conn team-id))
+      (let [{team-id :id} (retrieve-team-for-file conn file-id)]
         (check-read-permissions! conn profile-id team-id)
-        (get-users conn team-id)))))
+        (retrieve-users conn team-id)))))
 
 ;; This is a similar query to team members but can contain more data
 ;; because some user can be explicitly added to project or file (not
@@ -282,44 +233,44 @@
      join file as f on (p.id = f.project_id)
     where f.id = ?")
 
-(defn get-users
+(defn retrieve-users
   [conn team-id]
   (db/exec! conn [sql:team-users team-id team-id team-id]))
 
-(defn get-team-for-file
+(defn retrieve-team-for-file
   [conn file-id]
   (->> [sql:team-by-file file-id]
        (db/exec-one! conn)))
 
 ;; --- Query: Team Stats
 
-(declare get-team-stats)
+(declare retrieve-team-stats)
 
-(def ^:private schema:get-team-stats
-  [:map {:title "get-team-stats"}
-   [:team-id ::sm/uuid]])
+(s/def ::get-team-stats
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id]))
 
 (sv/defmethod ::get-team-stats
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-stats}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
-    (get-team-stats conn team-id)))
+    (retrieve-team-stats conn team-id)))
 
 (def sql:team-stats
   "select (select count(*) from project where team_id = ?) as projects,
           (select count(*) from file as f join project as p on (p.id = f.project_id) where p.team_id = ?) as files")
 
-(defn get-team-stats
+(defn retrieve-team-stats
   [conn team-id]
   (db/exec-one! conn [sql:team-stats team-id team-id]))
 
+
 ;; --- Query: Team invitations
 
-(def ^:private schema:get-team-invitations
-  [:map {:title "get-team-invitations"}
-   [:team-id ::sm/uuid]])
+(s/def ::get-team-invitations
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::team-id]))
 
 (def sql:team-invitations
   "select email_to as email, role, (valid_until < now()) as expired
@@ -331,8 +282,7 @@
        (mapv #(update % :role keyword))))
 
 (sv/defmethod ::get-team-invitations
-  {::doc/added "1.17"
-   ::sm/params schema:get-team-invitations}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id team-id)
@@ -347,32 +297,25 @@
 (declare ^:private create-team-role)
 (declare ^:private create-team-default-project)
 
-(def ^:private schema:create-team
-  [:map {:title "create-team"}
-   [:name :string]
-   [:features {:optional true} ::cfeat/features]
-   [:id {:optional true} ::sm/uuid]])
+(s/def ::create-team
+  (s/keys :req [::rpc/profile-id]
+          :req-un [::name]
+          :opt-un [::id]))
 
 (sv/defmethod ::create-team
-  {::doc/added "1.17"
-   ::sm/params schema:create-team}
-  [cfg {:keys [::rpc/profile-id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
-                                               ::quotes/profile-id profile-id})
+  {::doc/added "1.17"}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (db/with-atomic [conn pool]
+    (quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
+                               ::quotes/profile-id profile-id})
 
-                    (let [features (-> (cfeat/get-enabled-features cf/flags)
-                                       (cfeat/check-client-features! (:features params)))]
-                      (create-team cfg (assoc params
-                                              :profile-id profile-id
-                                              :features features))))))
+    (create-team conn (assoc params :profile-id profile-id))))
 
 (defn create-team
   "This is a complete team creation process, it creates the team
   object and all related objects (default role and default project)."
-  [cfg-or-conn params]
-  (let [conn    (db/get-connection cfg-or-conn)
-        team    (create-team* conn params)
+  [conn params]
+  (let [team    (create-team* conn params)
         params  (assoc params
                        :team-id (:id team)
                        :role :owner)
@@ -381,16 +324,13 @@
     (assoc team :default-project-id (:id project))))
 
 (defn- create-team*
-  [conn {:keys [id name is-default features] :as params}]
+  [conn {:keys [id name is-default] :as params}]
   (let [id         (or id (uuid/next))
-        is-default (if (boolean? is-default) is-default false)
-        features   (db/create-array conn "text" features)
-        team       (db/insert! conn :team
-                               {:id id
-                                :name name
-                                :features features
-                                :is-default is-default})]
-    (decode-row team)))
+        is-default (if (boolean? is-default) is-default false)]
+    (db/insert! conn :team
+                {:id id
+                 :name name
+                 :is-default is-default})))
 
 (defn- create-team-role
   [conn {:keys [profile-id team-id role] :as params}]
@@ -456,7 +396,7 @@
 (defn leave-team
   [conn {:keys [profile-id id reassign-to]}]
   (let [perms   (get-permissions conn profile-id id)
-        members (get-team-members conn id)]
+        members (retrieve-team-members conn id)]
 
     (cond
       ;; we can only proceed if there are more members in the team
@@ -540,15 +480,10 @@
 
 (s/def ::team-id ::us/uuid)
 (s/def ::member-id ::us/uuid)
-(s/def ::role #{:owner :admin :editor})
-
 ;; Temporarily disabled viewer role
 ;; https://tree.taiga.io/project/penpot/issue/1083
-(def valid-roles
-  #{:owner :admin :editor #_:viewer})
-
-(def schema:role
-  [::sm/one-of valid-roles])
+;; (s/def ::role #{:owner :admin :editor :viewer})
+(s/def ::role #{:owner :admin :editor})
 
 (defn role->params
   [role]
@@ -565,7 +500,7 @@
   ;; convenience, if this becomes a bottleneck or problematic,
   ;; we will change it to more efficient fetch mechanisms.
   (let [perms   (get-permissions conn profile-id team-id)
-        members (get-team-members conn team-id)
+        members (retrieve-team-members conn team-id)
         member  (d/seek #(= member-id (:id %)) members)
 
         is-owner? (:is-owner perms)
@@ -661,7 +596,7 @@
 
 (defn update-team-photo
   [{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id team-id] :as params}]
-  (let [team  (get-team pool :profile-id profile-id :team-id team-id)
+  (let [team  (retrieve-team pool profile-id team-id)
         photo (profile/upload-photo cfg params)]
 
     (db/with-atomic [conn pool]
@@ -673,8 +608,8 @@
 
       ;; Save new photo
       (db/update! pool :team
-                  {:photo-id (:id photo)}
-                  {:id team-id})
+        {:photo-id (:id photo)}
+        {:id team-id})
 
       (assoc team :photo-id (:id photo)))))
 
@@ -735,8 +670,7 @@
                           (role->params role))]
 
         ;; Insert the invited member to the team
-        (db/insert! conn :team-profile-rel params
-                    {::db/on-conflict-do-nothing? true})
+        (db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
 
         ;; If profile is not yet verified, mark it as verified because
         ;; accepting an invitation link serves as verification.
@@ -850,24 +784,14 @@
   (s/merge ::create-team
            (s/keys :req-un [::emails ::role])))
 
-
-(def ^:private schema:create-team-with-invitations
-  [:map {:title "create-team-with-invitations"}
-   [:name :string]
-   [:features {:optional true} ::cfeat/features]
-   [:id {:optional true} ::sm/uuid]
-   [:emails ::sm/set-of-emails]
-   [:role schema:role]])
-
 (sv/defmethod ::create-team-with-invitations
-  {::doc/added "1.17"
-   ::sm/params schema:create-team-with-invitations}
+  {::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id emails role] :as params}]
   (db/with-atomic [conn pool]
     (let [params   (assoc params :profile-id profile-id)
-          cfg      (assoc cfg ::db/conn conn)
-          team     (create-team cfg params)
-          profile  (db/get-by-id conn :profile profile-id)]
+          team     (create-team conn params)
+          profile  (db/get-by-id conn :profile profile-id)
+          cfg      (assoc cfg ::db/conn conn)]
 
       ;; Create invitations for all provided emails.
       (->> emails

backend/src/app/rpc/commands/verify_token.clj

@@ -105,7 +105,7 @@
                           ::quotes/team-id team-id})
 
     ;; Insert the invited member to the team
-    (db/insert! conn :team-profile-rel params {::db/on-conflict-do-nothing? true})
+    (db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
 
     ;; If profile is not yet verified, mark it as verified because
     ;; accepting an invitation link serves as verification.

backend/src/app/rpc/commands/viewer.clj

@@ -6,49 +6,27 @@
 
 (ns app.rpc.commands.viewer
   (:require
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
    [app.common.schema :as sm]
-   [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.comments :as comments]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]))
 
 ;; --- QUERY: View Only Bundle
 
-(defn- remove-not-allowed-pages
-  [data allowed]
-  (-> data
-      (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
-      (update :pages-index select-keys allowed)))
-
-(defn- get-view-only-bundle
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
-  (let [file    (files/get-file cfg file-id)
-
-        project (db/get conn :project
-                        {:id (:project-id file)}
-                        {:columns [:id :name :team-id]})
-
-        team    (-> (db/get conn :team {:id (:team-id project)})
-                    (teams/decode-row))
-
-        _       (-> (cfeat/get-team-enabled-features cf/flags team)
-                    (cfeat/check-client-features! (:features params))
-                    (cfeat/check-file-features! (:features file)))
-
-        file    (cond-> file
-                  (= :share-link (:type perms))
-                  (update :data remove-not-allowed-pages (:pages perms))
-
-                  :always
-                  (update :data select-keys [:id :options :pages :pages-index :components]))
+(defn- get-project
+  [conn id]
+  (db/get-by-id conn :project id {:columns [:id :name :team-id]}))
 
+(defn- get-bundle
+  [conn file-id profile-id features]
+  (let [file    (files/get-file conn file-id features)
+        project (get-project conn (:project-id file))
         libs    (files/get-file-libraries conn file-id)
         users   (comments/get-file-comments-users conn file-id profile-id)
         links   (->> (db/query conn :share-link {:file-id file-id})
@@ -63,43 +41,54 @@
                                  (dissoc :flags)))))
 
         fonts   (db/query conn :team-font-variant
-                          {:team-id (:id team)
+                          {:team-id (:team-id project)
                            :deleted-at nil})]
 
-    {:users users
+    {:file file
+     :users users
      :fonts fonts
      :project project
      :share-links links
-     :libraries libs
-     :file file
-     :team team
-     :permissions perms}))
+     :libraries libs}))
 
-(def schema:get-view-only-bundle
+(defn- remove-not-allowed-pages
+  [data allowed]
+  (-> data
+      (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
+      (update :pages-index select-keys allowed)))
+
+(defn get-view-only-bundle
+  [conn {:keys [profile-id file-id share-id features] :as params}]
+  (let [perms  (files/get-permissions conn profile-id file-id share-id)
+        bundle (-> (get-bundle conn file-id profile-id features)
+                   (assoc :permissions perms))]
+
+    ;; When we have neither profile nor share, we just return a not
+    ;; found response to the user.
+    (when-not perms
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "object not found"))
+
+    (update bundle :file
+            (fn [file]
+              (cond-> file
+                (= :share-link (:type perms))
+                (update :data remove-not-allowed-pages (:pages perms))
+
+                :always
+                (update :data select-keys [:id :options :pages :pages-index :components]))))))
+
+(sm/def! ::get-view-only-bundle
   [:map {:title "get-view-only-bundle"}
    [:file-id ::sm/uuid]
    [:share-id {:optional true} ::sm/uuid]
-   [:features {:optional true} ::cfeat/features]])
+   [:features {:optional true} files/schema:features]])
 
 (sv/defmethod ::get-view-only-bundle
   {::rpc/auth false
    ::doc/added "1.17"
-   ::sm/params schema:get-view-only-bundle}
-  [system {:keys [::rpc/profile-id file-id share-id] :as params}]
-  (db/run! system
-           (fn [{:keys [::db/conn] :as system}]
-             (let [perms  (files/get-permissions conn profile-id file-id share-id)
-                   params (-> params
-                              (assoc ::perms perms)
-                              (assoc :profile-id profile-id))]
-
-                  ;; When we have neither profile nor share, we just return a not
-               ;; found response to the user.
-               (when-not perms
-                 (ex/raise :type :not-found
-                           :code :object-not-found
-                           :hint "object not found"))
-
-               (get-view-only-bundle system params)))))
-
-
+   ::sm/params ::get-view-only-bundle}
+  [{:keys [::db/pool]} {:keys [::rpc/profile-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (get-view-only-bundle conn (assoc params :profile-id profile-id))))

backend/src/app/rpc/cond.clj

@@ -29,7 +29,7 @@
    [app.util.services :as-alias sv]
    [buddy.core.codecs :as bc]
    [buddy.core.hash :as bh]
-   [ring.response :as-alias rres]))
+   [yetti.response :as yrs]))
 
 (def
   ^{:dynamic true
@@ -57,7 +57,7 @@
           (let [key' (when (or key reuse-key?)
                        (some->> (get-object cfg params) (key-fn params) (fmt-key)))]
             (if (and (some? key) (= key key'))
-              (fn [_] {::rres/status 304})
+              (fn [_] {::yrs/status 304})
               (let [result (f cfg params)
                     etag   (or (and reuse-key? key')
                                (some-> result meta ::key fmt-key)

backend/src/app/rpc/doc.clj

@@ -16,7 +16,6 @@
    [app.common.schema.openapi :as oapi]
    [app.common.schema.registry :as sr]
    [app.config :as cf]
-   [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.util.json :as json]
@@ -28,7 +27,7 @@
    [integrant.core :as ig]
    [malli.transform :as mt]
    [pretty-spec.core :as ps]
-   [ring.response :as-alias rres]))
+   [yetti.response :as yrs]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DOC (human readable)
@@ -56,7 +55,6 @@
              :module (or (some-> (::module mdata) d/name)
                          (-> (:ns mdata) (str/split ".") last))
              :auth (::rpc/auth mdata true)
-             :sse (::sse/stream? mdata false)
              :webhook (::webhooks/event? mdata false)
              :docs (::sv/docstring mdata)
              :deprecated (::deprecated mdata)
@@ -88,11 +86,11 @@
       (let [params  (:query-params request)
             pstyle  (:type params "js")
             context (assoc context :param-style pstyle)]
-        {::rres/status 200
-         ::rres/body (-> (io/resource "app/templates/api-doc.tmpl")
-                         (tmpl/render context))}))
+        {::yrs/status 200
+         ::yrs/body (-> (io/resource "app/templates/api-doc.tmpl")
+                        (tmpl/render context))}))
     (fn [_]
-      {::rres/status 404})))
+      {::yrs/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OPENAPI / SWAGGER (v3.1)
@@ -143,7 +141,8 @@
 
               {:name (-> mdata ::sv/name d/name)
                :module (-> (:ns mdata) (str/split ".") last)
-               :repr {:post rpost}}))]
+               :repr {:post rpost}}))
+          ]
 
     (let [definitions (atom {})
           options {:registry sr/default-registry
@@ -159,27 +158,27 @@
                          (map (fn [doc]
                                 [(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
                          (into {})))]
-      {:openapi "3.0.0"
-       :info {:version (:main cf/version)}
-       :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
+    {:openapi "3.0.0"
+     :info {:version (:main cf/version)}
+     :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
                 ;; :description "penpot backend"
-                  }]
-       :security
-       {:api_key []}
+                }]
+     :security
+     {:api_key []}
 
-       :paths paths
-       :components {:schemas @definitions}})))
+     :paths paths
+     :components {:schemas @definitions}})))
 
 (defn openapi-json-handler
   [context]
   (if (contains? cf/flags :backend-openapi-doc)
     (fn [_]
-      {::rres/status 200
-       ::rres/headers {"content-type" "application/json; charset=utf-8"}
-       ::rres/body (json/encode context)})
+      {::yrs/status 200
+       ::yrs/headers {"content-type" "application/json; charset=utf-8"}
+       ::yrs/body (json/encode context)})
 
     (fn [_]
-      {::rres/status 404})))
+      {::yrs/status 404})))
 
 (defn openapi-handler
   []
@@ -190,12 +189,12 @@
             context    {:public-uri (cf/get :public-uri)
                         :swagger-js swagger-js
                         :swagger-css swagger-cs}]
-        {::rres/status 200
-         ::rres/headers {"content-type" "text/html"}
-         ::rres/body (-> (io/resource "app/templates/openapi.tmpl")
-                         (tmpl/render context))}))
+        {::yrs/status 200
+         ::yrs/headers {"content-type" "text/html"}
+         ::yrs/body (-> (io/resource "app/templates/openapi.tmpl")
+                        (tmpl/render context))}))
     (fn [_]
-      {::rres/status 404})))
+      {::yrs/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MODULE INIT

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [ring.response :as-alias rres]))
+   [yetti.response :as-alias yrs]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -77,4 +77,4 @@
              (fn [_ response]
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
-                 (update response ::rres/headers assoc "cache-control" val)))))
+                 (update response ::yrs/headers assoc "cache-control" val)))))

backend/src/app/setup/templates.clj

@@ -19,18 +19,14 @@
    [datoteka.fs :as fs]
    [integrant.core :as ig]))
 
-(def ^:private
-  schema:template
-  (sm/define
-    [:map {:title "Template"}
-     [:id ::sm/word-string]
-     [:name ::sm/word-string]
-     [:file-uri ::sm/word-string]]))
+(def ^:private schema:template
+  [:map {:title "Template"}
+   [:id ::sm/word-string]
+   [:name ::sm/word-string]
+   [:file-uri ::sm/word-string]])
 
-(def ^:private
-  schema:templates
-  (sm/define
-    [:vector schema:template]))
+(def ^:private schema:templates
+  [:vector schema:template])
 
 (defmethod ig/init-key ::setup/templates
   [_ _]
@@ -39,13 +35,13 @@
 
     (dm/verify!
      "expected a valid templates file"
-     (sm/check! schema:templates templates))
+     (sm/valid? schema:templates templates))
 
     (doseq [{:keys [id path] :as template} templates]
       (let [path (or path (fs/join dest id))]
         (if (fs/exists? path)
-          (l/dbg :hint "template file" :id id :state "present" :path (dm/str path))
-          (l/dbg :hint "template file" :id id :state "absent"))))
+          (l/debug :hint "template file" :id id :state "present" :path (dm/str path))
+          (l/debug :hint "template file" :id id :state "absent"))))
 
     templates))
 

backend/src/app/srepl.clj

@@ -10,7 +10,7 @@
    [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cf]
-   [app.srepl.cli]
+   [app.srepl.ext]
    [app.srepl.main]
    [app.util.json :as json]
    [app.util.locks :as locks]
@@ -36,9 +36,7 @@
         lock (locks/create)]
     (ccs/prepl *in*
                (fn [m]
-                 (binding [*out* out,
-                           *flush-on-newline* true,
-                           *print-readably* true]
+                 (binding [*out* out, *flush-on-newline* true, *print-readably* true]
                    (locks/locking lock
                      (println (json/encode-str m))))))))
 
@@ -46,10 +44,13 @@
 
 (s/def ::port ::us/integer)
 (s/def ::host ::us/not-empty-string)
+(s/def ::flag #{:urepl-server :prepl-server})
+(s/def ::type #{::prepl ::urepl})
+(s/def ::key (s/tuple ::type ::us/keyword))
 
 (defmethod ig/pre-init-spec ::server
   [_]
-  (s/keys :req [::host ::port]))
+  (s/keys :req [::flag ::host ::port]))
 
 (defmethod ig/prep-key ::server
   [[type _] cfg]
@@ -58,12 +59,6 @@
 (defmethod ig/init-key ::server
   [[type _] {:keys [::flag ::port ::host] :as cfg}]
   (when (contains? cf/flags flag)
-
-    (l/inf :hint "initializing repl server"
-           :name (name type)
-           :port port
-           :host host)
-
     (let [accept (case type
                    ::prepl 'app.srepl/json-repl
                    ::urepl 'app.srepl/user-repl)
@@ -72,8 +67,14 @@
                   :name (name type)
                   :accept accept}]
 
+      (l/info :msg "initializing repl server"
+              :name (name type)
+              :port port
+              :host host)
+
       (ccs/start-server params)
-      (assoc params :type type))))
+
+      params)))
 
 (defmethod ig/halt-key! ::server
   [_ params]

backend/src/app/srepl/components_v2.clj

@@ -1,296 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.srepl.components-v2
-  (:require
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
-   [app.common.logging :as l]
-   [app.common.pprint :as pp]
-   [app.db :as db]
-   [app.features.components-v2 :as feat]
-   [app.util.time :as dt]
-   [cuerdas.core :as str]
-   [promesa.core :as p]
-   [promesa.exec :as px]
-   [promesa.exec.semaphore :as ps]
-   [promesa.util :as pu]))
-
-(defn- print-stats!
-  [stats]
-  (->> stats
-       (into (sorted-map))
-       (pp/pprint)))
-
-(defn- report-progress-files
-  [tpoint]
-  (fn [_ _ oldv newv]
-    (when (not= (:processed/files oldv)
-                (:processed/files newv))
-      (let [total     (:total/files newv)
-            completed (:processed/files newv)
-            progress  (/ (* completed 100.0) total)
-            elapsed   (tpoint)]
-        (l/dbg :hint "progress"
-               :completed (:processed/files newv)
-               :total     (:total/files newv)
-               :progress  (str (int progress) "%")
-               :elapsed   (dt/format-duration elapsed))))))
-
-(defn- report-progress-teams
-  [tpoint on-progress]
-  (fn [_ _ oldv newv]
-    (when (not= (:processed/teams oldv)
-                (:processed/teams newv))
-      (let [total     (:total/teams newv)
-            completed (:processed/teams newv)
-            progress  (/ (* completed 100.0) total)
-            progress  (str (int progress) "%")
-            elapsed   (dt/format-duration (tpoint))]
-
-        (when (fn? on-progress)
-          (on-progress {:total total
-                        :elapsed elapsed
-                        :completed completed
-                        :progress progress}))
-
-        (l/dbg :hint "progress"
-               :completed completed
-               :progress progress
-               :elapsed elapsed)))))
-
-(defn- get-total-files
-  [pool & {:keys [team-id]}]
-  (if (some? team-id)
-    (let [sql (str/concat
-               "SELECT count(f.id) AS count FROM file AS f "
-               "  JOIN project AS p ON (p.id = f.project_id) "
-               " WHERE p.team_id = ? AND f.deleted_at IS NULL "
-               "  AND p.deleted_at IS NULL")
-          res (db/exec-one! pool [sql team-id])]
-      (:count res))
-
-    (let [sql (str/concat
-               "SELECT count(id) AS count FROM file "
-               " WHERE deleted_at IS NULL")
-          res (db/exec-one! pool [sql])]
-      (:count res))))
-
-(defn- get-total-teams
-  [pool]
-  (let [sql (str/concat
-             "SELECT count(id) AS count FROM team "
-             " WHERE deleted_at IS NULL")
-        res (db/exec-one! pool [sql])]
-    (:count res)))
-
-(defn migrate-file!
-  [system file-id & {:keys [rollback?] :or {rollback? true}}]
-
-  (l/dbg :hint "migrate:start")
-  (let [tpoint (dt/tpoint)]
-    (try
-      (binding [feat/*stats* (atom {})]
-        (-> (assoc system ::db/rollback rollback?)
-            (feat/migrate-file! file-id))
-
-        (-> (deref feat/*stats*)
-            (assoc :elapsed (dt/format-duration (tpoint)))))
-
-      (catch Throwable cause
-        (l/wrn :hint "migrate:error" :cause cause))
-
-      (finally
-        (let [elapsed (dt/format-duration (tpoint))]
-          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
-
-(defn migrate-files!
-  [{:keys [::db/pool] :as system}
-   & {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
-      :or {chunk-size 10
-           skip-on-error true
-           max-jobs 10
-           max-items Long/MAX_VALUE
-           preset :shutdown-on-failure
-           rollback? true
-           validate? false}}]
-  (letfn [(get-chunk [cursor]
-            (let [sql  (str/concat
-                        "SELECT id, created_at FROM file "
-                        " WHERE created_at < ? AND deleted_at IS NULL "
-                        " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! pool [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (seq rows)]))
-
-          (get-candidates []
-            (->> (d/iteration get-chunk
-                              :vf second
-                              :kf first
-                              :initk (or start-at (dt/now)))
-                 (take max-items)
-                 (map :id)))]
-
-    (l/dbg :hint "migrate:start")
-    (let [fsem   (ps/create :permits max-jobs)
-          total  (get-total-files pool)
-          stats  (atom {:files/total total})
-          tpoint (dt/tpoint)]
-
-      (add-watch stats :progress-report (report-progress-files tpoint))
-
-      (binding [feat/*stats* stats
-                feat/*semaphore* fsem
-                feat/*skip-on-error* skip-on-error]
-        (try
-          (pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)]
-
-            (run! (fn [file-id]
-                    (ps/acquire! feat/*semaphore*)
-                    (px/submit! scope (fn []
-                                        (-> (assoc system ::db/rollback rollback?)
-                                            (feat/migrate-file! file-id
-                                                                :validate? validate?
-                                                                :throw-on-validate? (not skip-on-error))))))
-                  (get-candidates))
-
-            (p/await! scope))
-
-          (-> (deref feat/*stats*)
-              (assoc :elapsed (dt/format-duration (tpoint))))
-
-          (catch Throwable cause
-            (l/dbg :hint "migrate:error" :cause cause))
-
-          (finally
-            (let [elapsed (dt/format-duration (tpoint))]
-              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))
-
-(defn migrate-team!
-  [{:keys [::db/pool] :as system} team-id
-   & {:keys [rollback? skip-on-error validate?]
-      :or {rollback? true skip-on-error true validate? false}}]
-  (l/dbg :hint "migrate:start")
-
-  (let [total  (get-total-files pool :team-id team-id)
-        stats  (atom {:total/files total})
-        tpoint (dt/tpoint)]
-
-    (add-watch stats :progress-report (report-progress-files tpoint))
-
-    (try
-      (binding [feat/*stats* stats
-                feat/*skip-on-error* skip-on-error]
-        (-> (assoc system ::db/rollback rollback?)
-            (feat/migrate-team! team-id
-                                :validate? validate?
-                                :throw-on-validate? (not skip-on-error)))
-
-        (print-stats!
-         (-> (deref feat/*stats*)
-             (dissoc :total/files)
-             (assoc :elapsed (dt/format-duration (tpoint))))))
-
-      (catch Throwable cause
-        (l/dbg :hint "migrate:error" :cause cause))
-
-      (finally
-        (let [elapsed (dt/format-duration (tpoint))]
-          (l/dbg :hint "migrate:end" :elapsed elapsed))))))
-
-(defn default-on-end
-  [stats]
-  (print-stats!
-   (-> stats
-       (update :elapsed/total dt/format-duration)
-       (dissoc :total/teams))))
-
-(defn migrate-teams!
-  [{:keys [::db/pool] :as system}
-   & {:keys [chunk-size max-jobs max-items start-at
-             rollback? validate? preset skip-on-error
-             max-time on-start on-progress on-error on-end]
-      :or {chunk-size 10000
-           validate? false
-           rollback? true
-           skip-on-error true
-           on-end default-on-end
-           preset :shutdown-on-failure
-           max-jobs Integer/MAX_VALUE
-           max-items Long/MAX_VALUE}}]
-
-  (letfn [(get-chunk [cursor]
-            (let [sql  (str/concat
-                        "SELECT id, created_at, features FROM team "
-                        " WHERE created_at < ? AND deleted_at IS NULL "
-                        " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! pool [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (seq rows)]))
-
-          (get-candidates []
-            (->> (d/iteration get-chunk
-                              :vf second
-                              :kf first
-                              :initk (or start-at (dt/now)))
-                 (map #(update % :features db/decode-pgarray #{}))
-                 (remove #(contains? (:features %) "ephimeral/v2-migration"))
-                 (take max-items)
-                 (map :id)))
-
-          (migrate-team [team-id]
-            (try
-              (-> (assoc system ::db/rollback rollback?)
-                  (feat/migrate-team! team-id
-                                      :validate? validate?
-                                      :throw-on-validate? (not skip-on-error)))
-              (catch Throwable cause
-                (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
-
-          (process-team [scope tpoint mtime team-id]
-            (ps/acquire! feat/*semaphore*)
-            (let [ts (tpoint)]
-              (if (and mtime (neg? (compare mtime ts)))
-                (l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts))
-                (px/submit! scope (partial migrate-team team-id)))))]
-
-    (l/dbg :hint "migrate:start")
-
-    (let [sem    (ps/create :permits max-jobs)
-          total  (get-total-teams pool)
-          stats  (atom {:total/teams (min total max-items)})
-          tpoint (dt/tpoint)
-          mtime  (some-> max-time dt/duration)]
-
-      (when (fn? on-start)
-        (on-start {:total total :rollback rollback?}))
-
-      (add-watch stats :progress-report (report-progress-teams tpoint on-progress))
-
-      (binding [feat/*stats* stats
-                feat/*semaphore* sem
-                feat/*skip-on-error* skip-on-error]
-        (try
-          (pu/with-open [scope (px/structured-task-scope :preset preset
-                                                         :factory :virtual)]
-            (loop [candidates (get-candidates)]
-              (when-let [team-id (first candidates)]
-                (when (process-team scope tpoint mtime team-id)
-                  (recur (rest candidates)))))
-
-            (p/await! scope))
-
-          (when (fn? on-end)
-            (-> (deref stats)
-                (assoc :elapsed/total (tpoint))
-                (on-end)))
-
-          (catch Throwable cause
-            (l/dbg :hint "migrate:error" :cause cause)
-            (when (fn? on-error)
-              (on-error cause)))
-
-          (finally
-            (let [elapsed (dt/format-duration (tpoint))]
-              (l/dbg :hint "migrate:end" :elapsed elapsed))))))))

backend/src/app/srepl/ext.clj

@@ -4,16 +4,14 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
-(ns app.srepl.cli
+(ns app.srepl.ext
   "PREPL API for external usage (CLI or ADMIN)"
   (:require
    [app.auth :as auth]
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
    [app.db :as db]
-   [app.main :as main]
    [app.rpc.commands.auth :as cmd.auth]
-   [app.srepl.components-v2]
    [app.util.json :as json]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
@@ -23,18 +21,18 @@
   (or (deref (requiring-resolve 'app.main/system))
       (deref (requiring-resolve 'user/system))))
 
-(defmulti ^:private exec-command ::cmd)
+(defmulti ^:private run-json-cmd* ::cmd)
 
-(defn exec
+(defn run-json-cmd
   "Entry point with external tools integrations that uses PREPL
   interface for interacting with running penpot backend."
   [data]
-  (let [data (json/decode data)]
-    (-> {::cmd (keyword (:cmd data "default"))}
-        (merge (:params data))
-        (exec-command))))
+  (let [data (json/decode data)
+        params (merge {::cmd (keyword (:cmd data "default"))}
+                      (:params data))]
+    (run-json-cmd* params)))
 
-(defmethod exec-command :create-profile
+(defmethod run-json-cmd* :create-profile
   [{:keys [fullname email password is-active]
     :or {is-active true}}]
   (when-let [system (get-current-system)]
@@ -48,7 +46,7 @@
         (->> (cmd.auth/create-profile! conn params)
              (cmd.auth/create-profile-rels! conn))))))
 
-(defmethod exec-command :update-profile
+(defmethod run-json-cmd* :update-profile
   [{:keys [fullname email password is-active]}]
   (when-let [system (get-current-system)]
     (db/with-atomic [conn (:app.db/pool system)]
@@ -69,7 +67,7 @@
                                 {::db/return-keys? false})]
             (pos? (:next.jdbc/update-count res))))))))
 
-(defmethod exec-command :delete-profile
+(defmethod run-json-cmd* :delete-profile
   [{:keys [email soft]}]
   (when-not email
     (ex/raise :type :assertion
@@ -89,7 +87,7 @@
                               {::db/return-keys? false}))]
         (pos? (:next.jdbc/update-count res))))))
 
-(defmethod exec-command :search-profile
+(defmethod run-json-cmd* :search-profile
   [{:keys [email]}]
   (when-not email
     (ex/raise :type :assertion
@@ -103,33 +101,11 @@
                      " where email similar to ? order by created_at desc limit 100")]
         (db/exec! conn [sql email])))))
 
-(defmethod exec-command :derive-password
+(defmethod run-json-cmd* :derive-password
   [{:keys [password]}]
   (auth/derive-password password))
 
-(defmethod exec-command :migrate-v2
-  [_]
-  (letfn [(on-start [{:keys [total rollback]}]
-            (println
-             (str/ffmt "The components/v2 migration started (rollback:%, teams:%)"
-                       (if rollback "on" "off")
-                       total)))
-
-          (on-progress [{:keys [total elapsed progress completed]}]
-            (println (str/ffmt "Progress % (total: %, completed: %, elapsed: %)"
-                               progress total completed elapsed)))
-          (on-error [cause]
-            (println "ERR:" (ex-message cause)))
-
-          (on-end [_]
-            (println "Migration finished"))]
-    (app.srepl.components-v2/migrate-teams! main/system
-                                            :on-start on-start
-                                            :on-error on-error
-                                            :on-progress on-progress
-                                            :on-end on-end)))
-
-(defmethod exec-command :default
+(defmethod run-json-cmd* :default
   [{:keys [::cmd]}]
   (ex/raise :type :internal
             :code :not-implemented

backend/src/app/srepl/fixes.clj

@@ -0,0 +1,400 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.srepl.fixes
+  "A collection of adhoc fixes scripts."
+  (:require
+   [app.common.data :as d]
+   [app.common.files.validate :as cfv]
+   [app.common.geom.shapes :as gsh]
+   [app.common.logging :as l]
+   [app.common.pages.helpers :as cph]
+   [app.common.pprint :refer [pprint]]
+   [app.common.types.component :as ctk]
+   [app.common.types.container :as ctn]
+   [app.common.types.file :as ctf]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.rpc.commands.files :as files]
+   [app.srepl.helpers :as h]
+   [app.util.blob :as blob]))
+
+(defn validate-file
+  [file]
+  (let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
+                  (cons file)
+                  (map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
+                  (d/index-by :id))
+
+        update-page (fn [page]
+                      (let [errors (cfv/validate-shape uuid/zero file page libs)]
+                        (when (seq errors)
+                          (println "******Errors in file " (:id file) " page " (:id page))
+                          (pprint errors {:level 3}))))]
+
+    (update file :data h/update-pages update-page)))
+
+(defn repair-orphaned-shapes
+  "There are some shapes whose parent has been deleted. This function
+  detects them and puts them as children of the root node."
+  ([data]
+   (letfn [(is-orphan? [shape objects]
+             (and (some? (:parent-id shape))
+                  (nil? (get objects (:parent-id shape)))))
+
+           (update-page [page]
+             (let [objects (:objects page)
+                   orphans (into #{} (filter #(is-orphan? % objects)) (vals objects))]
+               (if (seq orphans)
+                 (do
+                   (l/info :hint "found a file with orphans" :file-id (:id data) :broken-shapes (count orphans))
+                   (-> page
+                       (h/update-shapes (fn [shape]
+                                          (if (contains? orphans shape)
+                                            (assoc shape :parent-id uuid/zero)
+                                            shape)))
+                       (update-in [:objects uuid/zero :shapes] into (map :id) orphans)))
+                 page)))]
+
+     (h/update-pages data update-page)))
+
+  ;; special arity for to be called from h/analyze-files to search for
+  ;; files with possible issues
+
+  ([file state]
+   (repair-orphaned-shapes (:data file))
+   (update state :total (fnil inc 0))))
+
+(defn rename-layout-attrs
+  ([file]
+   (let [found? (volatile! false)]
+     (letfn [(update-shape
+               [shape]
+               (when (or (= (:layout-flex-dir shape) :reverse-row)
+                         (= (:layout-flex-dir shape) :reverse-column)
+                         (= (:layout-wrap-type shape) :no-wrap))
+                 (vreset! found? true))
+               (cond-> shape
+                 (= (:layout-flex-dir shape) :reverse-row)
+                 (assoc :layout-flex-dir :row-reverse)
+                 (= (:layout-flex-dir shape) :reverse-column)
+                 (assoc :layout-flex-dir :column-reverse)
+                 (= (:layout-wrap-type shape) :no-wrap)
+                 (assoc :layout-wrap-type :nowrap)))
+
+             (update-page
+               [page]
+               (h/update-shapes page update-shape))]
+
+       (let [new-file (update file :data h/update-pages update-page)]
+         (when @found?
+           (l/info :hint "Found attrs to rename in file"
+                   :id (:id file)
+                   :name (:name file)))
+         new-file))))
+
+   ([file state]
+    (rename-layout-attrs file)
+    (update state :total (fnil inc 0))))
+
+(defn fix-components-shaperefs
+  ([file]
+   (if-not (contains? (:features file) "components/v2")
+     (do
+       (println "   This file is not v2")
+       file)
+     (let [libs (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
+                     (cons file)
+                     (map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
+                     (d/index-by :id))
+
+           fix-copy-item
+           (fn fix-copy-item [allow-head shapes-copy shapes-base copy-id base-id]
+             (let [copy (first (filter #(= (:id %) copy-id) shapes-copy))
+                  ;; do nothing if it is a copy inside of a copy. It will be treated later
+                   stop? (and (not allow-head) (ctk/instance-head? copy))
+                   base (first (filter #(= (:id %) base-id) shapes-base))
+                   fci (partial fix-copy-item false shapes-copy shapes-base)
+
+                   updates (if (and
+                                (not stop?)
+                                (not= (:shape-ref copy) base-id))
+                             [[(:id copy) base-id]]
+                             [])
+
+                   child-updates (if (and
+                                      (not stop?)
+                                      ;; If the base has the same number of childrens than the copy, we asume
+                                      ;; that the shaperefs can be fixed ad pointed in the same order
+                                      (= (count (:shapes copy)) (count (:shapes base))))
+                                   (apply concat (map fci (:shapes copy) (:shapes base)))
+                                   [])]
+               (concat updates child-updates)))
+
+           fix-copy
+           (fn [objects updates copy]
+             (let [component        (ctf/find-component libs (:component-id copy) {:include-deleted? true})
+                   component-file   (get libs (:component-file copy))
+                   component-shapes (ctf/get-component-shapes (:data component-file) component)
+                   copy-shapes      (cph/get-children-with-self objects (:id copy))
+
+                   copy-updates (fix-copy-item true copy-shapes component-shapes (:id copy) (:main-instance-id component))]
+               (concat updates copy-updates)))
+
+           update-page
+           (fn [page]
+             (let [objects      (:objects page)
+                   fc           (partial fix-copy objects)
+                   copies       (->> objects
+                                     vals
+                                     (filter #(and (ctk/instance-head? %) (not (ctk/main-instance? %)))))
+                   updates      (reduce fc [] copies)
+                   updated-page (reduce (fn [p [id shape-ref]]
+                                          (assoc-in p [:objects id :shape-ref] shape-ref))
+                                        page
+                                        updates)]
+               (println "Page " (:name page) " - Fixing " (count updates))
+               updated-page))]
+
+       (println "Updating " (:name file) (:id file))
+       (-> file
+           (update :data h/update-pages update-page)
+           (assoc ::updated true)))))
+
+  ([file save?]
+   (let [file (-> file
+                  (update :data blob/decode)
+                  (fix-components-shaperefs))]
+     (when (and save? (::updated file))
+       (let [data     (blob/encode (:data file))]
+         (db/update! h/*conn* :file
+                     {:data data
+                      ;; :revn (:revn file)
+                      }
+                     {:id (:id file)})
+
+         (files/persist-pointers! h/*conn* (:id file)))))))
+
+(defn fix-component-root
+  ([file]
+   (let [update-shape (fn [page shape]
+                        (let [parent (get (:objects page) (:parent-id shape))]
+                          (if (and parent
+                                   (:component-root shape)
+                                   (:shape-ref parent))
+                            (do
+                              (println "   Shape " (:name shape) (:id shape))
+                              (dissoc shape :component-root))
+                            shape)))
+
+         update-page (fn [page]
+                       (println "Page " (:name page))
+                       (h/update-shapes page (partial update-shape page)))]
+
+     (println "Updating " (:name file) (:id file))
+     (update file :data h/update-pages update-page)))
+
+  ([file save?]
+   (let [file (-> file
+                  (update :data blob/decode)
+                  (fix-component-root))]
+     (when save?
+       (let [data     (blob/encode (:data file))]
+         (db/update! h/*conn* :file
+                     {:data data
+                      ;; :revn (:revn file)
+                      }
+                     {:id (:id file)})
+
+         (files/persist-pointers! h/*conn* (:id file)))))))
+
+(defn update-near-components
+  ([file]
+   (println "Updating " (:name file) (:id file))
+   (if-not (contains? (:features file) "components/v2")
+     (do
+       (println "   This file is not v2")
+       file)
+     (let [libs (->> (files/get-file-libraries h/*conn* (:id file))
+                     (cons file)
+                     (map #(files/get-file h/*conn* (:id %) (:features file)))
+                     (d/index-by :id))
+
+           update-shape
+           (fn [page shape]
+             (if-not (:shape-ref shape)
+               shape
+               (do
+                 ;; Uncomment println's to debug
+                 ;; (println "  -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
+                 (let [root-shape (ctn/get-copy-root (:objects page) shape)]
+                   (if root-shape
+                     (let [component (ctf/get-component libs (:component-file root-shape) (:component-id root-shape) {:include-deleted? true})
+                           component-file (get libs (:component-file root-shape))
+                           component-shapes (ctf/get-component-shapes (:data component-file) component)
+                           ref-shape (d/seek #(= (:id %) (:shape-ref shape)) component-shapes)]
+                       (if-not (and component component-file component-shapes)
+                         (do
+                           ;; (println "  -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
+                           ;; (when-not component (println "     (component not found)"))
+                           ;; (when-not component-file (println "     (component-file not found)"))
+                           ;; (when-not component-shapes (println "     (component-shapes not found)"))
+                           shape)
+                         (if ref-shape
+                           shape  ; This means that the copy is not nested, or this script already was run
+                           (let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape)) component-shapes)]
+                             (if near-shape
+                               (do
+                                 (println "  -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
+                                 (println "     new ref-shape " (:id near-shape))
+                                 (assoc shape :shape-ref (:id near-shape)))
+                               (do
+                                 ;; We assume in this case that this is a fostered sub instance, so we do nothing
+                                 ;; (println "  -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
+                                 ;; (println  (near-shape not found)")
+                                 shape))))))
+                     (do
+                       ;; (println "  -> Shape " (:name shape) (:id shape) " shape-ref " (:shape-ref shape))
+                       ;; (println "     (root shape not found)")
+                       shape))))))
+
+           update-page
+           (fn [page]
+             (println "Page " (:name page))
+             (h/update-shapes page (partial update-shape page)))]
+
+       (-> file
+           (update :data h/update-pages update-page)
+           (assoc ::updated true)))))
+
+  ([file save?]
+   (let [file (-> file
+                  (update :data blob/decode)
+                  (update-near-components))]
+     (when (and save? (::updated file))
+       (let [data     (blob/encode (:data file))]
+         (db/update! h/*conn* :file
+                     {:data data
+                      ;; :revn (:revn file)
+                      }
+                     {:id (:id file)})
+
+         (files/persist-pointers! h/*conn* (:id file)))))))
+
+(defn fix-main-shape-name
+  ([file]
+   (println "Updating " (:name file) (:id file))
+   (if-not (contains? (:features file) "components/v2")
+     (do
+       (println "   This file is not v2")
+       file)
+     (let [libs (->> (files/get-file-libraries h/*conn* (:id file))
+                     (cons file)
+                     (map #(files/get-file h/*conn* (:id %) (:features file)))
+                     (d/index-by :id))
+
+           update-shape
+           (fn [shape]
+             (if-not (ctk/instance-head? shape)
+               shape
+               (let [component (ctf/get-component libs (:component-file shape) (:component-id shape) {:include-deleted? true})
+                     [_path name] (cph/parse-path-name (:name shape))
+                     full-name (cph/clean-path (str (:path component) "/" (:name component)))]
+                 (if (= name (:name component))
+                   (assoc shape :name full-name)
+                   shape))))
+
+
+           update-page
+           (fn [page]
+             (println "Page " (:name page))
+             (h/update-shapes page update-shape))]
+
+       (-> file
+           (update :data h/update-pages update-page)
+           (assoc ::updated true)))))
+
+  ([file save?]
+   (let [file (-> file
+                  (update :data blob/decode)
+                  (fix-main-shape-name))]
+     (when (and save? (::updated file))
+       (let [data     (blob/encode (:data file))]
+         (db/update! h/*conn* :file
+                     {:data data
+                      ;; :revn (:revn file)
+                      }
+                     {:id (:id file)})
+
+         (files/persist-pointers! h/*conn* (:id file)))))))
+
+(defn fix-touched
+  "For all copies, compare all synced attributes with the main, and set the touched attribute if needed."
+  ([file]
+   (let [libraries (->> (files/get-file-libraries app.srepl.helpers/*conn* (:id file))
+                        (map #(files/get-file app.srepl.helpers/*conn* (:id %) (:features file)))
+                        (d/index-by :id))
+
+         update-shape (fn [page shape]
+                        (if (ctk/in-component-copy? shape)
+                          (let [ref-shape (ctf/find-ref-shape file
+                                                              (:objects page)
+                                                              libraries
+                                                              shape
+                                                              :include-deleted? true)
+                                fix-touched-attr
+                                (fn [shape [attr group]]
+                                  (if (nil? ref-shape)
+                                    shape
+                                    (let [equal?
+                                          (if (= group :geometry-group)
+                                            (if (#{:width :height} attr)
+                                              (gsh/close-attrs? attr (get shape attr) (get ref-shape attr) 1)
+                                              true)
+                                            (gsh/close-attrs? attr (get shape attr) (get ref-shape attr)))]
+                                      (when (and (not equal?) (not (cph/touched-group? shape group)))
+                                        (println " -> set touched " (:name shape) (:id shape) attr group))
+                                      (cond-> shape
+                                        (and (not equal?) (not (cph/touched-group? shape group)))
+                                        (update :touched cph/set-touched-group group)))))
+                                
+                                fix-touched-children
+                                (fn [shape]
+                                  (let [matches? (fn [[child-id ref-child-id]]
+                                                   (let [child (ctn/get-shape page child-id)]
+                                                     (= (:shape-ref child) ref-child-id)))
+                                        equal? (every? matches? (d/zip (:shapes shape) (:shapes ref-shape)))]
+                                      (when (and (not equal?) (not (cph/touched-group? shape :shapes)))
+                                        (println " -> set touched " (:name shape) (:id shape) :shapes :shapes-group))
+                                      (cond-> shape
+                                        (and (not equal?) (not (cph/touched-group? shape :shapes-group)))
+                                        (update :touched cph/set-touched-group :shapes-group))))]
+
+                            (as-> shape $
+                              (reduce fix-touched-attr $ ctk/sync-attrs)
+                              (fix-touched-children $)))
+
+                          shape))
+
+         update-page (fn [page]
+                       (println "Page " (:name page))
+                       (h/update-shapes page (partial update-shape page)))]
+
+     (println "Updating " (:name file) (:id file))
+     (update file :data h/update-pages update-page)))
+
+  ([file save?]
+   (let [file (-> file
+                  (update :data blob/decode)
+                  (fix-touched))]
+     (when save?
+       (let [data     (blob/encode (:data file))]
+         (db/update! h/*conn* :file
+                     {:data data
+                      :revn (inc (:revn file))}
+                     {:id (:id file)})
+
+         (files/persist-pointers! h/*conn* (:id file)))))))

backend/src/app/srepl/helpers.clj

@@ -9,24 +9,21 @@
   (:refer-clojure :exclude [parse-uuid])
   #_:clj-kondo/ignore
   (:require
+   [app.auth :refer [derive-password]]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.changes :as cpc]
+   [app.common.files.features :as ffeat]
    [app.common.files.migrations :as pmg]
-   [app.common.files.repair :as repair]
-   [app.common.files.validate :as validate]
    [app.common.logging :as l]
+   [app.common.pages :as cp]
    [app.common.pprint :refer [pprint]]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cfg]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.fdata :as feat.fdata]
    [app.main :refer [system]]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-update :as files-update]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
@@ -40,7 +37,7 @@
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]))
 
-(def ^:dynamic *system* nil)
+(def ^:dynamic *conn* nil)
 
 (defn println!
   [& params]
@@ -53,197 +50,148 @@
     v
     (d/parse-uuid v)))
 
+(defn resolve-connectable
+  [o]
+  (if (db/connection? o)
+    o
+    (if (db/pool? o)
+      o
+      (or (::db/conn o)
+          (::db/pool o)))))
+
+(defn reset-password!
+  "Reset a password to a specific one for a concrete user or all users
+  if email is `:all` keyword."
+  [system & {:keys [email password] :or {password "123123"} :as params}]
+  (us/verify! (contains? params :email) "`email` parameter is mandatory")
+  (db/with-atomic [conn (:app.db/pool system)]
+    (let [password (derive-password password)]
+      (if (= email :all)
+        (db/exec! conn ["update profile set password=?" password])
+        (let [email (str/lower email)]
+          (db/exec! conn ["update profile set password=? where email=?" password email]))))))
+
 (defn reset-file-data!
   "Hardcode replace of the data of one file."
   [system id data]
-  (db/tx-run! system (fn [system]
-                       (db/update! system :file
-                                   {:data data}
-                                   {:id id}))))
+  (db/with-atomic [conn (:app.db/pool system)]
+    (db/update! conn :file
+                {:data data}
+                {:id id})))
 
 (defn get-file
   "Get the migrated data of one file."
-  [system id & {:keys [migrate?] :or {migrate? true}}]
-  (db/run! system
-           (fn [system]
-             (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-               (-> (files/get-file system id :migrate? migrate?)
-                   (update :data feat.fdata/process-pointers deref))))))
-
-(defn validate
-  "Validate structure, referencial integrity and semantic coherence of
-    all contents of a file. Returns a list of errors."
   [system id]
-  (db/tx-run! system
-              (fn [{:keys [::db/conn] :as system}]
-                (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                  (let [id   (if (string? id) (parse-uuid id) id)
-                        file (files/get-file system id)
-                        libs (->> (files/get-file-libraries conn id)
-                                  (into [file] (map (fn [{:keys [id]}]
-                                                      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                                                        (-> (files/get-file system id :migrate? false)
-                                                            (feat.fdata/process-pointers deref)
-                                                            (pmg/migrate-file))))))
-                                  (d/index-by :id))]
-                    (validate/validate-file file libs))))))
-
-(defn repair!
-  "Repair the list of errors detected by validation."
-  [system id]
-  (db/tx-run! system
-              (fn [{:keys [::db/conn] :as system}]
-                (binding [pmap/*tracked* (pmap/create-tracked)
-                          pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                  (let [id      (if (string? id) (parse-uuid id) id)
-                        file    (files/get-file system id)
-                        libs    (->> (files/get-file-libraries conn id)
-                                     (into [file] (map (fn [{:keys [id]}]
-                                                         (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                                                           (-> (files/get-file system id :migrate? false)
-                                                               (feat.fdata/process-pointers deref)
-                                                               (pmg/migrate-file))))))
-                                     (d/index-by :id))
-                        errors  (validate/validate-file file libs)
-                        changes (repair/repair-file file libs errors)
-
-                        file    (-> file
-                                    (update :revn inc)
-                                    (update :data cpc/process-changes changes)
-                                    (update :data blob/encode))]
-
-                    (when (contains? (:features file) "fdata/pointer-map")
-                      (feat.fdata/persist-pointers! system id))
-
-                    (db/update! conn :file
-                                {:revn (:revn file)
-                                 :data (:data file)
-                                 :data-backend nil
-                                 :modified-at (dt/now)
-                                 :has-media-trimmed false}
-                                {:id (:id file)})
-                    :repaired)))))
+  (db/with-atomic [conn (:app.db/pool system)]
+    (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
+      (-> (db/get-by-id conn :file id)
+          (update :data blob/decode)
+          (update :data pmg/migrate-data)
+          (files/process-pointers deref)))))
 
 (defn update-file!
   "Apply a function to the data of one file. Optionally save the changes or not.
   The function receives the decoded and migrated file data."
-  [system & {:keys [update-fn id rollback? migrate? inc-revn?]
-             :or {rollback? true migrate? true inc-revn? true}}]
-  (letfn [(process-file [{:keys [::db/conn] :as system} {:keys [features] :as file}]
-            (binding [pmap/*tracked* (pmap/create-tracked)
-                      pmap/*load-fn* (partial feat.fdata/load-pointer system id)
-                      cfeat/*wrap-with-pointer-map-fn*
-                      (if (contains? features "fdata/pointer-map") pmap/wrap identity)
-                      cfeat/*wrap-with-objects-map-fn*
-                      (if (contains? features "fdata/objectd-map") omap/wrap identity)]
+  [system & {:keys [update-fn id save? migrate? inc-revn?]
+             :or {save? false migrate? true inc-revn? true}}]
+  (db/with-atomic [conn (:app.db/pool system)]
+    (let [file (-> (db/get-by-id conn :file id {::db/for-update? true})
+                   (update :features db/decode-pgarray #{}))]
+      (binding [*conn* conn
+                pmap/*tracked* (atom {})
+                pmap/*load-fn* (partial files/load-pointer conn id)
+                ffeat/*wrap-with-pointer-map-fn*
+                (if (contains? (:features file) "storage/pointer-map") pmap/wrap identity)
+                ffeat/*wrap-with-objects-map-fn*
+                (if (contains? (:features file) "storage/objectd-map") omap/wrap identity)]
+        (let [file (-> file
+                       (update :data blob/decode)
+                       (cond-> migrate? (update :data pmg/migrate-data))
+                       (update-fn)
+                       (cond-> inc-revn? (update :revn inc)))]
+          (when save?
+            (let [features (db/create-array conn "text" (:features file))
+                  data     (blob/encode (:data file))]
+              (db/update! conn :file
+                          {:data data
+                           :revn (:revn file)
+                           :features features}
+                          {:id id})
 
-              (let [file     (cond-> (update-fn file)
-                               inc-revn? (update :revn inc))
-                    features (db/create-array conn "text" (:features file))
-                    data     (blob/encode (:data file))]
+              (when (contains? (:features file) "storage/pointer-map")
+                (files/persist-pointers! conn id))))
 
-                (db/update! conn :file
-                            {:data data
-                             :revn (:revn file)
-                             :features features}
-                            {:id id}))
+          (dissoc file :data))))))
 
-              (when (contains? (:features file) "fdata/pointer-map")
-                (feat.fdata/persist-pointers! system id))
-
-              (dissoc file :data)))]
-
-    (db/tx-run! system
-                (fn [system]
-                  (binding [*system* system]
-                    (try
-                      (->> (files/get-file system id :migrate? migrate?)
-                           (process-file system))
-                      (finally
-                        (when rollback?
-                          (db/rollback! system)))))))))
+(def ^:private sql:retrieve-files-chunk
+  "SELECT id, name, features, created_at, revn, data FROM file
+    WHERE created_at < ? AND deleted_at is NULL
+    ORDER BY created_at desc LIMIT ?")
 
 (defn analyze-files
   "Apply a function to all files in the database, reading them in
   batches. Do not change data.
 
   The `on-file` parameter should be a function that receives the file
-  and the previous state and returns the new state.
-
-  Emits rollback at the end of operation."
-  [system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?]
+  and the previous state and returns the new state."
+  [system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init]
              :or {chunk-size 10 max-items Long/MAX_VALUE}}]
   (letfn [(get-chunk [conn cursor]
-            (let [sql  (str "SELECT id, created_at FROM file "
-                            " WHERE created_at < ? AND deleted_at is NULL "
-                            " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! conn [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (map :id rows)]))
+            (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
+              [(some->> rows peek :created-at) (seq rows)]))
 
           (get-candidates [conn]
             (->> (d/iteration (partial get-chunk conn)
                               :vf second
                               :kf first
                               :initk (or start-at (dt/now)))
-                 (take max-items)))
+                 (take max-items)
+                 (map #(-> %
+                           (update :data blob/decode)
+                           (update :features db/decode-pgarray #{})))))
 
           (on-error* [cause file]
             (println "unexpected exception happened on processing file: " (:id file))
-            (strace/print-stack-trace cause))
+            (strace/print-stack-trace cause))]
 
-          (process-file [{:keys [::db/conn] :as system} file-id]
-            (let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)]
-                         (-> (files/get-file system file-id)
-                             (update :data feat.fdata/process-pointers deref)))
+    (when (fn? on-init) (on-init))
 
-                  libs (when with-libraries?
-                         (->> (files/get-file-libraries conn file-id)
-                              (into [file] (map (fn [{:keys [id]}]
-                                                  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                                                    (-> (files/get-file system id)
-                                                        (update :data feat.fdata/process-pointers deref))))))
-                              (d/index-by :id)))]
-              (try
-                (if with-libraries?
-                  (on-file file libs)
-                  (on-file file))
-                (catch Throwable cause
-                  ((or on-error on-error*) cause file)))))]
+    (db/with-atomic [conn (:app.db/pool system)]
+      (doseq [file (get-candidates conn)]
+        (binding [*conn* conn
+                  pmap/*tracked* (atom {})
+                  pmap/*load-fn* (partial files/load-pointer conn (:id file))
+                  ffeat/*wrap-with-pointer-map-fn*
+                  (if (contains? (:features file) "storage/pointer-map") pmap/wrap identity)
+                  ffeat/*wrap-with-objects-map-fn*
+                  (if (contains? (:features file) "storage/objects-map") omap/wrap identity)]
+          (try
+            (on-file file)
+            (catch Throwable cause
+              ((or on-error on-error*) cause file))))))
 
-    (db/tx-run! system
-                (fn [{:keys [::db/conn] :as system}]
-                  (try
-                    (binding [*system* system]
-                      (when (fn? on-init) (on-init))
-                      (run! (partial process-file system) (get-candidates conn)))
-                    (finally
-                      (when (fn? on-end)
-                        (ex/ignoring (on-end)))
-                      (db/rollback! system)))))))
+    (when (fn? on-end) (on-end))))
 
 (defn process-files!
   "Apply a function to all files in the database, reading them in
   batches."
 
-  [system & {:keys [chunk-size
-                    max-items
-                    workers
-                    start-at
-                    on-file
-                    on-error
-                    on-end
-                    on-init
-                    rollback?]
-             :or {chunk-size 10
-                  max-items Long/MAX_VALUE
-                  workers 1
-                  rollback? true}}]
+  [{:keys [::db/pool] :as system} & {:keys [chunk-size
+                                            max-items
+                                            workers
+                                            start-at
+                                            on-file
+                                            on-error
+                                            on-end
+                                            on-init]
+                                     :or {chunk-size 10
+                                          max-items Long/MAX_VALUE
+                                          workers 1}}]
+
   (letfn [(get-chunk [conn cursor]
-            (let [sql  (str "SELECT id, created_at FROM file "
-                            " WHERE created_at < ? AND deleted_at is NULL "
-                            " ORDER BY created_at desc LIMIT ?")
-                  rows (db/exec! conn [sql cursor chunk-size])]
-              [(some->> rows peek :created-at) (map :id rows)]))
+            (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
+              [(some->> rows peek :created-at)
+               (map #(update % :features db/decode-pgarray #{}) rows)]))
 
           (get-candidates [conn]
             (->> (d/iteration (partial get-chunk conn)
@@ -256,43 +204,39 @@
             (println! "unexpected exception happened on processing file: " (:id file))
             (strace/print-stack-trace cause))
 
-          (process-file [system file-id]
+          (process-file [conn file]
             (try
-              (let [{:keys [features] :as file} (files/get-file system file-id)]
-                (binding [pmap/*tracked* (pmap/create-tracked)
-                          pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)
-                          cfeat/*wrap-with-pointer-map-fn*
-                          (if (contains? features "fdata/pointer-map") pmap/wrap identity)
-                          cfeat/*wrap-with-objects-map-fn*
-                          (if (contains? features "fdata/objectd-map") omap/wrap identity)]
-
-                  (on-file file)
-
-                  (when (contains? features "fdata/pointer-map")
-                    (feat.fdata/persist-pointers! system file-id))))
-
+              (binding [*conn* conn
+                        pmap/*tracked* (atom {})
+                        pmap/*load-fn* (partial files/load-pointer conn (:id file))
+                        ffeat/*wrap-with-pointer-map-fn*
+                        (if (contains? (:features file) "storage/pointer-map") pmap/wrap identity)
+                        ffeat/*wrap-with-objects-map-fn*
+                        (if (contains? (:features file) "storage/objectd-map") omap/wrap identity)]
+                (on-file file))
               (catch Throwable cause
-                ((or on-error on-error*) cause file-id))))
+                ((or on-error on-error*) cause file))))
 
           (run-worker [in index]
-            (db/tx-run! system
-                        (fn [system]
-                          (binding [*system* system]
-                            (loop [i 0]
-                              (when-let [file-id (sp/take! in)]
-                                (println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name))
-                                (process-file system file-id)
-                                (recur (inc i)))))
-
-                          (when rollback?
-                            (db/rollback! system)))))
+            (db/with-atomic [conn pool]
+              (loop [i 0]
+                (when-let [file (sp/take! in)]
+                  (println! "=> worker: index:" index "| loop:" i "| file:" (:id file) "|" (px/get-name))
+                  (process-file conn file)
+                  (recur (inc i))))))
 
           (run-producer [input]
-            (db/tx-run! system (fn [{:keys [::db/conn]}]
-                                 (doseq [file-id (get-candidates conn)]
-                                   (println! "=> producer:" file-id "|" (px/get-name))
-                                   (sp/put! input file-id))
-                                 (sp/close! input))))]
+            (db/with-atomic [conn pool]
+              (doseq [file (get-candidates conn)]
+                (println! "=> producer:" (:id file) "|" (px/get-name))
+                (sp/put! input file))
+              (sp/close! input)))
+
+          (start-worker [input index]
+            (px/thread
+              {:name (str "penpot/srepl/worker/" index)}
+              (run-worker input index)))
+          ]
 
     (when (fn? on-init) (on-init))
 
@@ -301,12 +245,19 @@
                      {:name "penpot/srepl/producer"}
                      (run-producer input))
           threads  (->> (range workers)
-                        (map (fn [index]
-                               (px/thread
-                                 {:name (str "penpot/srepl/worker/" index)}
-                                 (run-worker input index))))
+                        (map (partial start-worker input))
                         (cons producer)
                         (doall))]
 
       (run! p/await! threads)
       (when (fn? on-end) (on-end)))))
+
+(defn update-pages
+  "Apply a function to all pages of one file. The function receives a page and returns an updated page."
+  [data f]
+  (update data :pages-index update-vals f))
+
+(defn update-shapes
+  "Apply a function to all shapes of one page The function receives a shape and returns an updated shape"
+  [page f]
+  (update page :objects update-vals f))

backend/src/app/srepl/main.clj

@@ -8,23 +8,19 @@
   "A collection of adhoc fixes scripts."
   #_:clj-kondo/ignore
   (:require
-   [app.auth :refer [derive-password]]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
-   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as p]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as features.fdata]
-   [app.main :as main]
    [app.msgbus :as mbus]
    [app.rpc.commands.auth :as auth]
-   [app.rpc.commands.files-snapshot :as fsnap]
    [app.rpc.commands.profile :as profile]
-   [app.srepl.cli :as cli]
+   [app.rpc.commands.files-snapshot :as fsnap]
+   [app.srepl.fixes :as f]
    [app.srepl.helpers :as h]
    [app.storage :as sto]
    [app.util.blob :as blob]
@@ -33,7 +29,6 @@
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.pprint :refer [pprint print-table]]
-   [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]))
 
 (defn print-available-tasks
@@ -113,71 +108,43 @@
         (db/delete! conn :http-session {:profile-id (:id profile)})
         :blocked))))
 
-(defn reset-password!
-  "Reset a password to a specific one for a concrete user or all users
-  if email is `:all` keyword."
-  [system & {:keys [email password] :or {password "123123"} :as params}]
-  (us/verify! (contains? params :email) "`email` parameter is mandatory")
-  (db/with-atomic [conn (:app.db/pool system)]
-    (let [password (derive-password password)]
-      (if (= email :all)
-        (db/exec! conn ["update profile set password=?" password])
-        (let [email (str/lower email)]
-          (db/exec! conn ["update profile set password=? where email=?" password email]))))))
-
 (defn enable-objects-map-feature-on-file!
   [system & {:keys [save? id]}]
-  (h/update-file! system
-                  :id id
-                  :update-fn features.fdata/enable-objects-map
-                  :save? save?))
+  (letfn [(update-file [{:keys [features] :as file}]
+            (if (contains? features "storage/objects-map")
+              file
+              (-> file
+                  (update :data migrate)
+                  (update :features conj "storage/objects-map"))))
+
+          (migrate [data]
+            (-> data
+                (update :pages-index update-vals #(update % :objects omap/wrap))
+                (update :components update-vals #(update % :objects omap/wrap))))]
+
+    (h/update-file! system
+                    :id id
+                    :update-fn update-file
+                    :save? save?)))
 
 (defn enable-pointer-map-feature-on-file!
   [system & {:keys [save? id]}]
-  (h/update-file! system
-                  :id id
-                  :update-fn features.fdata/enable-pointer-map
-                  :save? save?))
+  (letfn [(update-file [{:keys [features] :as file}]
+            (if (contains? features "storage/pointer-map")
+              file
+              (-> file
+                  (update :data migrate)
+                  (update :features conj "storage/pointer-map"))))
 
-(defn enable-team-feature!
-  [system team-id feature]
-  (dm/verify!
-   "feature should be supported"
-   (contains? cfeat/supported-features feature))
+          (migrate [data]
+            (-> data
+                (update :pages-index update-vals pmap/wrap)
+                (update :components pmap/wrap)))]
 
-  (let [team-id (if (string? team-id)
-                  (parse-uuid team-id)
-                  team-id)]
-    (db/tx-run! system
-                (fn [{:keys [::db/conn]}]
-                  (let [team     (-> (db/get conn :team {:id team-id})
-                                     (update :features db/decode-pgarray #{}))
-                        features (conj (:features team) feature)]
-                    (when (not= features (:features team))
-                      (db/update! conn :team
-                                  {:features (db/create-array conn "text" features)}
-                                  {:id team-id})
-                      :enabled))))))
-
-(defn disable-team-feature!
-  [system team-id feature]
-  (dm/verify!
-   "feature should be supported"
-   (contains? cfeat/supported-features feature))
-
-  (let [team-id (if (string? team-id)
-                  (parse-uuid team-id)
-                  team-id)]
-    (db/tx-run! system
-                (fn [{:keys [::db/conn]}]
-                  (let [team     (-> (db/get conn :team {:id team-id})
-                                     (update :features db/decode-pgarray #{}))
-                        features (disj (:features team) feature)]
-                    (when (not= features (:features team))
-                      (db/update! conn :team
-                                  {:features (db/create-array conn "text" features)}
-                                  {:id team-id})
-                      :disabled))))))
+    (h/update-file! system
+                    :id id
+                    :update-fn update-file
+                    :save? save?)))
 
 (defn enable-storage-features-on-file!
   [system & {:as params}]
@@ -208,7 +175,9 @@
   collectable file-changes entry."
   [system & {:keys [file-id label]}]
   (let [file-id (h/parse-uuid file-id)]
-    (db/tx-run! system fsnap/take-file-snapshot! {:file-id file-id :label label})))
+    (db/tx-run! system
+                (fn [cfg]
+                  (fsnap/take-file-snapshot! cfg {:file-id file-id :label label})))))
 
 (defn restore-file-snapshot!
   [system & {:keys [file-id id]}]
@@ -326,7 +295,8 @@
                   (= op :profile-id)
                   (if (coll? param)
                     (sequence (keep parse-uuid) param)
-                    (resolve-dest param))))))]
+                    (resolve-dest param))))))
+          ]
 
     (->> (resolve-dest dest)
          (filter some?)

backend/src/app/storage.clj

@@ -18,6 +18,7 @@
    [app.storage.impl :as impl]
    [app.storage.s3 :as ss3]
    [app.util.time :as dt]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
@@ -39,7 +40,7 @@
                    :fs ::sfs/backend))))
 
 (defmethod ig/pre-init-spec ::storage [_]
-  (s/keys :req [::db/pool ::backends]))
+  (s/keys :req [::db/pool ::wrk/executor ::backends]))
 
 (defmethod ig/init-key ::storage
   [_ {:keys [::backends ::db/pool] :as cfg}]
@@ -321,11 +322,10 @@
 ;; and the object is still valid) or deleted (no more references to
 ;; this object so is ready to be deleted).
 
-(declare sql:retrieve-file-media-object-nrefs)
-(declare sql:retrieve-file-object-thumbnail-nrefs)
-(declare sql:retrieve-profile-nrefs)
-(declare sql:retrieve-team-font-variant-nrefs)
 (declare sql:retrieve-touched-objects-chunk)
+(declare sql:retrieve-file-media-object-nrefs)
+(declare sql:retrieve-team-font-variant-nrefs)
+(declare sql:retrieve-profile-nrefs)
 
 (defmethod ig/pre-init-spec ::gc-touched-task [_]
   (s/keys :req [::db/pool]))
@@ -341,9 +341,6 @@
           (get-profile-nrefs [conn id]
             (-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs))
 
-          (get-file-object-thumbnails [conn id]
-            (-> (db/exec-one! conn [sql:retrieve-file-object-thumbnail-nrefs id]) :nrefs))
-
           (mark-freeze-in-bulk [conn ids]
             (db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)"
                                 (db/create-array conn "uuid" ids)]))
@@ -403,7 +400,8 @@
                 (do
                   (some->> (seq to-freeze) (mark-freeze-in-bulk conn))
                   (some->> (seq to-delete) (mark-delete-in-bulk conn))
-                  [(count to-freeze) (count to-delete)]))))]
+                  [(count to-freeze) (count to-delete)]))))
+          ]
 
     (fn [_]
       (db/with-atomic [conn pool]
@@ -412,10 +410,9 @@
                groups    (retrieve-touched conn)]
           (if-let [[bucket ids] (first groups)]
             (let [[f d] (case bucket
-                          "file-media-object"     (process-objects! conn get-file-media-object-nrefs ids bucket)
-                          "team-font-variant"     (process-objects! conn get-team-font-variant-nrefs ids bucket)
-                          "file-object-thumbnail" (process-objects! conn get-file-object-thumbnails ids bucket)
-                          "profile"               (process-objects! conn get-profile-nrefs ids bucket)
+                          "file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket)
+                          "team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket)
+                          "profile"           (process-objects! conn get-profile-nrefs ids bucket)
                           (ex/raise :type :internal
                                     :code :unexpected-unknown-reference
                                     :hint (dm/fmt "unknown reference %" bucket)))]
@@ -438,9 +435,6 @@
   "select ((select count(*) from file_media_object where media_id = ?) +
            (select count(*) from file_media_object where thumbnail_id = ?)) as nrefs")
 
-(def sql:retrieve-file-object-thumbnail-nrefs
-  "select (select count(*) from file_tagged_object_thumbnail where media_id = ?) as nrefs")
-
 (def sql:retrieve-team-font-variant-nrefs
   "select ((select count(*) from team_font_variant where woff1_file_id = ?) +
            (select count(*) from team_font_variant where woff2_file_id = ?) +

backend/src/app/storage/fs.clj

@@ -18,8 +18,8 @@
    [datoteka.io :as io]
    [integrant.core :as ig])
   (:import
-   java.nio.file.Files
-   java.nio.file.Path))
+   java.nio.file.Path
+   java.nio.file.Files))
 
 ;; --- BACKEND INIT
 

backend/src/app/storage/impl.clj

@@ -11,6 +11,7 @@
    [app.common.exceptions :as ex]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
+   [app.worker :as-alias wrk]
    [buddy.core.codecs :as bc]
    [buddy.core.hash :as bh]
    [clojure.java.io :as jio]
@@ -200,7 +201,7 @@
     (str "blake2b:" result)))
 
 (defn resolve-backend
-  [{:keys [::db/pool] :as storage} backend-id]
+  [{:keys [::db/pool ::wrk/executor] :as storage} backend-id]
   (let [backend (get-in storage [::sto/backends backend-id])]
     (when-not backend
       (ex/raise :type :internal
@@ -208,6 +209,7 @@
                 :hint (dm/fmt "backend '%' not configured" backend-id)))
     (-> backend
         (assoc ::sto/id backend-id)
+        (assoc ::wrk/executor executor)
         (assoc ::db/pool pool))))
 
 (defrecord StorageObject [id size created-at expired-at touched-at backend])

backend/src/app/storage/s3.clj

@@ -17,6 +17,7 @@
    [app.storage.impl :as impl]
    [app.storage.tmp :as tmp]
    [app.util.time :as dt]
+   [app.worker :as wrk]
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [datoteka.fs :as fs]
@@ -39,6 +40,7 @@
    software.amazon.awssdk.core.async.AsyncRequestBody
    software.amazon.awssdk.core.async.AsyncResponseTransformer
    software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
+   software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
    software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
    software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
    software.amazon.awssdk.regions.Region
@@ -75,10 +77,9 @@
 (s/def ::bucket ::us/string)
 (s/def ::prefix ::us/string)
 (s/def ::endpoint ::us/string)
-(s/def ::io-threads ::us/integer)
 
 (defmethod ig/pre-init-spec ::backend [_]
-  (s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads]))
+  (s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor]))
 
 (defmethod ig/prep-key ::backend
   [_ {:keys [::prefix ::region] :as cfg}]
@@ -113,7 +114,8 @@
                 ::client
                 ::presigner]
           :opt [::prefix
-                ::sto/id]))
+                ::sto/id
+                ::wrk/executor]))
 
 ;; --- API IMPL
 
@@ -159,6 +161,7 @@
 
 ;; --- HELPERS
 
+(def default-eventloop-threads 4)
 (def default-timeout
   (dt/duration {:seconds 30}))
 
@@ -168,18 +171,18 @@
   (Region/of (name region)))
 
 (defn- build-s3-client
-  [{:keys [::region ::endpoint ::io-threads]}]
+  [{:keys [::region ::endpoint ::wrk/executor]}]
   (let [aconfig (-> (ClientAsyncConfiguration/builder)
+                    (.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
                     (.build))
 
         sconfig (-> (S3Configuration/builder)
                     (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
                     (.build))
 
-        thr-num (or io-threads (min 16 (px/get-available-processors)))
         hclient (-> (NettyNioAsyncHttpClient/builder)
                     (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
-                                                (.numberOfThreads (int thr-num))))
+                                                (.numberOfThreads (int default-eventloop-threads))))
                     (.connectionAcquisitionTimeout default-timeout)
                     (.connectionTimeout default-timeout)
                     (.readTimeout default-timeout)
@@ -194,7 +197,9 @@
                       builder (cond-> ^S3AsyncClientBuilder builder
                                 (some? endpoint)
                                 (.endpointOverride (URI. endpoint)))]
-                  (.build ^S3AsyncClientBuilder builder))]
+                  (.build ^S3AsyncClientBuilder builder))
+
+        ]
 
     (reify
       clojure.lang.IDeref
@@ -221,7 +226,6 @@
   [id subscriber sem content]
   (px/thread
     {:name "penpot/s3/uploader"
-     :virtual true
      :daemon true}
     (l/trace :hint "start upload thread"
              :object-id (str id)
@@ -263,15 +267,15 @@
       (Optional/of (long (impl/get-size content))))
 
     (^void subscribe [_ ^Subscriber subscriber]
-      (let [sem (Semaphore. 0)
-            thr (upload-thread id subscriber sem content)]
-        (.onSubscribe subscriber
-                      (reify Subscription
-                        (cancel [_]
-                          (px/interrupt! thr)
-                          (.release sem 1))
-                        (request [_ n]
-                          (.release sem (int n)))))))))
+     (let [sem (Semaphore. 0)
+           thr (upload-thread id subscriber sem content)]
+       (.onSubscribe subscriber
+                     (reify Subscription
+                       (cancel [_]
+                         (px/interrupt! thr)
+                         (.release sem 1))
+                       (request [_ n]
+                         (.release sem (int n)))))))))
 
 
 (defn- put-object
@@ -309,7 +313,7 @@
     ;; to the filesystem and then read with buffered inputstream; if
     ;; not, read the contento into memory using bytearrays.
     (if (> ^long size (* 1024 1024 2))
-      (let [path (tmp/tempfile :prefix "penpot.storage.s3." :min-age "6h")
+      (let [path (tmp/tempfile :prefix "penpot.storage.s3.")
             rxf  (AsyncResponseTransformer/toFile ^Path path)]
         (->> (.getObject ^S3AsyncClient client
                          ^GetObjectRequest gor

backend/src/app/storage/tmp.clj

@@ -29,7 +29,7 @@
 
 (defmethod ig/prep-key ::cleaner
   [_ cfg]
-  (assoc cfg ::min-age (dt/duration "60m")))
+  (assoc cfg ::min-age (dt/duration "30m")))
 
 (defmethod ig/init-key ::cleaner
   [_ cfg]
@@ -42,15 +42,14 @@
 
 (defn- io-loop
   [{:keys [::min-age] :as cfg}]
-  (l/inf :hint "started tmp cleaner" :default-min-age (dt/format-duration min-age))
+  (l/info :hint "started tmp file cleaner")
   (try
     (loop []
-      (when-let [[path min-age'] (sp/take! queue)]
-        (let [min-age (or min-age' min-age)]
-          (l/dbg :hint "schedule tempfile deletion" :path path
+      (when-let [path (sp/take! queue)]
+        (l/debug :hint "schedule tempfile deletion" :path path
                  :expires-at (dt/plus (dt/now) min-age))
-          (px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
-          (recur))))
+        (px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
+        (recur)))
     (catch InterruptedException _
       (l/trace :hint "cleaner interrupted"))
     (finally
@@ -58,11 +57,11 @@
 
 (defn- remove-temp-file
   "Permanently delete tempfile"
-  [{:keys [::wrk/executor]} path]
+  [{:keys [::wrk/executor path]}]
   (when (fs/exists? path)
     (px/run! executor
              (fn []
-               (l/dbg :hint "permanently delete tempfile" :path path)
+               (l/debug :hint "permanently delete tempfile" :path path)
                (fs/delete path)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -71,17 +70,17 @@
 
 (defn tempfile
   "Returns a tmpfile candidate (without creating it)"
-  [& {:keys [suffix prefix min-age]
+  [& {:keys [suffix prefix]
       :or {prefix "penpot."
            suffix ".tmp"}}]
-  (let [path (fs/tempfile :suffix suffix :prefix prefix)]
-    (sp/offer! queue [path (some-> min-age dt/duration)])
-    path))
+  (let [candidate (fs/tempfile :suffix suffix :prefix prefix)]
+    (sp/offer! queue candidate)
+    candidate))
 
 (defn create-tempfile
-  [& {:keys [suffix prefix min-age]
+  [& {:keys [suffix prefix]
       :or {prefix "penpot."
            suffix ".tmp"}}]
   (let [path (fs/create-tempfile :suffix suffix :prefix prefix)]
-    (sp/offer! queue [path (some-> min-age dt/duration)])
+    (sp/offer! queue path)
     path))

backend/src/app/tasks/file_gc.clj

@@ -13,14 +13,13 @@
    [app.common.data :as d]
    [app.common.files.migrations :as pmg]
    [app.common.logging :as l]
-   [app.common.thumbnails :as thc]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
    [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.media :as media]
+   [app.rpc.commands.files :as files]
    [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
@@ -112,21 +111,18 @@
   (let [xform (comp
                (map :objects)
                (mapcat vals)
-               (mapcat (fn [obj]
+               (keep (fn [{:keys [type] :as obj}]
+                       (case type
+                         :path  (get-in obj [:fill-image :id])
+                         :bool  (get-in obj [:fill-image :id])
                          ;; NOTE: because of some bug, we ended with
                          ;; many shape types having the ability to
                          ;; have fill-image attribute (which initially
                          ;; designed for :path shapes).
-                         (sequence
-                          (keep :id)
-                          (concat [(:fill-image obj)
-                                   (:metadata obj)]
-                                  (map :fill-image (:fills obj))
-                                  (map :stroke-image (:strokes obj))
-                                  (->> (:content obj)
-                                       (tree-seq map? :children)
-                                       (mapcat :fills)
-                                       (map :fill-image)))))))
+                         :group (get-in obj [:fill-image :id])
+                         :image (get-in obj [:metadata :id])
+
+                         nil))))
         pages (concat
                (vals (:pages-index data))
                (vals (:components data)))]
@@ -142,10 +138,10 @@
                     (remove #(contains? used (:id %))))]
 
     (doseq [mobj unused]
-      (l/dbg :hint "delete file media object"
-             :id (:id mobj)
-             :media-id (:media-id mobj)
-             :thumbnail-id (:thumbnail-id mobj))
+      (l/debug :hint "delete file media object"
+               :id (:id mobj)
+               :media-id (:media-id mobj)
+               :thumbnail-id (:thumbnail-id mobj))
 
       ;; NOTE: deleting the file-media-object in the database
       ;; automatically marks as touched the referenced storage
@@ -156,40 +152,34 @@
 
 (defn- clean-file-object-thumbnails!
   [{:keys [::db/conn ::sto/storage]} file-id data]
-  (let [stored (->> (db/query conn :file-tagged-object-thumbnail
+  (let [stored (->> (db/query conn :file-object-thumbnail
                               {:file-id file-id}
                               {:columns [:object-id]})
                     (into #{} (map :object-id)))
 
         using  (into #{}
-                     (comp
-                      (mapcat (fn [{:keys [id objects]}]
-                                (->> (ctt/get-frames objects)
-                                     (map #(assoc % :page-id id)))))
-                      (mapcat (fn [{:keys [id page-id]}]
-                                (list
-                                 (thc/fmt-object-id file-id page-id id "frame")
-                                 (thc/fmt-object-id file-id page-id id "component")))))
-
+                     (mapcat (fn [{:keys [id objects]}]
+                               (->> (ctt/get-frames objects)
+                                    (map #(str id (:id %))))))
                      (vals (:pages-index data)))
 
         unused (set/difference stored using)]
 
     (when (seq unused)
-      (let [sql (str "delete from file_tagged_object_thumbnail "
+      (let [sql (str "delete from file_object_thumbnail "
                      " where file_id=? and object_id=ANY(?)"
                      " returning media_id")
             res (db/exec! conn [sql file-id (db/create-array conn "text" unused)])]
 
-        (l/dbg :hint "delete file object thumbnails"
-               :file-id (str file-id)
-               :total (count res))
-
         (doseq [media-id (into #{} (keep :media-id) res)]
           ;; Mark as deleted the storage object related with the
           ;; photo-id field.
-          (l/trc :hint "touch file object thumbnail storage object" :id (str media-id))
-          (sto/touch-object! storage media-id))))))
+          (l/trace :hint "mark storage object as deleted" :id media-id)
+          (sto/del-object! storage media-id))
+
+        (l/debug :hint "delete file object thumbnails"
+                 :file-id file-id
+                 :total (count res))))))
 
 (defn- clean-file-thumbnails!
   [{:keys [::db/conn ::sto/storage]} file-id revn]
@@ -199,15 +189,15 @@
         res (db/exec! conn [sql file-id revn])]
 
     (when (seq res)
-      (l/dbg :hint "delete file thumbnails"
-             :file-id (str file-id)
-             :total (count res))
-
       (doseq [media-id (into #{} (keep :media-id) res)]
         ;; Mark as deleted the storage object related with the
         ;; media-id field.
-        (l/trc :hint "delete file thumbnail storage object" :id (str media-id))
-        (sto/del-object! storage media-id)))))
+        (l/trace :hint "mark storage object as deleted" :id media-id)
+        (sto/del-object! storage media-id))
+
+      (l/debug :hint "delete file thumbnails"
+               :file-id file-id
+               :total (count res)))))
 
 (def ^:private
   sql:get-files-for-library
@@ -252,7 +242,7 @@
                        (mapv :id))]
 
       (when (seq unused)
-        (l/dbg :hint "clean deleted components" :total (count unused))
+        (l/debug :hint "clean deleted components" :total (count unused))
 
         (let [data (reduce ctkl/delete-component data unused)]
           (db/update! conn :file
@@ -271,9 +261,9 @@
                             " limit 1;")
                   rows (db/exec! conn [sql file-id cursor])]
               [(some-> rows peek :created-at)
-               (mapcat (comp feat.fdata/get-used-pointer-ids blob/decode :data) rows)]))]
+               (mapcat (comp files/get-all-pointer-ids blob/decode :data) rows)]))]
 
-    (let [used (into (feat.fdata/get-used-pointer-ids data)
+    (let [used (into (files/get-all-pointer-ids data)
                      (d/iteration get-pointers-chunk
                                   :vf second
                                   :kf first
@@ -285,15 +275,15 @@
           rows (db/exec! conn [sql file-id used])]
 
       (doseq [fragment-id (map :id rows)]
-        (l/trc :hint "remove unused file data fragment" :id (str fragment-id))
+        (l/trace :hint "remove unused file data fragment" :id (str fragment-id))
         (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id})))))
 
 (defn- process-file
   [{:keys [::db/conn] :as cfg} {:keys [id data revn modified-at features] :as file}]
-  (l/dbg :hint "processing file" :file-id (str id) :modified-at modified-at)
+  (l/debug :hint "processing file" :id id :modified-at modified-at)
 
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-            pmap/*tracked* (pmap/create-tracked)]
+  (binding [pmap/*load-fn* (partial files/load-pointer conn id)
+            pmap/*tracked* (atom {})]
     (let [data (-> (blob/decode data)
                    (assoc :id id)
                    (pmg/migrate-data))]
@@ -303,7 +293,7 @@
       (clean-file-thumbnails! cfg id revn)
       (clean-deleted-components! conn id data)
 
-      (when (contains? features "fdata/pointer-map")
+      (when (contains? features "storage/pointer-map")
         (clean-data-fragments! conn id data))
 
       ;; Mark file as trimmed
@@ -311,4 +301,4 @@
                   {:has-media-trimmed true}
                   {:id id})
 
-      (feat.fdata/persist-pointers! cfg id))))
+      (files/persist-pointers! conn id))))

backend/src/app/tasks/objects_gc.clj

@@ -151,7 +151,7 @@
                                      {::db/return-keys? false})
                   count  (db/get-update-count result)]
               (when (pos? count)
-                (l/debug :hint "mark team for deletion" :id (str id)))
+                (l/debug :hint "mark team for deletion" :id (str id) ))
 
               (+ total count)))]
 

backend/src/app/tasks/telemetry.clj

@@ -178,7 +178,7 @@
 (defn- retrieve-enabled-auth-providers
   [conn]
   (let [sql  (str "select auth_backend as backend, count(*) as total "
-                  "  from profile group by 1")
+                 "  from profile group by 1")
         rows (db/exec! conn [sql])]
     (->> rows
          (map (fn [{:keys [backend total]}]

backend/src/app/util/locks.clj

@@ -8,8 +8,8 @@
   "A syntactic helpers for using locks."
   (:refer-clojure :exclude [locking])
   (:import
-   java.util.concurrent.locks.Lock
-   java.util.concurrent.locks.ReentrantLock))
+   java.util.concurrent.locks.ReentrantLock
+   java.util.concurrent.locks.Lock))
 
 (defn create
   []

backend/src/app/util/objects_map.clj

@@ -335,7 +335,8 @@
   Iterable
   (iterator [this]
     (when-not loaded? (load! this))
-    (ObjectsMapIterator. (.iterator ^Iterable positions) this)))
+    (ObjectsMapIterator. (.iterator ^Iterable positions) this))
+  )
 
 (defn create
   ([]

backend/src/app/util/pointer_map.clj

@@ -60,10 +60,6 @@
 
 (declare create)
 
-(defn create-tracked
-  []
-  (atom {}))
-
 (defprotocol IPointerMap
   (get-id [_])
   (load! [_])
@@ -77,7 +73,7 @@
 
   IPointerMap
   (load! [_]
-    (l/trace :hint "pointer-map:load" :id (str id))
+    (l/trace :hint "pointer-map:load" :id id)
 
     (when-not *load-fn*
       (throw (UnsupportedOperationException. "load is not supported when *load-fn* is not bind")))

backend/src/app/util/svg.clj

@@ -0,0 +1,51 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.util.svg
+  (:require
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [clojure.xml :as xml]
+   [cuerdas.core :as str])
+  (:import
+   javax.xml.XMLConstants
+   java.io.InputStream
+   javax.xml.parsers.SAXParserFactory
+   clojure.lang.XMLHandler
+   org.apache.commons.io.IOUtils))
+
+(defn- secure-parser-factory
+  [^InputStream input ^XMLHandler handler]
+  (.. (doto (SAXParserFactory/newInstance)
+        (.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
+        (.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
+      (newSAXParser)
+      (parse input handler)))
+
+(defn parse
+  [^String data]
+  (try
+    (dm/with-open [istream (IOUtils/toInputStream data "UTF-8")]
+      (xml/parse istream secure-parser-factory))
+    (catch Exception e
+      (l/warn :hint "error on processing svg"
+              :message (ex-message e))
+      (ex/raise :type :validation
+                :code :invalid-svg-file
+                :hint "invalid svg file"
+                :cause e))))
+
+
+;; --- PROCESSORS
+
+(defn strip-doctype
+  [data]
+  (cond-> data
+    (str/includes? data "<!DOCTYPE")
+    (str/replace #"<\!DOCTYPE[^>]*>" "")))
+
+(def pre-process strip-doctype)

backend/src/app/util/time.clj

@@ -371,7 +371,8 @@
     ::sm/decode instant
     :gen/gen (tgen/fmap (fn [i] (in-past i))  tgen/pos-int)
     ::oapi/type "string"
-    ::oapi/format "iso"}})
+    ::oapi/format "iso"
+    }})
 
 (sm/def! ::duration
   {:type :durations
@@ -382,4 +383,5 @@
     :title "duration"
     ::sm/decode duration
     ::oapi/type "string"
-    ::oapi/format "duration"}})
+    ::oapi/format "duration"
+    }})

backend/src/app/util/websocket.clj

@@ -15,8 +15,8 @@
    [app.util.time :as dt]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]
-   [ring.request :as rreq]
-   [ring.websocket :as rws]
+   [yetti.request :as yr]
+   [yetti.util :as yu]
    [yetti.websocket :as yws])
   (:import
    java.nio.ByteBuffer))
@@ -50,7 +50,7 @@
 
 (declare start-io-loop!)
 
-(defn listener
+(defn handler
   "A WebSocket upgrade handler factory. Returns a handler that can be
   used to upgrade to websocket connection. This handler implements the
   basic custom protocol on top of websocket connection with all the
@@ -61,157 +61,166 @@
   It also accepts some options that allows you parametrize the
   protocol behavior. The options map will be used as-as for the
   initial data of the `ws` data structure"
-  [request & {:keys [::on-rcv-message
-                     ::on-snd-message
-                     ::on-connect
-                     ::input-buff-size
-                     ::output-buff-size
-                     ::idle-timeout]
-              :or {input-buff-size 64
-                   output-buff-size 64
-                   idle-timeout 60000
-                   on-connect identity
-                   on-snd-message identity-3
-                   on-rcv-message identity-3}
-              :as options}]
+  [& {:keys [::on-rcv-message
+             ::on-snd-message
+             ::on-connect
+             ::input-buff-size
+             ::output-buff-size
+             ::idle-timeout]
+      :or {input-buff-size 64
+           output-buff-size 64
+           idle-timeout 60000
+           on-connect identity
+           on-snd-message identity-3
+           on-rcv-message identity-3}
+      :as options}]
 
   (assert (fn? on-rcv-message) "'on-rcv-message' should be a function")
   (assert (fn? on-snd-message) "'on-snd-message' should be a function")
   (assert (fn? on-connect) "'on-connect' should be a function")
 
-  (let [input-ch   (sp/chan :buf input-buff-size)
-        output-ch  (sp/chan :buf output-buff-size)
-        hbeat-ch   (sp/chan :buf (sp/sliding-buffer 6))
-        close-ch   (sp/chan)
-        ip-addr    (parse-client-ip request)
-        uagent     (rreq/get-header request "user-agent")
-        id         (uuid/next)
-        state      (atom {})
-        beats      (atom #{})
-        options    (-> options
-                       (update ::handler wrap-handler)
-                       (assoc ::id id)
-                       (assoc ::state state)
-                       (assoc ::beats beats)
-                       (assoc ::created-at (dt/now))
-                       (assoc ::input-ch input-ch)
-                       (assoc ::heartbeat-ch hbeat-ch)
-                       (assoc ::output-ch output-ch)
-                       (assoc ::close-ch close-ch)
-                       (assoc ::remote-addr ip-addr)
-                       (assoc ::user-agent uagent))]
+  (fn [{:keys [::yws/channel] :as request}]
+    (let [input-ch   (sp/chan :buf input-buff-size)
+          output-ch  (sp/chan :buf output-buff-size)
+          hbeat-ch   (sp/chan :buf (sp/sliding-buffer 6))
+          close-ch   (sp/chan)
 
-    {:on-open
-     (fn on-open [channel]
-       (l/dbg :fn "on-open" :conn-id (str id))
-       (let [options (-> options
+          ip-addr    (parse-client-ip request)
+          uagent     (yr/get-header request "user-agent")
+          id         (uuid/next)
+          state      (atom {})
+          beats      (atom #{})
+
+          options    (-> options
+                         (update ::handler wrap-handler)
+                         (assoc ::id id)
+                         (assoc ::state state)
+                         (assoc ::beats beats)
+                         (assoc ::created-at (dt/now))
+                         (assoc ::input-ch input-ch)
+                         (assoc ::heartbeat-ch hbeat-ch)
+                         (assoc ::output-ch output-ch)
+                         (assoc ::close-ch close-ch)
                          (assoc ::channel channel)
+                         (assoc ::remote-addr ip-addr)
+                         (assoc ::user-agent uagent)
                          (on-connect))
-             timeout (dt/duration idle-timeout)]
 
-         (yws/set-idle-timeout! channel timeout)
-         (px/submit! :vthread (partial start-io-loop! options))))
+          on-ws-open
+          (fn [channel]
+            (l/trace :fn "on-ws-open" :conn-id id)
+            (let [timeout (dt/duration idle-timeout)
+                  name    (str "penpot/websocket/io-loop/" id)]
+              (yws/idle-timeout! channel timeout)
+              (px/fn->thread (partial start-io-loop! options)
+                             {:name name :virtual true})))
 
-     :on-close
-     (fn on-close [_channel code reason]
-       (l/dbg :fn "on-close"
-              :conn-id (str id)
-              :code code
-              :reason reason)
-       (sp/close! close-ch))
+          on-ws-terminate
+          (fn [_ code reason]
+            (l/trace :fn "on-ws-terminate"
+                     :conn-id id
+                     :code code
+                     :reason reason)
+            (sp/close! close-ch))
 
-     :on-error
-     (fn on-error [_channel cause]
-       (sp/close! close-ch cause))
+          on-ws-error
+          (fn [_ cause]
+            (sp/close! close-ch cause))
 
-     :on-message
-     (fn on-message [_channel message]
-       (when (string? message)
-         (sp/offer! input-ch message)
-         (swap! state assoc ::last-activity-at (dt/now))))
+          on-ws-message
+          (fn [_ message]
+            (sp/offer! input-ch message)
+            (swap! state assoc ::last-activity-at (dt/now)))
 
-     :on-pong
-     (fn on-pong [_channel data]
-       (sp/put! hbeat-ch data))}))
+          on-ws-pong
+          (fn [_ buffers]
+            ;; (l/trace :fn "on-ws-pong" :buffers (pr-str buffers))
+            (sp/put! hbeat-ch (yu/copy-many buffers)))]
+
+      (yws/on-close! channel (fn [_]
+                               (sp/close! close-ch)))
+
+      {:on-open on-ws-open
+       :on-error on-ws-error
+       :on-close on-ws-terminate
+       :on-text on-ws-message
+       :on-pong on-ws-pong})))
 
 (defn- handle-ping!
   [{:keys [::id ::beats ::channel] :as wsp} beat-id]
-  (l/trc :hint "send ping" :beat beat-id :conn-id (str id))
-  (rws/ping channel (encode-beat beat-id))
+  (l/trace :hint "ping" :beat beat-id :conn-id id)
+  (yws/ping! channel (encode-beat beat-id))
   (let [issued (swap! beats conj (long beat-id))]
     (not (>= (count issued) max-missed-heartbeats))))
 
 (defn- start-io-loop!
-  [{:keys [::id ::close-ch ::input-ch ::output-ch ::heartbeat-ch
-           ::channel ::handler ::beats ::on-rcv-message ::on-snd-message]
-    :as wsp}]
-  (try
-    (handler wsp {:type :open})
-    (loop [i 0]
-      (let [ping-ch (sp/timeout-chan heartbeat-interval)
-            [msg p] (sp/alts! [close-ch input-ch output-ch heartbeat-ch ping-ch])]
-        (when (rws/open? channel)
-          (cond
-            (identical? p ping-ch)
-            (if (handle-ping! wsp i)
-              (recur (inc i))
-              (do
-                (l/trc :hint "closing" :reason "missing to many pings")
-                (rws/close channel 8802 "missing to many pings")))
+  [{:keys [::id ::close-ch ::input-ch ::output-ch ::heartbeat-ch ::channel ::handler ::beats ::on-rcv-message ::on-snd-message] :as wsp}]
+  (px/thread
+    {:name (str "penpot/websocket/io-loop/" id)
+     :virtual true}
+    (try
+      (handler wsp {:type :open})
+      (loop [i 0]
+        (let [ping-ch (sp/timeout-chan heartbeat-interval)
+              [msg p] (sp/alts! [close-ch input-ch output-ch heartbeat-ch ping-ch])]
+          (when (yws/connected? channel)
+            (cond
+              (identical? p ping-ch)
+              (if (handle-ping! wsp i)
+                (recur (inc i))
+                (yws/close! channel 8802 "missing to many pings"))
 
-            (or (identical? p close-ch) (nil? msg))
-            (do :nothing)
+              (or (identical? p close-ch) (nil? msg))
+              (do :nothing)
 
-            (identical? p heartbeat-ch)
-            (let [beat (decode-beat msg)]
-              (l/trc :hint "pong received" :beat beat :conn-id (str id))
-              (swap! beats disj beat)
-              (recur i))
+              (identical? p heartbeat-ch)
+              (let [beat (decode-beat msg)]
+                ;; (l/trace :hint "pong" :beat beat :conn-id id)
+                (swap! beats disj beat)
+                (recur i))
 
-            (identical? p input-ch)
-            (let [message (t/decode-str msg)
-                  message (on-rcv-message message)
-                  {:keys [request-id] :as response} (handler wsp message)]
-              (when (map? response)
-                (sp/put! output-ch
-                         (cond-> response
-                           (some? request-id)
-                           (assoc :request-id request-id))))
-              (recur i))
+              (identical? p input-ch)
+              (let [message (t/decode-str msg)
+                    message (on-rcv-message message)
+                    {:keys [request-id] :as response} (handler wsp message)]
+                (when (map? response)
+                  (sp/put! output-ch
+                           (cond-> response
+                             (some? request-id)
+                             (assoc :request-id request-id))))
+                (recur i))
 
-            (identical? p output-ch)
-            (let [message (on-snd-message msg)
-                  message (t/encode-str message {:type :json-verbose})]
-              (rws/send channel message)
-              (recur i))))))
+              (identical? p output-ch)
+              (let [message (on-snd-message msg)
+                    message (t/encode-str message {:type :json-verbose})]
+                ;; (l/trace :hint "writing message to output" :message msg)
+                (yws/send! channel message)
+                (recur i))))))
 
-    (catch java.nio.channels.ClosedChannelException _)
-    (catch java.net.SocketException _)
-    (catch java.io.IOException _)
+      (catch java.nio.channels.ClosedChannelException _)
+      (catch java.net.SocketException _)
+      (catch java.io.IOException _)
 
-    (catch InterruptedException _cause
-      (l/dbg :hint "websocket thread interrumpted" :conn-id id))
+      (catch InterruptedException _
+        (l/debug :hint "websocket thread interrumpted" :conn-id id))
 
-    (catch Throwable cause
-      (l/err :hint "unhandled exception on websocket thread"
-             :conn-id id
-             :cause cause))
-    (finally
-      (try
+      (catch Throwable cause
+        (l/error :hint "unhandled exception on websocket thread"
+                 :conn-id id
+                 :cause cause))
+
+      (finally
         (handler wsp {:type :close})
 
-        (when (rws/open? channel)
+        (when (yws/connected? channel)
           ;; NOTE: we need to ignore all exceptions here because
           ;; there can be a race condition that first returns that
           ;; channel is connected but on closing, will raise that
           ;; channel is already closed.
           (ex/ignoring
-           (rws/close channel 8899 "terminated")))
+           (yws/close! channel 8899 "terminated")))
 
         (when-let [on-disconnect (::on-disconnect wsp)]
           (on-disconnect))
 
-        (catch Throwable cause
-          (throw cause)))
-
-      (l/trc :hint "websocket thread terminated" :conn-id id))))
+        (l/trace :hint "websocket thread terminated" :conn-id id)))))

backend/src/app/worker.clj

@@ -25,45 +25,43 @@
    [promesa.core :as p]
    [promesa.exec :as px])
   (:import
-   java.util.concurrent.Executor
-   java.util.concurrent.Future
-   java.util.concurrent.ThreadPoolExecutor))
+   java.util.concurrent.ExecutorService
+   java.util.concurrent.ForkJoinPool
+   java.util.concurrent.Future))
 
 (set! *warn-on-reflection* true)
 
-(s/def ::executor #(instance? Executor %))
+(s/def ::executor #(instance? ExecutorService %))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Executor
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(s/def ::parallelism ::us/integer)
+
 (defmethod ig/pre-init-spec ::executor [_]
-  (s/keys :req []))
+  (s/keys :req [::parallelism]))
 
 (defmethod ig/init-key ::executor
-  [_ _]
-  (let [factory (px/thread-factory :prefix  "penpot/default/")
-        executor (px/cached-executor :factory factory :keepalive 30000)]
-    (l/inf :hint "starting executor")
-    (reify
-      java.lang.AutoCloseable
-      (close [_]
-        (l/inf :hint "stoping executor")
-        (px/shutdown! executor))
+  [skey {:keys [::parallelism]}]
+  (let [prefix  (if (vector? skey) (-> skey first name) "default")
+        tname   (str "penpot/" prefix "/%s")
+        ttype   (cf/get :worker-executor-type :fjoin)]
+    (case ttype
+      :fjoin
+      (let [factory (px/forkjoin-thread-factory :name tname)]
+        (px/forkjoin-executor {:factory factory
+                               :core-size (px/get-available-processors)
+                               :parallelism parallelism
+                               :async true}))
 
-      clojure.lang.IDeref
-      (deref [_]
-        {:active (.getPoolSize ^ThreadPoolExecutor executor)
-         :running (.getActiveCount ^ThreadPoolExecutor executor)
-         :completed (.getCompletedTaskCount ^ThreadPoolExecutor executor)})
-
-      Executor
-      (execute [_ runnable]
-        (.execute ^Executor executor ^Runnable runnable)))))
+      :cached
+      (let [factory (px/thread-factory :name tname)]
+        (px/cached-executor :factory factory)))))
 
 (defmethod ig/halt-key! ::executor
   [_ instance]
-  (.close ^java.lang.AutoCloseable instance))
+  (px/shutdown! instance))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASKS REGISTRY
@@ -113,38 +111,42 @@
 
 (defmethod ig/init-key ::monitor
   [_ {:keys [::executor ::mtx/metrics ::interval ::name]}]
-  (letfn [(monitor! [executor prev-completed]
-            (let [labels        (into-array String [(d/name name)])
-                  stats         (deref executor)
+  (letfn [(monitor! [^ForkJoinPool executor prev-steals]
+            (let [running   (.getRunningThreadCount executor)
+                  queued    (.getQueuedSubmissionCount executor)
+                  active    (.getPoolSize executor)
+                  steals    (.getStealCount executor)
+                  labels    (into-array String [(d/name name)])
 
-                  completed     (:completed stats)
-                  completed-inc (- completed prev-completed)
-                  completed-inc (if (neg? completed-inc) 0 completed-inc)]
+                  steals-inc (- steals prev-steals)
+                  steals-inc (if (neg? steals-inc) 0 steals-inc)]
 
               (mtx/run! metrics
                         :id :executor-active-threads
                         :labels labels
-                        :val (:active stats))
-
+                        :val active)
               (mtx/run! metrics
                         :id :executor-running-threads
+                        :labels labels :val running)
+              (mtx/run! metrics
+                        :id :executors-queued-submissions
                         :labels labels
-                        :val (:running stats))
-
+                        :val queued)
               (mtx/run! metrics
                         :id :executors-completed-tasks
                         :labels labels
-                        :inc completed-inc)
+                        :inc steals-inc)
 
-              completed-inc))]
+              steals))]
 
     (px/thread
       {:name "penpot/executors-monitor" :virtual true}
       (l/inf :hint "monitor: started" :name name)
       (try
-        (loop [completed 0]
-          (px/sleep interval)
-          (recur (long (monitor! executor completed))))
+        (loop [steals 0]
+          (when-not (px/shutdown? executor)
+            (px/sleep interval)
+            (recur (long (monitor! executor steals)))))
         (catch InterruptedException _cause
           (l/trc :hint "monitor: interrupted" :name name))
         (catch Throwable cause

backend/test/backend_tests/bounce_handling_test.clj

@@ -28,7 +28,7 @@
 (defn bounce-report
   [{:keys [token email] :or {email "user@example.com"}}]
   {"notificationType" "Bounce",
-   "bounce" {"feedbackId" "010701776d7dd251-c08d280d-9f47-41aa-b959-0094fec779d9-000000",
+   "bounce" {"feedbackId""010701776d7dd251-c08d280d-9f47-41aa-b959-0094fec779d9-000000",
              "bounceType" "Permanent",
              "bounceSubType" "General",
              "bouncedRecipients" [{"emailAddress" email,
@@ -113,7 +113,8 @@
     (t/is (= "permanent" (:kind result)))
     (t/is (= "general" (:category result)))
     (t/is (= ["user@example.com"] (mapv :email (:recipients result))))
-    (t/is (= (:id profile) (:profile-id result)))))
+    (t/is (= (:id profile) (:profile-id result)))
+    ))
 
 (t/deftest test-parse-complaint-report
   (let [profile (th/create-profile* 1)
@@ -128,7 +129,8 @@
     (t/is (= "abuse" (:kind result)))
     (t/is (= nil (:category result)))
     (t/is (= ["user@example.com"] (into [] (:recipients result))))
-    (t/is (= (:id profile) (:profile-id result)))))
+    (t/is (= (:id profile) (:profile-id result)))
+    ))
 
 (t/deftest test-parse-complaint-report-without-token
   (let [props   (:app.setup/props th/*system*)
@@ -139,7 +141,8 @@
     (t/is (= "abuse" (:kind result)))
     (t/is (= nil (:category result)))
     (t/is (= ["user@example.com"] (into [] (:recipients result))))
-    (t/is (= nil (:profile-id result)))))
+    (t/is (= nil (:profile-id result)))
+    ))
 
 (t/deftest test-process-bounce-report
   (let [profile (th/create-profile* 1)
@@ -166,7 +169,9 @@
       (t/is (= "user@example.com" (get-in rows [0 :email]))))
 
     (let [prof (db/get-by-id pool :profile (:id profile))]
-      (t/is (false? (:is-muted prof))))))
+      (t/is (false? (:is-muted prof))))
+
+    ))
 
 (t/deftest test-process-complaint-report
   (let [profile (th/create-profile* 1)
@@ -196,7 +201,9 @@
 
 
     (let [prof (db/get-by-id pool :profile (:id profile))]
-      (t/is (false? (:is-muted prof))))))
+      (t/is (false? (:is-muted prof))))
+
+    ))
 
 (t/deftest test-process-bounce-report-to-self
   (let [profile (th/create-profile* 1)

backend/test/backend_tests/email_sending_test.clj

@@ -6,9 +6,9 @@
 
 (ns backend-tests.email-sending-test
   (:require
+   [backend-tests.helpers :as th]
    [app.db :as db]
    [app.email :as emails]
-   [backend-tests.helpers :as th]
    [clojure.test :as t]
    [promesa.core :as p]))