📎 Update changelog

Signed-off-by: jdo-odoo <108932862+jdo-odoo@users.noreply.github.com>

.circleci/config.yml

@@ -33,12 +33,17 @@ jobs:
           command: |
             clojure -M:dev:test
 
+      - run:
+          name: "NODE tests"
+          working_directory: "./common"
+          command: |
+            yarn run test
+
       - save_cache:
           paths:
             - ~/.m2
           key: v1-dependencies-{{ checksum "common/deps.edn"}}
 
-
   test-frontend:
     docker:
       - image: penpotapp/devenv:latest
@@ -87,6 +92,44 @@ jobs:
             - ~/.m2
           key: v1-dependencies-{{ checksum "frontend/deps.edn"}}
 
+  test-components:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g -Xms2g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+
+      - run:
+          name: Install dependencies
+          working_directory: "./frontend"
+          command: |
+            yarn
+            npx playwright install --with-deps
+
+      - run:
+          name: Build Storybook
+          working_directory: "./frontend"
+          command: yarn run build:storybook
+
+      - run:
+          name: Serve Storybook and run tests
+          working_directory: "./frontend"
+          command: |
+            npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
+              "npx http-server storybook-static --port 6006 --silent" \
+              "npx wait-on tcp:6006 && yarn test:storybook"
 
   test-integration:
     docker:
@@ -174,7 +217,6 @@ jobs:
             - ~/.m2
           key: v1-dependencies-{{ checksum "backend/deps.edn" }}
 
-
   test-exporter:
     docker:
       - image: penpotapp/devenv:latest
@@ -204,11 +246,36 @@ jobs:
             yarn run fmt:clj:check
             yarn run lint:clj
 
+  test-render-wasm:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+    environment:
+
+    steps:
+      - checkout
+
+      - run:
+          name: "fmt check"
+          working_directory: "./render-wasm"
+          command: |
+            cargo fmt --check
+
+      - run:
+          name: "cargo tests"
+          working_directory: "./render-wasm"
+          command: |
+            ./test
+
 workflows:
   penpot:
     jobs:
       - test-frontend
+      - test-components
       - test-integration
       - test-backend
       - test-common
       - test-exporter
+      - test-render-wasm

.clj-kondo/config.edn

@@ -58,6 +58,12 @@
   :redundant-do
   {:level :off}
 
+  :redundant-ignore
+  {:level :off}
+
+  :redundant-nested-call
+  {:level :off}
+
   :earmuffed-var-not-dynamic
   {:level :off}
 

.editorconfig

@@ -11,3 +11,9 @@ end_of_line = lf
 insert_final_newline = true
 
 trim_trailing_whitespace = true
+
+[*.{rs}]
+indent_size = 4
+indent_style = space
+
+end_of_line = lf

.github/workflows/commit-checker.yml

@@ -0,0 +1,50 @@
+name: 'Commit Message Check'
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+  push:
+    branches:
+      - main
+      - develop
+      - staging
+
+jobs:
+  check-commit-message:
+    name: Check Commit Message
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Commit Type
+        uses: gsactions/commit-message-checker@v2
+        with:
+          pattern: '^:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):\s[A-Z].*[^.]$'
+          flags: 'gm'
+          error: 'Commit should match CONTRIBUTING.md guideline'
+          checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
+          accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
+
+      # - name: Check Line Length
+      #   uses: gsactions/commit-message-checker@v2
+      #   with:
+      #     pattern: '^[^#].{74}'
+      #     error: 'The maximum line length of 74 characters is exceeded.'
+      #     excludeDescription: 'true' # optional: this excludes the description body of a pull request
+      #     excludeTitle: 'true' # optional: this excludes the title of a pull request
+      #     checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
+      #     accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is ue
+      # - name: Check for Resolves / Fixes
+      #   uses: gsactions/commit-message-checker@v2
+      #   with:
+      #     pattern: '^.+(Resolves|Fixes): \#[0-9]+$'
+      #     error: 'You need at least one "Resolves|Fixes: #<issue number>" line.'
+
+

.nvmrc

@@ -1 +1 @@
-v20.11.1
+v22.13.1

CHANGES.md

@@ -1,17 +1,141 @@
 # CHANGELOG
 
-## 2.5.0
-
-### :rocket: Epics and highlights
-
-### :boom: Breaking changes & Deprecations
-
-### :heart: Community contributions (Thank you!)
+## 2.5.4
 
 ### :sparkles: New features
 
+- Add support for WEBP format on shape export [Github #6053](https://github.com/penpot/penpot/pull/6053) and [Github #6074](https://github.com/penpot/penpot/pull/6074)
+
 ### :bug: Bugs fixed
 
+- Fix feature loading on workspace when opening a file in a background
+  tab [Taiga #10377](https://tree.taiga.io/project/penpot/issue/10377)
+- Fix minor inconsistencies on RPC `get-file-libraries` and `get-file`
+  methods (add missing team-id prop)
+- Fix problem with viewer role and inspect mode [Taiga #9751](https://tree.taiga.io/project/penpot/issue/9751)
+- Fix error when clicking on a comment at the viewer's sidebar [Taiga #10465](https://tree.taiga.io/project/penpot/issue/10465)
+
+## 2.5.3
+
+### :bug: Bugs fixed
+
+- Component sync issues with multiple tabs [Taiga #10471](https://tree.taiga.io/project/penpot/issue/10471)
+
+## 2.5.2
+
+### :sparkles: New features
+
+- When the workspace is empty, set default the board creation tool [Taiga #9425](https://tree.taiga.io/project/penpot/us/9425)
+
+### :bug: Bugs fixed
+
+- Fix scroll on storybook docs [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+- Navigate tracking event firing multiple times [Taiga #10415](https://tree.taiga.io/project/penpot/issue/10415)
+- Fix problem with selection colors [Taiga #10376](https://tree.taiga.io/project/penpot/issue/10376)
+- Fix scroll on storybook icons list [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+
+## 2.5.1
+
+### :sparkles: New features
+
+- Improve Nginx entryponit to get the resolvers dinamically by default
+
+## 2.5.0
+
+### :boom: Breaking changes & Deprecations
+
+Although this is not a breaking change, we believe it’s important to highlight it in this
+section:
+
+This release includes a fix for an internal bug in Penpot that caused incorrect handling
+of media assets (e.g., fill images). The issue has been resolved since version 2.4.3, so
+no new incorrect references will be generated. However, existing files may still contain
+incorrect references.
+
+To address this, we’ve provided a script to correct these references in existing files.
+
+While having incorrect references generally doesn’t result in visible issues, there are
+rare cases where it can cause problems. For example, if a component library (containing
+images) is deleted, and that library is being used in other files, running the FileGC task
+(responsible for freeing up space and performing logical deletions) could leave those
+files with broken references to the images.
+
+To execute script:
+
+```bash
+docker exec -ti <container-name-or-id> ./run.sh app.migrations.media-refs '{:max-jobs 1}'
+```
+
+If you have a big database and many cores available, you can reduce the time of processing
+all files by increasing paralelizacion changing the `max-jobs` value from 1 to N (where N
+is a number of cores)
+
+### :sparkles: New features
+
+- [GRADIENTS] New gradients UI with multi-stop support. [Taiga #3418](https://tree.taiga.io/project/penpot/epic/3418)
+- [GRADIENTS] Radial Gradient [Taiga #8768](https://tree.taiga.io/project/penpot/us/8768)
+- Shareable link pointing to an specific board. [Taiga #3219](https://tree.taiga.io/project/penpot/us/3219)
+- Copy styles in CSS [Taiga #9401](https://tree.taiga.io/project/penpot/us/9401)
+- Copy/paste shape styles (fills, strokes, shadows, etc..) [Taiga #8937](https://tree.taiga.io/project/penpot/us/8937)
+- Copy text content to clipboard [Taiga #9970](https://tree.taiga.io/project/penpot/us/9970?milestone=424203)
+- Resize board to fit content option [Taiga #4707](https://tree.taiga.io/project/penpot/us/4707)
+- Rename selected layer via Board name [Taiga #9430](https://tree.taiga.io/project/penpot/us/9430)
+- [COMMENTS] Mention Functionality with and Sidebar Filters [Taiga #9237](https://tree.taiga.io/project/penpot/us/9237)
+- [COMMENTS] Visual Changes in Comments [Taiga #9234](https://tree.taiga.io/project/penpot/us/9234)
+- [COMMENTS] Notifications in Backend, Profile Section, and Mention Email Notification [Taiga #9233](https://tree.taiga.io/project/penpot/us/9233)
+
+### :bug: Bugs fixed
+
+- Fix missing state refresh on notifications update [Taiga #10253](https://tree.taiga.io/project/penpot/issue/10253)
+- Fix icon visualization on select component [Taiga #8889](https://tree.taiga.io/project/penpot/issue/8889)
+- Fix typo on integration tests docs [Taiga #10112](https://tree.taiga.io/project/penpot/issue/10112)
+- Fix menu shadow color [Taiga #10102](https://tree.taiga.io/project/penpot/issue/10102)
+- Fix problem with alt key measures being stuck [Taiga #9348](https://tree.taiga.io/project/penpot/issue/9348)
+- Fix error when reseting stroke cap
+- Fix problem with strokes not refreshing in Safari [Taiga #9040](https://tree.taiga.io/project/penpot/issue/9040)
+- Fix problem with multiple color changes [Taiga #9631](https://tree.taiga.io/project/penpot/issue/9631)
+- Fix create new layers in a component copy [Taiga #10037](https://tree.taiga.io/project/penpot/issue/10037)
+- Fix problem in plugins with zoomIntoView [Plugins #189](https://github.com/penpot/penpot-plugins/issues/189)
+- Fix problem in plugins with renaming components [Taiga #10060](https://tree.taiga.io/project/penpot/issue/10060)
+- Added upload svg with images method [#5489](https://github.com/penpot/penpot/issues/5489)
+- Fix problem with root frame parent reference [Taiga #9437](https://tree.taiga.io/project/penpot/issue/9437)
+- Fix change flex direction using plugins API [Taiga #9407](https://tree.taiga.io/project/penpot/issue/9407)
+- Fix problem opening url when page-id didn't exist [Taiga #10157](https://tree.taiga.io/project/penpot/issue/10157)
+- Fix problem with onboarding to a team [Taiga #10143](https://tree.taiga.io/project/penpot/issue/10143)
+- Fix problem with grid layout crashing [Taiga #10127](https://tree.taiga.io/project/penpot/issue/10127)
+- Fix rename locked boards [Taiga #10174](https://tree.taiga.io/project/penpot/issue/10174)
+- Fix update-libraries dialog disappear when clicking outside [Taiga #10238](https://tree.taiga.io/project/penpot/issue/10238)
+- Fix incorrect handling of team access requests with deleted/recreated users
+- Fix incorect handling of profile settings related to invitation notifications [Taiga #10252](https://tree.taiga.io/project/penpot/issue/10252)
+
+## 2.4.3
+
+### :bug: Bugs fixed
+
+- Fix errors from editable select on measures menu [Taiga #9888](https://tree.taiga.io/project/penpot/issue/9888)
+- Fix exception on importing some templates from templates slider
+- Consolidate adding share button to workspace
+- Fix problem when pasting text [Taiga #9929](https://tree.taiga.io/project/penpot/issue/9929)
+- Fix incorrect media reference handling on component instantiation
+
+## 2.4.2
+
+### :bug: Bugs fixed
+
+- Fix detach when top copy is dangling and nested copy is not [Taiga #9699](https://tree.taiga.io/project/penpot/issue/9699)
+- Fix problem in plugins with `replaceColor` method [#174](https://github.com/penpot/penpot-plugins/issues/174)
+- Fix issue with recursive commponents [Taiga #9903](https://tree.taiga.io/project/penpot/issue/9903)
+- Fix missing methods reference on API Docs
+- Fix memory usage issue on file-gc asynchronous task (related to snapshots feature)
+
+
+## 2.4.1
+
+### :bug: Bugs fixed
+
+- Fix error when importing files with touched components [Taiga #9625](https://tree.taiga.io/project/penpot/issue/9625)
+- Fix problem when changing color libraries [Plugins #184](https://github.com/penpot/penpot-plugins/issues/184)
+
 
 ## 2.4.0
 
@@ -24,30 +148,36 @@
   (penpot). Because of that, the default NGINX listen port is now 8080 instead of 80, so
   you will have to modify your infrastructure to apply this change.
 
-- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because, 
-  starting with the next versions, Redis is no longer distributed under an open-source license. 
-  On-premise users are obviously free to upgrade to the version they are using or a more modern one. 
-  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume 
-  associated with the Redis container because the 7.2 storage format may not be compatible with what 
-  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate 
+- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because,
+  starting with the next versions, Redis is no longer distributed under an open-source license.
+  On-premise users are obviously free to upgrade to the version they are using or a more modern one.
+  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume
+  associated with the Redis container because the 7.2 storage format may not be compatible with what
+  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate
   whether to move to an open-source version of Redis (such as https://valkey.io/).
 
 ### :heart: Community contributions (Thank you!)
 
 ### :sparkles: New features
 
-- Viewer role for team members  [Taiga #1056 & #6590](https://tree.taiga.io/project/penpot/us/1056 & https://tree.taiga.io/project/penpot/us/6590)
-- File history versions management [Taiga](https://tree.taiga.io/project/penpot/us/187?milestone=411120)
+- Viewer role for team members [Taiga #1056](https://tree.taiga.io/project/penpot/us/1056) & [Taiga #6590](https://tree.taiga.io/project/penpot/us/6590)
+- File history versions management [Taiga #187](https://tree.taiga.io/project/penpot/us/187?milestone=411120)
 - Rename selected layer via keyboard shortcut and context menu option [Taiga #8882](https://tree.taiga.io/project/penpot/us/8882)
+- New .penpot file format [Taiga #8657](https://tree.taiga.io/project/penpot/us/8657)
 
 ### :bug: Bugs fixed
 
+- Fix problem with some texts desynchronization [Taiga #9379](https://tree.taiga.io/project/penpot/issue/9379)
+- Fix problem with reoder grid layers [#5446](https://github.com/penpot/penpot/issues/5446)
+- Fix problem with swap component style [#9542](https://tree.taiga.io/project/penpot/issue/9542)
 
 ## 2.3.3
 
 ### :bug: Bugs fixed
 
 - Fix problem creating manual overlay interactions [Taiga #9146](https://tree.taiga.io/project/penpot/issue/9146)
+- Fix plugins list default URL
+- Activate plugins feature by default
 
 ## 2.3.2
 

CONTRIBUTING.md

@@ -82,9 +82,10 @@ Where type is:
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
 - :whale: `:whale:` a commit for docker related stuff
-- :rewind: `:rewind:` a commit that reverts changes
 - :paperclip: `:paperclip:` a commit with other not relevant changes
 - :arrow_up: `:arrow_up:` a commit with dependencies updates
+- :arrow_down: `:arrow_down:` a commit with dependencies downgrades
+- :fire: `:fire:` a commit that removes files or code
 
 More info:
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a

backend/deps.edn

@@ -6,7 +6,7 @@
   org.clojure/clojure {:mvn/version "1.12.0"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.6-6"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.6-9"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,7 +17,7 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.4.0.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.5.2.RELEASE"}
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
   funcool/yetti
@@ -26,24 +26,25 @@
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
-  com.github.seancorfield/next.jdbc {:mvn/version "1.3.955"}
+  com.github.seancorfield/next.jdbc
+  {:mvn/version "1.3.994"}
   metosin/reitit-core {:mvn/version "0.7.2"}
-  nrepl/nrepl {:mvn/version "1.3.0"}
-  cider/cider-nrepl {:mvn/version "0.50.2"}
+  nrepl/nrepl {:mvn/version "1.3.1"}
+  cider/cider-nrepl {:mvn/version "0.52.0"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.4"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.46.1.3"}
+  org.postgresql/postgresql {:mvn/version "42.7.5"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.48.0.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "6.0.0"}
+  com.zaxxer/HikariCP {:mvn/version "6.2.1"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.8"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.18.1"}
+  org.jsoup/jsoup {:mvn/version "1.18.3"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -54,12 +55,11 @@
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
   dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.1"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.2"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.28.26"}
-  }
+  software.amazon.awssdk/s3 {:mvn/version "2.28.26"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -74,7 +74,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.5" :git/sha "2a21b7a"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.6" :git/sha "52cf7d6"}}
    :ns-default build}
 
   :test
@@ -88,8 +88,8 @@
 
   :jmx-remote
   {:jvm-opts ["-Dcom.sun.management.jmxremote"
-              "-Dcom.sun.management.jmxremote.port=9091"
-              "-Dcom.sun.management.jmxremote.rmi.port=9091"
+              "-Dcom.sun.management.jmxremote.port=9090"
+              "-Dcom.sun.management.jmxremote.rmi.port=9090"
               "-Dcom.sun.management.jmxremote.local.only=false"
               "-Dcom.sun.management.jmxremote.authenticate=false"
               "-Dcom.sun.management.jmxremote.ssl=false"

backend/package.json

@@ -4,7 +4,7 @@
   "license": "MPL-2.0",
   "author": "Kaleidos INC",
   "private": true,
-  "packageManager": "yarn@4.3.1",
+  "packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728",
   "repository": {
     "type": "git",
     "url": "https://github.com/penpot/penpot"

backend/resources/app/email/comment-mention/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has mentioned you on a comment at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-mention/en.subj

@@ -0,0 +1 @@
+Mentioned in comment

backend/resources/app/email/comment-mention/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has mentioned you on a comment at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/comment-notification/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has commented at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4#31EFB8" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-notification/en.subj

@@ -0,0 +1 @@
+New comment

backend/resources/app/email/comment-notification/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has commented at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/comment-thread/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has created a comment in a thread you've been mentioned at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4#31EFB8" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-thread/en.subj

@@ -0,0 +1 @@
+New response in comment

backend/resources/app/email/comment-thread/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has created a comment in a thread you've been mentioned at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/invite-to-team/en.html

@@ -195,12 +195,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/verify-token?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Accept invite </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> ACCEPT INVITE </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/join-team/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/projects"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Go to the Team </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GO TO THE TEAM </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/password-recovery/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/recovery?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Reset password </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> RESET PASSWORD </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/register/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/verify-token?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Verify email </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> VERIFY EMAIL </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/request-file-access-yourpenpot-view/en.html

@@ -204,12 +204,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -251,4 +251,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access-yourpenpot-view/en.txt

@@ -6,7 +6,7 @@ Since this file is in your Penpot team, you can provide access by sending a view
 
 To proceed, please click the link below to generate and send the view-only link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 

backend/resources/app/email/request-file-access-yourpenpot/en.html

@@ -227,12 +227,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -274,4 +274,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access-yourpenpot/en.txt

@@ -19,7 +19,7 @@ Alternatively, you can create and share a view-only link to the file. This will
 
 Click the link below to generate and send the link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 

backend/resources/app/email/request-file-access/en.html

@@ -211,12 +211,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape }}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Give access to “{{team-name|abbreviate:25}}” Team </a>
+                            <a href="{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape }}"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GIVE ACCESS TO “{{team-name|abbreviate:25}}” TEAM </a>
                           </td>
                         </tr>
                       </table>
@@ -244,12 +244,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -292,4 +292,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access/en.txt

@@ -13,7 +13,7 @@ This will automatically include {{requested-by|abbreviate:25}} in the team, so t
 
 Click the link below to provide team access:
 
-{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id{{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 
@@ -23,8 +23,7 @@ Alternatively, you can create and share a view-only link to the file. This will
 
 Click the link below to generate and send the link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
-
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 If you do not wish to grant access at this time, you can simply disregard this email.

backend/resources/app/email/request-team-access/en.html

@@ -202,12 +202,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Give access to “{{team-name|abbreviate:25}}” </a>
+                            <a href="{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GIVE ACCESS TO “{{team-name|abbreviate:25}}” TEAM </a>
                           </td>
                         </tr>
                       </table>
@@ -249,4 +249,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-team-access/en.txt

@@ -4,7 +4,7 @@ Hello!
 
 To provide access, please click the link below:
 
-{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 If you do not wish to grant access at this time, you can simply disregard this email.

backend/resources/app/templates/debug.tmpl

@@ -114,37 +114,13 @@ Debug Main Page
     </fieldset>
     <fieldset>
       <legend>Import binfile:</legend>
-      <desc>Import penpot file in binary
-      format. If <strong>overwrite</strong> is checked, all files will
-      be overwritten using the same ids found in the file instead of
-      generating a new ones.</desc>
+      <desc>Import penpot file in binary format.</desc>
 
       <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
         <div class="row">
           <input type="file" name="file" value="" />
         </div>
 
-        <div class="row">
-          <label>Overwrite?</label>
-          <input type="checkbox" name="overwrite" />
-          <br />
-          <small>
-            Instead of creating a new file with all relations remapped,
-            reuses all ids and updates/overwrites the objects that are
-            already exists on the database.
-            <strong>Warning, this operation should be used with caution.</strong>
-          </small>
-        </div>
-
-        <div class="row">
-          <label>Migrate?</label>
-          <input type="checkbox" name="migrate" />
-          <br />
-          <small>
-            Applies the file migrations on the importation process.
-          </small>
-        </div>
-
         <div class="row">
           <input type="submit" name="upload" value="Upload" />
         </div>

backend/scripts/repl

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+export PENPOT_SECRET_KEY=super-secret-devenv-key
 export PENPOT_HOST=devenv
 export PENPOT_FLAGS="\
        $PENPOT_FLAGS \
@@ -69,20 +70,17 @@ export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
 export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
 
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full"
+export JAVA_OPTS="--enable-preview \
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
+       -Djdk.tracePinnedThreads=full \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints";
 
-# Enable preview
-export OPTIONS="$OPTIONS -J--enable-preview"
+export OPTIONS="-A:jmx-remote -A:dev"
 
 # Setup HEAP
 # export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"

backend/scripts/run

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-backend-asserts \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-file-snapshot \
+       enable-tiered-file-data-storage";
+
+export JAVA_OPTS="
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints";
+
+export CLOJURE_OPTIONS="-A:dev"
+
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+
+entrypoint=${1:-app.main};
+shift 1;
+set -ex
+
+clojure $CLOJURE_OPTIONS -A:dev -M -m $entrypoint "$@";

backend/scripts/run.template.sh

@@ -18,7 +18,7 @@ if [ -f ./environ ]; then
     source ./environ
 fi
 
-export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow -Dpolyglot.engine.WarnInterpreterOnly=false --enable-preview $JVM_OPTS"
+export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --enable-preview $JVM_OPTS"
 
 ENTRYPOINT=${1:-app.main};
 

backend/scripts/start-dev

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+export PENPOT_SECRET_KEY=super-secret-devenv-key
 export PENPOT_HOST=devenv
 export PENPOT_FLAGS="\
        $PENPOT_FLAGS \

backend/src/app/auth.clj

@@ -8,7 +8,7 @@
   (:require
    [buddy.hashers :as hashers]))
 
-(def default-params
+(def ^:private default-options
   {:alg :argon2id
    :memory 32768 ;; 32 MiB
    :iterations 3
@@ -16,12 +16,12 @@
 
 (defn derive-password
   [password]
-  (hashers/derive password default-params))
+  (hashers/derive password default-options))
 
 (defn verify-password
   [attempt password]
   (try
-    (hashers/verify attempt password)
+    (hashers/verify attempt password default-options)
     (catch Throwable _
       {:update false
        :valid false})))

backend/src/app/binfile/common.clj

@@ -9,9 +9,9 @@
   binfile format implementations and management rpc methods."
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
+   [app.common.files.helpers :as cfh]
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
@@ -20,17 +20,19 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.components-v2 :as feat.compv2]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
-   [clojure.walk :as walk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [datoteka.fs :as fs]
+   [datoteka.io :as io]))
 
 (set! *warn-on-reflection* true)
 
@@ -52,6 +54,30 @@
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(def file-attrs
+  #{:id
+    :name
+    :migrations
+    :features
+    :project-id
+    :is-shared
+    :version
+    :data})
+
+(defn parse-file-format
+  [template]
+  (assert (fs/path? template) "expected InputStream for `template`")
+
+  (with-open [^java.lang.AutoCloseable input (io/input-stream template)]
+    (let [buffer (byte-array 4)]
+      (io/read-to-buffer input buffer)
+      (if (and (= (aget buffer 0) 80)
+               (= (aget buffer 1) 75)
+               (= (aget buffer 2) 3)
+               (= (aget buffer 3) 4))
+        :binfile-v3
+        :binfile-v1))))
+
 (def xf-map-id
   (map :id))
 
@@ -123,16 +149,44 @@
     features (assoc :features (db/decode-pgarray features #{}))
     data     (assoc :data (blob/decode data))))
 
+(defn decode-file
+  "A general purpose file decoding function that resolves all external
+  pointers, run migrations and return plain vanilla file map"
+  [cfg {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+    (let [file (->> file
+                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (feat.fdata/resolve-file-data cfg))]
+
+      (-> file
+          (update :features db/decode-pgarray #{})
+          (update :data blob/decode)
+          (update :data feat.fdata/process-pointers deref)
+          (update :data feat.fdata/process-objects (partial into {}))
+          (update :data assoc :id id)
+          (fmg/migrate-file)))))
+
 (defn get-file
-  [cfg file-id]
+  "Get file, resolve all features and apply migrations.
+
+  Usefull when you have plan to apply massive or not cirurgical
+  operations on file, because it removes the ovehead of lazy fetching
+  and decoding."
+  [cfg file-id & {:as opts}]
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                 (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                   (when-let [file (db/get* conn :file {:id file-id}
-                                            {::db/remove-deleted false})]
-                     (-> file
-                         (decode-row)
-                         (update :data feat.fdata/process-pointers deref)
-                         (update :data feat.fdata/process-objects (partial into {}))))))))
+                 (some->> (db/get* conn :file {:id file-id}
+                                   (assoc opts ::db/remove-deleted false))
+                          (decode-file cfg)))))
+
+(defn clean-file-features
+  [file]
+  (update file :features (fn [features]
+                           (if (set? features)
+                             (-> features
+                                 (cfeat/migrate-legacy-features)
+                                 (set/difference cfeat/frontend-only-features)
+                                 (set/difference cfeat/backend-only-features))
+                             #{}))))
 
 (defn get-project
   [cfg project-id]
@@ -163,10 +217,8 @@
   "Given a set of file-id's, return all matching relations with the libraries"
   [cfg ids]
 
-  (dm/assert!
-   "expected a set of uuids"
-   (and (set? ids)
-        (every? uuid? ids)))
+  (assert (set? ids) "expected a set of uuids")
+  (assert (every? uuid? ids) "expected a set of uuids")
 
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (let [ids (db/create-array conn "uuid" ids)
@@ -215,49 +267,70 @@
             :data nil}
            {::sql/columns [:media-id :file-id :revn]}))
 
+(def ^:private sql:get-missing-media-references
+  "SELECT fmo.*
+     FROM file_media_object AS fmo
+    WHERE fmo.id = ANY(?::uuid[])
+      AND file_id != ?")
 
-(def ^:private
-  xform:collect-media-id
-  (comp
-   (map :objects)
-   (mapcat vals)
-   (mapcat (fn [obj]
-             ;; NOTE: because of some bug, we ended with
-             ;; many shape types having the ability to
-             ;; have fill-image attribute (which initially
-             ;; designed for :path shapes).
-             (sequence
-              (keep :id)
-              (concat [(:fill-image obj)
-                       (:metadata obj)]
-                      (map :fill-image (:fills obj))
-                      (map :stroke-image (:strokes obj))
-                      (->> (:content obj)
-                           (tree-seq map? :children)
-                           (mapcat :fills)
-                           (map :fill-image))))))))
+(defn update-media-references!
+  "Given a file and a coll of media-refs, check if all provided
+  references are correct or fix them in-place"
+  [{:keys [::db/conn] :as cfg} {file-id :id :as file} media-refs]
+  (let [missing-index
+        (reduce (fn [result {:keys [id] :as fmo}]
+                  (assoc result id
+                         (-> fmo
+                             (assoc :id (uuid/next))
+                             (assoc :file-id file-id)
+                             (dissoc :created-at)
+                             (dissoc :deleted-at))))
+                {}
+                (db/exec! conn [sql:get-missing-media-references
+                                (->> (into #{} xf-map-id media-refs)
+                                     (db/create-array conn "uuid"))
+                                file-id]))
 
-(defn collect-used-media
-  "Given a fdata (file data), returns all media references."
-  [data]
-  (-> #{}
-      (into xform:collect-media-id (vals (:pages-index data)))
-      (into xform:collect-media-id (vals (:components data)))
-      (into (keys (:media data)))))
+        lookup-index
+        (fn [id]
+          (if-let [mobj (get missing-index id)]
+            (do
+              (l/trc :hint "lookup index"
+                     :file-id (str file-id)
+                     :snap-id (str (:snapshot-id file))
+                     :id (str id)
+                     :result (str (get mobj :id)))
+              (get mobj :id))
+
+            id))
+
+        update-shapes
+        (fn [data {:keys [page-id shape-id]}]
+          (d/update-in-when data [:pages-index page-id :objects shape-id] cfh/relink-refs lookup-index))
+
+        file
+        (update file :data #(reduce update-shapes % media-refs))]
+
+    (doseq [[old-id item] missing-index]
+      (l/dbg :hint "create missing references"
+             :file-id (str file-id)
+             :snap-id (str (:snapshot-id file))
+             :old-id (str old-id)
+             :id (str (:id item)))
+      (db/insert! conn :file-media-object item
+                  {::db/return-keys false}))
+
+    file))
+
+(def sql:get-file-media
+  "SELECT * FROM file_media_object WHERE id = ANY(?)")
 
 (defn get-file-media
-  [cfg {:keys [data id] :as file}]
+  [cfg {:keys [data] :as file}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (let [ids (collect-used-media data)
-                       ids (db/create-array conn "uuid" ids)
-                       sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")]
-
-                   ;; We assoc the file-id again to the file-media-object row
-                   ;; because there are cases that used objects refer to other
-                   ;; files and we need to ensure in the exportation process that
-                   ;; all ids matches
-                   (->> (db/exec! conn [sql ids])
-                        (mapv #(assoc % :file-id id)))))))
+                 (let [used (cfh/collect-used-media data)
+                       used (db/create-array conn "uuid" used)]
+                   (db/exec! conn [sql:get-file-media used])))))
 
 (def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
@@ -301,48 +374,7 @@
   replace the old :component-file reference with the new
   ones, using the provided file-index."
   [data]
-  (letfn [(process-map-form [form]
-            (cond-> form
-              ;; Relink image shapes
-              (and (map? (:metadata form))
-                   (= :image (:type form)))
-              (update-in [:metadata :id] lookup-index)
-
-              ;; Relink paths with fill image
-              (map? (:fill-image form))
-              (update-in [:fill-image :id] lookup-index)
-
-              ;; This covers old shapes and the new :fills.
-              (uuid? (:fill-color-ref-file form))
-              (update :fill-color-ref-file lookup-index)
-
-              ;; This covers the old shapes and the new :strokes
-              (uuid? (:stroke-color-ref-file form))
-              (update :stroke-color-ref-file lookup-index)
-
-              ;; This covers all text shapes that have typography referenced
-              (uuid? (:typography-ref-file form))
-              (update :typography-ref-file lookup-index)
-
-              ;; This covers the component instance links
-              (uuid? (:component-file form))
-              (update :component-file lookup-index)
-
-              ;; This covers the shadows and grids (they have directly
-              ;; the :file-id prop)
-              (uuid? (:file-id form))
-              (update :file-id lookup-index)))
-
-          (process-form [form]
-            (if (map? form)
-              (try
-                (process-map-form form)
-                (catch Throwable cause
-                  (l/warn :hint "failed form" :form (pr-str form) ::l/sync? true)
-                  (throw cause)))
-              form))]
-
-    (walk/postwalk process-form data)))
+  (cfh/relink-refs data lookup-index))
 
 (defn- relink-media
   "A function responsible of process the :media attr of file data and
@@ -373,8 +405,9 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (let [library (get-file cfg library-id)]
-                (ctf/absorb-assets data (:data library))))
+              (if-let [library (get-file cfg library-id)]
+                (ctf/absorb-assets data (:data library))
+                data))
             data
             library-ids)))
 
@@ -384,20 +417,9 @@
     (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
-(defn- fix-version
-  [file]
-  (let [file (fmg/fix-version file)]
-    ;; FIXME: We're temporarily activating all migrations because a
-    ;; problem in the environments messed up with the version numbers
-    ;; When this problem is fixed delete the following line
-    (if (> (:version file) 22)
-      (assoc file :version 22)
-      file)))
-
 (defn process-file
   [{:keys [id] :as file}]
   (-> file
-      (fix-version)
       (update :data (fn [fdata]
                       (-> fdata
                           (assoc :id id)
@@ -411,100 +433,92 @@
                           (update :colors relink-colors)
                           (d/without-nils))))))
 
-(defn- upsert-file!
-  [conn file]
-  (let [sql (str "INSERT INTO file (id, project_id, name, revn, version, is_shared, data, created_at, modified_at) "
-                 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
-                 "ON CONFLICT (id) DO UPDATE SET data=?, version=?")]
-    (db/exec-one! conn [sql
-                        (:id file)
-                        (:project-id file)
-                        (:name file)
-                        (:revn file)
-                        (:version file)
-                        (:is-shared file)
-                        (:data file)
-                        (:created-at file)
-                        (:modified-at file)
-                        (:data file)
-                        (:version file)])))
+(defn encode-file
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+  (let [file (if (contains? (:features file) "fdata/objects-map")
+               (feat.fdata/enable-objects-map file)
+               file)
 
-(defn persist-file!
-  "Applies all the final validations and perist the file."
-  [{:keys [::db/conn ::timestamp] :as cfg} {:keys [id] :as file}]
+        file (if (contains? (:features file) "fdata/pointer-map")
+               (binding [pmap/*tracked* (pmap/create-tracked)]
+                 (let [file (feat.fdata/enable-pointer-map file)]
+                   (feat.fdata/persist-pointers! cfg id)
+                   file))
+               file)]
 
-  (dm/assert!
-   "expected valid timestamp"
-   (dt/instant? timestamp))
+    (-> file
+        (update :features db/encode-pgarray conn "text")
+        (update :data blob/encode))))
 
-  (let [file   (-> file
-                   (assoc :created-at timestamp)
-                   (assoc :modified-at timestamp)
-                   (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
-                   (update :features
-                           (fn [features]
-                             (let [features (cfeat/check-supported-features! features)]
-                               (-> (::features cfg #{})
-                                   (set/difference cfeat/frontend-only-features)
-                                   (set/union features))))))
+(defn get-params-from-file
+  [file]
+  (let [params {:has-media-trimmed (:has-media-trimmed file)
+                :ignore-sync-until (:ignore-sync-until file)
+                :project-id (:project-id file)
+                :features (:features file)
+                :name (:name file)
+                :is-shared (:is-shared file)
+                :version (:version file)
+                :data (:data file)
+                :id (:id file)
+                :deleted-at (:deleted-at file)
+                :created-at (:created-at file)
+                :modified-at (:modified-at file)
+                :revn (:revn file)
+                :vern (:vern file)}]
 
-        _      (when (contains? cf/flags :file-schema-validation)
-                 (fval/validate-file-schema! file))
+    (-> (d/without-nils params)
+        (assoc :data-backend nil)
+        (assoc :data-ref-id nil))))
 
-        _      (when (contains? cf/flags :soft-file-schema-validation)
-                 (let [result (ex/try! (fval/validate-file-schema! file))]
-                   (when (ex/exception? result)
-                     (l/error :hint "file schema validation error" :cause result))))
+(defn insert-file!
+  "Insert a new file into the database table"
+  [{:keys [::db/conn] :as cfg} file & {:as opts}]
+  (feat.fmigr/upsert-migrations! conn file)
+  (let [params (-> (encode-file cfg file)
+                   (get-params-from-file))]
+    (db/insert! conn :file params opts)))
 
-        file   (if (contains? (:features file) "fdata/objects-map")
-                 (feat.fdata/enable-objects-map file)
-                 file)
+(defn update-file!
+  "Update an existing file on the database."
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
+  (let [file   (encode-file cfg file)
+        params (-> (get-params-from-file file)
+                   (dissoc :id))]
 
-        file   (if (contains? (:features file) "fdata/pointer-map")
-                 (binding [pmap/*tracked* (pmap/create-tracked)]
-                   (let [file (feat.fdata/enable-pointer-map file)]
-                     (feat.fdata/persist-pointers! cfg id)
-                     file))
-                 file)
+    ;; If file was already offloaded, we touch the underlying storage
+    ;; object for properly trigger storage-gc-touched task
+    (when (feat.fdata/offloaded? file)
+      (some->> (:data-ref-id file) (sto/touch-object! storage)))
 
-        params (-> file
-                   (update :features db/encode-pgarray conn "text")
-                   (update :data blob/encode))]
+    (feat.fmigr/upsert-migrations! conn file)
+    (db/update! conn :file params {:id id} opts)))
 
-    (if (::overwrite cfg)
-      (upsert-file! conn params)
-      (db/insert! conn :file params ::db/return-keys false))
+(defn save-file!
+  "Applies all the final validations and perist the file, binfile
+  specific, should not be used outside of binfile domain"
+  [{:keys [::timestamp] :as cfg} file & {:as opts}]
 
-    file))
+  (assert (dt/instant? timestamp) "expected valid timestamp")
 
+  (let [file (-> file
+                 (assoc :created-at timestamp)
+                 (assoc :modified-at timestamp)
+                 (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
+                 (update :features
+                         (fn [features]
+                           (-> (::features cfg #{})
+                               (set/union features)
+                               ;; We never want to store
+                               ;; frontend-only features on file
+                               (set/difference cfeat/frontend-only-features)))))]
 
-(defn register-pending-migrations
-  "All features that are enabled and requires explicit migration are
-  added to the state for a posterior migration step."
-  [cfg {:keys [id features] :as file}]
-  (doseq [feature (-> (::features cfg)
-                      (set/difference cfeat/no-migration-features)
-                      (set/difference cfeat/backend-only-features)
-                      (set/difference features))]
-    (vswap! *state* update :pending-to-migrate (fnil conj []) [feature id]))
+    (when (contains? cf/flags :file-schema-validation)
+      (fval/validate-file-schema! file))
 
-  file)
+    (when (contains? cf/flags :soft-file-schema-validation)
+      (let [result (ex/try! (fval/validate-file-schema! file))]
+        (when (ex/exception? result)
+          (l/error :hint "file schema validation error" :cause result))))
 
-
-(defn apply-pending-migrations!
-  "Apply alredy registered pending migrations to files"
-  [cfg]
-  (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
-    (case feature
-      "components/v2"
-      (feat.compv2/migrate-file! cfg file-id
-                                 :validate? (::validate cfg true)
-                                 :skip-on-graphic-error? true)
-
-      "fdata/shape-data-type"
-      nil
-
-      (ex/raise :type :internal
-                :code :no-migration-defined
-                :hint (str/ffmt "no migation for feature '%' on file importation" feature)
-                :feature feature))))
+    (insert-file! cfg file opts)))

backend/src/app/binfile/migrations.clj

@@ -0,0 +1,45 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.binfile.migrations
+  "A binfile related migrations handling"
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [app.features.components-v2 :as feat.compv2]
+   [clojure.set :as set]
+   [cuerdas.core :as str]))
+
+(defn register-pending-migrations!
+  "All features that are enabled and requires explicit migration are
+  added to the state for a posterior migration step."
+  [cfg {:keys [id features] :as file}]
+  (doseq [feature (-> (::features cfg)
+                      (set/difference cfeat/no-migration-features)
+                      (set/difference cfeat/backend-only-features)
+                      (set/difference features))]
+    (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature id]))
+
+  file)
+
+(defn apply-pending-migrations!
+  "Apply alredy registered pending migrations to files"
+  [cfg]
+  (doseq [[feature file-id] (-> bfc/*state* deref :pending-to-migrate)]
+    (case feature
+      "components/v2"
+      (feat.compv2/migrate-file! cfg file-id
+                                 :validate? (::validate cfg true)
+                                 :skip-on-graphic-error? true)
+
+      "fdata/shape-data-type"
+      nil
+
+      (ex/raise :type :internal
+                :code :no-migration-defined
+                :hint (str/ffmt "no migation for feature '%' on file importation" feature)
+                :feature feature))))

backend/src/app/binfile/v1.clj

@@ -9,6 +9,7 @@
   (:refer-clojure :exclude [assert])
   (:require
    [app.binfile.common :as bfc]
+   [app.binfile.migrations :as bfm]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -298,7 +299,7 @@
 (defmulti write-section ::section)
 
 (defn write-export!
-  [{:keys [::include-libraries ::embed-assets] :as cfg}]
+  [{:keys [::bfc/include-libraries ::bfc/embed-assets] :as cfg}]
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
@@ -323,7 +324,7 @@
             [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
 
 (defmethod write-section :v1/metadata
-  [{:keys [::output ::ids ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/ids ::bfc/include-libraries] :as cfg}]
   (if-let [fids (get-files cfg ids)]
     (let [lids (when include-libraries
                  (bfc/get-libraries cfg ids))
@@ -335,7 +336,7 @@
               :hint "unable to retrieve files for export")))
 
 (defmethod write-section :v1/files
-  [{:keys [::output ::embed-assets ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/embed-assets ::bfc/include-libraries] :as cfg}]
 
   ;; Initialize SIDS with empty vector
   (vswap! bfc/*state* assoc :sids [])
@@ -382,7 +383,7 @@
       (vswap! bfc/*state* update :sids into bfc/xf-map-media-id thumbnails))))
 
 (defmethod write-section :v1/rels
-  [{:keys [::output ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/include-libraries] :as cfg}]
   (let [ids  (-> bfc/*state* deref :files set)
         rels (when include-libraries
                (bfc/get-files-rels cfg ids))]
@@ -421,25 +422,19 @@
 (defmulti read-import ::version)
 (defmulti read-section ::section)
 
-(s/def ::profile-id ::us/uuid)
-(s/def ::project-id ::us/uuid)
-(s/def ::input io/input-stream?)
-(s/def ::overwrite? (s/nilable ::us/boolean))
+(s/def ::bfc/profile-id ::us/uuid)
+(s/def ::bfc/project-id ::us/uuid)
+(s/def ::bfc/input io/input-stream?)
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
 
-;; FIXME: replace with schema
 (s/def ::read-import-options
-  (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
-          :opt [::overwrite? ::ignore-index-errors?]))
+  (s/keys :req [::db/pool ::sto/storage ::bfc/project-id ::bfc/profile-id ::bfc/input]
+          :opt [::ignore-index-errors?]))
 
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
-  format. There are some options for customize the importation
-  behavior:
-
-  `::bfc/overwrite`: if true, instead of creating new files and remapping id references,
-  it reuses all ids and updates existing objects; defaults to `false`."
-  [{:keys [::input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
+  format."
+  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
 
   (dm/assert!
    "expected input stream"
@@ -453,7 +448,7 @@
     (read-import (assoc options ::version version ::bfc/timestamp timestamp))))
 
 (defn- read-import-v1
-  [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}]
+  [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}]
 
   (bfc/disable-database-timeouts! cfg)
 
@@ -473,13 +468,13 @@
                 (let [options (-> cfg
                                   (assoc ::bfc/features features)
                                   (assoc ::section section)
-                                  (assoc ::input input))]
+                                  (assoc ::bfc/input input))]
                   (binding [bfc/*options* options]
                     (events/tap :progress {:op :import :section section})
                     (read-section options))))
               [:v1/metadata :v1/files :v1/rels :v1/sobjects])
 
-        (bfc/apply-pending-migrations! cfg)
+        (bfm/apply-pending-migrations! cfg)
 
         ;; Knowing that the ids of the created files are in index,
         ;; just lookup them and return it as a set
@@ -491,7 +486,7 @@
   (db/tx-run! options read-import-v1))
 
 (defmethod read-section :v1/metadata
-  [{:keys [::input]}]
+  [{:keys [::bfc/input]}]
   (let [{:keys [version files]} (read-obj! input)]
     (l/dbg :hint "metadata readed"
            :version (:full version)
@@ -508,18 +503,8 @@
               (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/")))))
         thumbnails))
 
-(defn- clean-features
-  [file]
-  (update file :features (fn [features]
-                           (if (set? features)
-                             (-> features
-                                 (cfeat/migrate-legacy-features)
-                                 (set/difference cfeat/backend-only-features))
-                             #{}))))
-
 (defmethod read-section :v1/files
-  [{:keys [::db/conn ::input ::project-id ::bfc/overwrite ::name] :as system}]
-
+  [{:keys [::bfc/input ::bfc/project-id ::bfc/name] :as system}]
   (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))]
     (let [file       (read-obj! input)
           media      (read-obj! input)
@@ -527,7 +512,7 @@
           file-id    (:id file)
           file-id'   (bfc/lookup-index file-id)
 
-          file       (clean-features file)
+          file       (bfc/clean-file-features file)
           thumbnails (:thumbnails file)]
 
       (when (not= file-id expected-file-id)
@@ -577,15 +562,12 @@
           (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id']))
 
         (l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
-        (bfc/persist-file! system file)
-
-        (when overwrite
-          (db/delete! conn :file-thumbnail {:file-id file-id'}))
+        (bfc/save-file! system file ::db/return-keys false)
 
         file-id'))))
 
 (defmethod read-section :v1/rels
-  [{:keys [::db/conn ::input ::bfc/timestamp]}]
+  [{:keys [::db/conn ::bfc/input ::bfc/timestamp]}]
   (let [rels (read-obj! input)
         ids  (into #{} (-> bfc/*state* deref :files))]
     ;; Insert all file relations
@@ -609,7 +591,7 @@
                   ::l/sync? true))))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::db/conn ::input ::bfc/overwrite ::bfc/timestamp] :as cfg}]
+  [{:keys [::db/conn ::bfc/input ::bfc/timestamp] :as cfg}]
   (let [storage (sto/resolve cfg)
         ids     (read-obj! input)
         thumb?  (into #{} (map :media-id) (:thumbnails @bfc/*state*))]
@@ -662,8 +644,7 @@
                       (-> item
                           (assoc :file-id file-id)
                           (d/update-when :media-id bfc/lookup-index)
-                          (d/update-when :thumbnail-id bfc/lookup-index))
-                      {::db/on-conflict-do-nothing? overwrite}))))
+                          (d/update-when :thumbnail-id bfc/lookup-index))))))
 
     (doseq [item (:thumbnails @bfc/*state*)]
       (let [item (update item :media-id bfc/lookup-index)]
@@ -672,8 +653,7 @@
                :media-id (str (:media-id item))
                :object-id (:object-id item)
                ::l/sync? true)
-        (db/insert! conn :file-tagged-object-thumbnail item
-                    {::db/on-conflict-do-nothing? overwrite})))))
+        (db/insert! conn :file-tagged-object-thumbnail item)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HIGH LEVEL API
@@ -683,17 +663,17 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
   (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
+   "expected a set of uuid's for `::bfc/ids` parameter"
    (and (set? ids)
         (every? uuid? ids)))
 
@@ -728,12 +708,12 @@
                 :cause @cs)))))
 
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg))))
 
   (dm/assert!
    "expected instance of jio/IOFactory for `input`"
@@ -747,7 +727,7 @@
     (try
       (binding [*position* (atom 0)]
         (pu/with-open [input (io/input-stream input)]
-          (read-import! (assoc cfg ::input input))))
+          (read-import! (assoc cfg ::bfc/input input))))
 
       (catch ZstdIOException cause
         (ex/raise :type :validation

backend/src/app/binfile/v2.clj

@@ -297,7 +297,7 @@
                         (set/difference (:features file)))]
       (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature (:id file)]))
 
-    (bfc/persist-file! cfg file))
+    (bfc/save-file! cfg file ::db/return-keys false))
 
   (doseq [thumbnail (read-seq cfg :file-object-thumbnail file-id)]
     (let [thumbnail (-> thumbnail

backend/src/app/binfile/v3.clj

@@ -9,9 +9,11 @@
   (:refer-clojure :exclude [read])
   (:require
    [app.binfile.common :as bfc]
+   [app.binfile.migrations :as bfm]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.schema :as sm]
@@ -22,10 +24,12 @@
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
+   [app.common.types.tokens-lib :as cto]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.storage.impl :as sto.impl]
    [app.util.events :as events]
@@ -55,7 +59,7 @@
      [:map
       [:id ::sm/uuid]
       [:name :string]
-      [:project-id ::sm/uuid]]]]
+      [:features ::cfeat/features]]]]
 
    [:relations {:optional true}
     [:vector
@@ -77,10 +81,15 @@
    [:tag :string]
    [:media-id ::sm/uuid]])
 
+(def ^:private schema:file
+  [:merge
+   ctf/schema:file
+   [:map [:options {:optional true} ctf/schema:options]]])
+
 ;; --- ENCODERS
 
 (def encode-file
-  (sm/encoder ::ctf/file sm/json-transformer))
+  (sm/encoder schema:file sm/json-transformer))
 
 (def encode-page
   (sm/encoder ::ctp/page sm/json-transformer))
@@ -100,6 +109,9 @@
 (def encode-typography
   (sm/encoder ::cty/typography sm/json-transformer))
 
+(def encode-tokens-lib
+  (sm/encoder ::cto/tokens-lib sm/json-transformer))
+
 (def encode-plugin-data
   (sm/encoder ::ctpg/plugin-data sm/json-transformer))
 
@@ -124,7 +136,7 @@
   (sm/decoder ::ctcl/color sm/json-transformer))
 
 (def decode-file
-  (sm/decoder ::ctf/file sm/json-transformer))
+  (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
   (sm/decoder ::ctp/page sm/json-transformer))
@@ -135,6 +147,9 @@
 (def decode-typography
   (sm/decoder ::cty/typography sm/json-transformer))
 
+(def decode-tokens-lib
+  (sm/decoder ::cto/tokens-lib sm/json-transformer))
+
 (def decode-plugin-data
   (sm/decoder ::ctpg/plugin-data sm/json-transformer))
 
@@ -170,6 +185,9 @@
 (def validate-typography
   (sm/check-fn ::cty/typography))
 
+(def validate-tokens-lib
+  (sm/check-fn ::cto/tokens-lib))
+
 (def validate-plugin-data
   (sm/check-fn ::ctpg/plugin-data))
 
@@ -190,20 +208,24 @@
   (.closeEntry output))
 
 (defn- get-file
-  [{:keys [::embed-assets ::include-libraries] :as cfg} file-id]
+  [{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id]
 
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
   (let [detach?  (and (not embed-assets) (not include-libraries))]
-    (cond-> (bfc/get-file cfg file-id)
-      detach?
-      (-> (ctf/detach-external-references file-id)
-          (dissoc :libraries))
+    (db/tx-run! cfg (fn [cfg]
+                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
+                        detach?
+                        (-> (ctf/detach-external-references file-id)
+                            (dissoc :libraries))
 
-      embed-assets
-      (update :data #(bfc/embed-assets cfg % file-id)))))
+                        embed-assets
+                        (update :data #(bfc/embed-assets cfg % file-id))
+
+                        :always
+                        (bfc/clean-file-features))))))
 
 (defn- resolve-extension
   [mtype]
@@ -242,6 +264,7 @@
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
   (let [file         (get-file cfg file-id)
+
         media        (->> (bfc/get-file-media cfg file)
                           (map (fn [media]
                                  (dissoc media :file-id))))
@@ -250,6 +273,7 @@
         typographies (:typographies data)
         components   (:components data)
         colors       (:colors data)
+        tokens-lib   (:tokens-lib data)
 
         pages        (:pages data)
         pages-index  (:pages-index data)
@@ -258,12 +282,16 @@
 
     (vswap! bfc/*state* update :files assoc file-id
             {:id file-id
-             :project-id (:project-id file)
-             :name (:name file)})
+             :name (:name file)
+             :features (:features file)})
 
-    (let [file (cond-> (dissoc file :data)
+    (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
                  (assoc :options (:options data))
+
+                 :always
+                 (dissoc :data)
+
                  :always
                  (encode-file))
           path (str "files/" file-id ".json")]
@@ -319,12 +347,17 @@
         (write-entry! output path color)))
 
     (doseq [[id object] typographies]
-      (let [path  (str "files/" file-id "/typographies/" id ".json")
-            color (encode-typography object)]
-        (write-entry! output path color)))))
+      (let [path       (str "files/" file-id "/typographies/" id ".json")
+            typography (encode-typography object)]
+        (write-entry! output path typography)))
+
+    (when tokens-lib
+      (let [path           (str "files/" file-id "/tokens.json")
+            encoded-tokens (encode-tokens-lib tokens-lib)]
+        (write-entry! output path encoded-tokens)))))
 
 (defn- export-files
-  [{:keys [::ids ::include-libraries ::output] :as cfg}]
+  [{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}]
   (let [ids  (into ids (when include-libraries (bfc/get-libraries cfg ids)))
         rels (if include-libraries
                (->> (bfc/get-files-rels cfg ids)
@@ -457,6 +490,14 @@
         {:entry entry
          :id (parse-uuid id)}))))
 
+(defn- match-tokens-lib-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/tokens.json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry}))))
+
 (defn- match-thumbnail-entry-fn
   [file-id]
   (let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+)/([^/]+).json$")
@@ -490,7 +531,7 @@
 
 (defn- match-storage-entry-fn
   []
-  (let [pattern (str "^objects/([^/]+).json$")
+  (let [pattern "^objects/([^/]+).json$"
         pattern (re-pattern pattern)]
     (fn [entry]
       (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
@@ -502,8 +543,13 @@
   (with-open [reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
+(defn- read-plain-entry
+  [^ZipFile input entry]
+  (with-open [reader (zip-entry-reader input entry)]
+    (json/read reader)))
+
 (defn- read-file
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
@@ -511,7 +557,7 @@
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -520,7 +566,7 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -534,7 +580,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -547,7 +593,7 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-component-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -560,7 +606,7 @@
        (not-empty)))
 
 (defn- read-file-typographies
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -572,8 +618,15 @@
                {})
        (not-empty)))
 
+(defn- read-file-tokens-lib
+  [{:keys [::bfc/input ::file-id ::entries]}]
+  (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
+    (->> (read-plain-entry input entry)
+         (decode-tokens-lib)
+         (validate-tokens-lib))))
+
 (defn- read-file-shapes
-  [{:keys [::input ::file-id ::page-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -586,7 +639,7 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
@@ -602,7 +655,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -617,9 +670,10 @@
        (not-empty)))
 
 (defn- read-file-data
-  [{:keys [] :as cfg}]
+  [cfg]
   (let [colors       (read-file-colors cfg)
         typographies (read-file-typographies cfg)
+        tokens-lib   (read-file-tokens-lib cfg)
         components   (read-file-components cfg)
         plugin-data  (read-file-plugin-data cfg)
         pages        (read-file-pages cfg)]
@@ -628,11 +682,12 @@
      :pages-index (into {} pages)
      :colors colors
      :typographies typographies
+     :tokens-lib tokens-lib
      :components components
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::db/conn ::project-id ::file-id ::file-name] :as cfg}]
+  [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
   (let [file-id'   (bfc/lookup-index file-id)
         file       (read-file cfg)
         media      (read-file-media cfg)
@@ -672,7 +727,7 @@
                    (cond-> (:options file)
                      (assoc :options (:options file))))
 
-          file (-> file
+          file (-> (select-keys file bfc/file-attrs)
                    (assoc :id file-id')
                    (assoc :data data)
                    (assoc :name file-name)
@@ -680,12 +735,9 @@
                    (dissoc :options)
                    (bfc/process-file))]
 
-      (->> file
-           (bfc/register-pending-migrations cfg)
-           (bfc/persist-file! cfg))
 
-      (when (::bfc/overwrite cfg)
-        (db/delete! conn :file-thumbnail {:file-id file-id'}))
+      (bfm/register-pending-migrations! cfg file)
+      (bfc/save-file! cfg file ::db/return-keys false)
 
       file-id')))
 
@@ -708,7 +760,7 @@
                      :library-file-id libr-id})))))
 
 (defn- import-storage-objects
-  [{:keys [::input ::entries ::bfc/timestamp] :as cfg}]
+  [{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}]
   (events/tap :progress {:section :storage-objects})
 
   (let [storage (sto/resolve cfg)
@@ -775,12 +827,12 @@
                      (d/update-when :thumbnail-id bfc/lookup-index))]
 
       (l/dbg :hint "inserting file media object"
+             :old-id (str (:id item))
              :id (str (:id params))
              :file-id (str (:file-id params))
              ::l/sync? true)
 
-      (db/insert! conn :file-media-object params
-                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+      (db/insert! conn :file-media-object params))))
 
 (defn- import-file-thumbnails
   [{:keys [::db/conn] :as cfg}]
@@ -800,11 +852,10 @@
              :media-id (str media-id)
              ::l/sync? true)
 
-      (db/insert! conn :file-tagged-object-thumbnail params
-                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+      (db/insert! conn :file-tagged-object-thumbnail params))))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::input ::name] :or {timestamp (dt/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
 
   (dm/assert!
    "expected zip file"
@@ -825,14 +876,17 @@
                 :manifest manifest))
 
     ;; Check if all files referenced on manifest are present
-    (doseq [{file-id :id} (:files manifest)]
+    (doseq [{file-id :id features :features} (:files manifest)]
       (let [path (str "files/" file-id ".json")]
+
         (when-not (get-zip-entry input path)
           (ex/raise :type :validation
                     :code :invalid-binfile-v3
                     :hint "some files referenced on manifest not found"
                     :path path
-                    :file-id file-id))))
+                    :file-id file-id))
+
+        (cfeat/check-supported-features! features)))
 
     (events/tap :progress {:section :manifest})
 
@@ -862,7 +916,7 @@
                             (import-file-media cfg)
                             (import-file-thumbnails cfg)
 
-                            (bfc/apply-pending-migrations! cfg)
+                            (bfm/apply-pending-migrations! cfg)
 
                             ids)))))))
 
@@ -872,17 +926,17 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
   (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
+   "expected a set of uuid's for `::bfc/ids` parameter"
    (and (set? ids)
         (every? uuid? ids)))
 
@@ -924,14 +978,13 @@
                 :aborted @ab
                 :cause @cs)))))
 
-
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg))))
 
   (dm/assert!
    "expected instance of jio/IOFactory for `input`"
@@ -944,7 +997,7 @@
     (l/info :hint "import: started" :id (str id))
     (try
       (with-open [input (ZipFile. (fs/file input))]
-        (import-files (assoc cfg ::input input)))
+        (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
         (vreset! cs cause)

backend/src/app/config.clj

@@ -12,6 +12,7 @@
    [app.common.exceptions :as ex]
    [app.common.flags :as flags]
    [app.common.schema :as sm]
+   [app.common.uri :as u]
    [app.common.version :as v]
    [app.util.overrides]
    [app.util.time :as dt]
@@ -42,7 +43,6 @@
    :rpc-rlimit-config "resources/rlimit.edn"
    :rpc-climit-config "resources/climit.edn"
 
-   :auto-file-snapshot-total 10
    :auto-file-snapshot-every 5
    :auto-file-snapshot-timeout "3h"
 
@@ -101,7 +101,6 @@
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
 
-    [:auto-file-snapshot-total {:optional true} ::sm/int]
     [:auto-file-snapshot-every {:optional true} ::sm/int]
     [:auto-file-snapshot-timeout {:optional true} ::dt/duration]
 
@@ -144,6 +143,8 @@
     [:quotes-comments-per-file {:optional true} ::sm/int]
     [:quotes-snapshots-per-file {:optional true} ::sm/int]
     [:quotes-snapshots-per-team {:optional true} ::sm/int]
+    [:quotes-team-access-requests-per-team {:optional true} ::sm/int]
+    [:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
 
     [:auth-data-cookie-domain {:optional true} :string]
     [:auth-token-cookie-name {:optional true} :string]
@@ -228,19 +229,16 @@
     [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
     [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
-(def default-flags
-  [:enable-backend-api-doc
-   :enable-backend-openapi-doc
-   :enable-backend-worker
-   :enable-secure-session-cookies
-   :enable-email-verification
-   :enable-v2-migration])
-
 (defn- parse-flags
   [config]
-  (flags/parse flags/default
-               default-flags
-               (:flags config)))
+  (let [public-uri  (c/get config :public-uri)
+        public-uri  (some-> public-uri (u/uri))
+        extra-flags (if (and public-uri
+                             (= (:scheme public-uri) "http")
+                             (not= (:host public-uri) "localhost"))
+                      #{:disable-secure-session-cookies}
+                      #{})]
+    (flags/parse flags/default extra-flags (:flags config))))
 
 (defn read-env
   [prefix]

backend/src/app/db.clj

@@ -270,19 +270,17 @@
     :else           (throw (IllegalArgumentException. "unable to resolve connectable"))))
 
 (def ^:private params-mapping
-  {::return-keys? :return-keys
-   ::return-keys :return-keys})
+  {::return-keys :return-keys})
 
 (defn rename-opts
   [opts]
   (set/rename-keys opts params-mapping))
 
 (def ^:private default-insert-opts
-  {:builder-fn sql/as-kebab-maps
-   :return-keys true})
+  (assoc sql/default-opts :return-keys true))
 
 (def ^:private default-opts
-  {:builder-fn sql/as-kebab-maps})
+  sql/default-opts)
 
 (defn exec!
   ([ds sv] (exec! ds sv nil))
@@ -333,7 +331,7 @@
 (defn update!
   "A helper that build an UPDATE SQL statement and executes it.
 
-   Given a connectable object, a table name, a hash map of columns and
+  Given a connectable object, a table name, a hash map of columns and
   values to set, and either a hash map of columns and values to search
   on or a vector of a SQL where clause and parameters, perform an
   update on the table.
@@ -413,10 +411,20 @@
                 :hint "database object not found"))
     row))
 
+(def ^:private default-plan-opts
+  (-> default-opts
+      (assoc :fetch-size 1000)
+      (assoc :concurrency :read-only)
+      (assoc :cursors :close)
+      (assoc :result-type :forward-only)))
+
 (defn plan
-  [ds sql]
-  (-> (get-connectable ds)
-      (jdbc/plan sql sql/default-opts)))
+  ([ds sql]
+   (-> (get-connectable ds)
+       (jdbc/plan sql default-plan-opts)))
+  ([ds sql opts]
+   (-> (get-connectable ds)
+       (jdbc/plan sql (merge default-plan-opts opts)))))
 
 (defn cursor
   "Return a lazy seq of rows using server side cursors"

backend/src/app/db/sql.clj

@@ -15,14 +15,15 @@
 (defn kebab-case [s] (str/replace s #"_" "-"))
 (defn snake-case [s] (str/replace s #"-" "_"))
 
-(def default-opts
-  {:table-fn snake-case
-   :column-fn snake-case})
-
 (defn as-kebab-maps
   [rs opts]
   (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
 
+(def default-opts
+  {:table-fn snake-case
+   :column-fn snake-case
+   :builder-fn as-kebab-maps})
+
 (defn insert
   ([table key-map]
    (insert table key-map nil))
@@ -38,7 +39,10 @@
 
 (defn insert-many
   [table cols rows opts]
-  (let [opts (merge default-opts opts)]
+  (let [opts (merge default-opts opts)
+        opts (cond-> opts
+               (::on-conflict-do-nothing opts)
+               (assoc :suffix "ON CONFLICT DO NOTHING"))]
     (sql/for-insert-multi table cols rows opts)))
 
 (defn select

backend/src/app/email.clj

@@ -226,8 +226,8 @@
    [:priority {:optional true} [:enum :high :low]]
    [:extra-data {:optional true} ::sm/text]])
 
-(def ^:private valid-context?
-  (sm/validator schema:context))
+(def ^:private check-context
+  (sm/check-fn schema:context))
 
 (defn template-factory
   [& {:keys [id schema]}]
@@ -236,10 +236,8 @@
                    (sm/check-fn schema)
                    (constantly nil))]
     (fn [context]
-      (assert (valid-context? context) "expected a valid context")
-      (check-fn context)
-
-      (let [email (build-email-template id context)]
+      (let [context (-> context check-context check-fn)
+            email   (build-email-template id context)]
         (when-not email
           (ex/raise :type :internal
                     :code :email-template-does-not-exists
@@ -271,7 +269,7 @@
   "Schedule an already defined email to be sent using asynchronously
   using worker task."
   [{:keys [::conn ::factory] :as context}]
-  (assert (db/connection? conn) "expected a valid database connection")
+  (assert (db/connectable? conn) "expected a valid database connection or pool")
 
   (let [email (if factory
                 (factory context)
@@ -348,7 +346,7 @@
    [:subject ::sm/text]
    [:content ::sm/text]])
 
-(def feedback
+(def user-feedback
   "A profile feedback email."
   (template-factory
    :id ::feedback
@@ -451,6 +449,45 @@
    :id ::request-team-access
    :schema schema:request-team-access))
 
+(def ^:private schema:comment-mention
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-mention
+  (template-factory
+   :id ::comment-mention
+   :schema schema:comment-mention))
+
+(def ^:private schema:comment-thread
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-thread
+  (template-factory
+   :id ::comment-thread
+   :schema schema:comment-thread))
+
+(def ^:private schema:comment-notification
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-notification
+  (template-factory
+   :id ::comment-notification
+   :schema schema:comment-notification))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; BOUNCE/COMPLAINS HELPERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/features/components_v2.clj

@@ -884,8 +884,10 @@
                                :shapes (or (:shapes shape) [])
                                :hide-in-viewer (if frame? (boolean (:hide-in-viewer shape)) true)
                                :show-content   (if frame? (boolean (:show-content shape)) true)
-                               :rx (or (:rx shape) 0)
-                               :ry (or (:ry shape) 0)))
+                               :r1 (or (:r1 shape) 0)
+                               :r2 (or (:r2 shape) 0)
+                               :r3 (or (:r3 shape) 0)
+                               :r4 (or (:r4 shape) 0)))
                       shape))]
             (-> file-data
                 (update :pages-index update-vals fix-container)
@@ -1628,9 +1630,19 @@
             fdata (migrate-graphics fdata)]
         (update fdata :options assoc :components-v2 true)))))
 
+;; FIXME: revisit this fn
+(defn- fix-version*
+  [{:keys [version] :as file}]
+  (if (int? version)
+    file
+    (let [version (or (-> file :data :version) 0)]
+      (-> file
+          (assoc :version version)
+          (update :data dissoc :version)))))
+
 (defn- fix-version
   [file]
-  (let [file (fmg/fix-version file)]
+  (let [file (fix-version* file)]
     (if (> (:version file) 22)
       (assoc file :version 22)
       file)))

backend/src/app/features/file_migrations.clj

@@ -0,0 +1,39 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.file-migrations
+  "Backend specific code for file migrations. Implemented as permanent feature of files."
+  (:require
+   [app.common.data :as d]
+   [app.common.files.migrations :as fmg :refer [xf:map-name]]
+   [app.db :as db]
+   [app.db.sql :as-alias sql]))
+
+(def ^:private sql:get-file-migrations
+  "SELECT name FROM file_migration WHERE file_id = ? ORDER BY created_at ASC")
+
+(defn resolve-applied-migrations
+  [cfg {:keys [id] :as file}]
+  (let [conn (db/get-connection cfg)]
+    (assoc file :migrations
+           (->> (db/plan conn [sql:get-file-migrations id])
+                (transduce xf:map-name conj (d/ordered-set))
+                (not-empty)))))
+
+(defn upsert-migrations!
+  "Persist or update file migrations. Return the updated/inserted number
+  of rows"
+  [conn {:keys [id] :as file}]
+  (let [migrations (or (-> file meta ::fmg/migrated)
+                       (-> file :migrations not-empty)
+                       fmg/available-migrations)
+        columns    [:file-id :name]
+        rows       (mapv (fn [name] [id name]) migrations)]
+
+    (-> (db/insert-many! conn :file-migration columns rows
+                         {::db/return-keys false
+                          ::sql/on-conflict-do-nothing true})
+        (db/get-update-count))))

backend/src/app/http.clj

@@ -42,8 +42,8 @@
 (def default-params
   {::port 6060
    ::host "0.0.0.0"
-   ::max-body-size (* 1024 1024 30)             ; default 30 MiB
-   ::max-multipart-body-size (* 1024 1024 120)}) ; default 120 MiB
+   ::max-body-size 31457280              ; default  30 MiB
+   ::max-multipart-body-size 367001600}) ; default 350 MiB
 
 (defmethod ig/expand-key ::server
   [k v]

backend/src/app/http/debug.clj

@@ -7,9 +7,12 @@
 (ns app.http.debug
   (:refer-clojure :exclude [error-handler])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
    [app.common.uuid :as uuid]
@@ -19,6 +22,7 @@
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
+   [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
    [app.srepl.helpers :as srepl]
    [app.storage :as-alias sto]
@@ -280,23 +284,23 @@
       (ex/raise :type :validation
                 :code :missing-arguments))
 
-    (let [path (tmp/tempfile :prefix "penpot.export.")]
+    (let [path (tmp/tempfile :prefix "penpot.export." :min-age "30m")]
       (with-open [output (io/output-stream path)]
         (-> cfg
-            (assoc ::bf.v1/ids file-ids)
-            (assoc ::bf.v1/embed-assets embed?)
-            (assoc ::bf.v1/include-libraries libs?)
-            (bf.v1/export-files! output)))
+            (assoc ::bfc/ids file-ids)
+            (assoc ::bfc/embed-assets embed?)
+            (assoc ::bfc/include-libraries libs?)
+            (bf.v3/export-files! output)))
 
       (if clone?
         (let [profile    (profile/get-profile pool profile-id)
               project-id (:default-project-id profile)
               cfg        (assoc cfg
-                                ::bf.v1/overwrite false
-                                ::bf.v1/profile-id profile-id
-                                ::bf.v1/project-id project-id
-                                ::bf.v1/input path)]
-          (bf.v1/import-files! cfg)
+                                ::bfc/overwrite false
+                                ::bfc/profile-id profile-id
+                                ::bfc/project-id project-id
+                                ::bfc/input path)]
+          (bf.v3/import-files! cfg)
           {::yres/status  200
            ::yres/headers {"content-type" "text/plain"}
            ::yres/body    "OK CLONED"})
@@ -316,22 +320,27 @@
 
   (let [profile    (profile/get-profile pool profile-id)
         project-id (:default-project-id profile)
-        overwrite? (contains? params :overwrite)
-        migrate?   (contains? params :migrate)]
+        team       (teams/get-team pool
+                                   :profile-id profile-id
+                                   :project-id project-id)]
 
     (when-not project-id
       (ex/raise :type :validation
                 :code :missing-project
                 :hint "project not found"))
 
-    (let [path (-> params :file :path)
-          cfg  (assoc cfg
-                      ::bf.v1/overwrite overwrite?
-                      ::bf.v1/migrate migrate?
-                      ::bf.v1/profile-id profile-id
-                      ::bf.v1/project-id project-id
-                      ::bf.v1/input path)]
-      (bf.v1/import-files! cfg)
+    (let [path   (-> params :file :path)
+          format (bfc/parse-file-format path)
+          cfg    (assoc cfg
+                        ::bfc/profile-id profile-id
+                        ::bfc/project-id project-id
+                        ::bfc/input path
+                        ::bfc/features (cfeat/get-team-enabled-features cf/flags team))]
+
+      (if (= format :binfile-v3)
+        (bf.v3/import-files! cfg)
+        (bf.v1/import-files! cfg))
+
       {::yres/status  200
        ::yres/headers {"content-type" "text/plain"}
        ::yres/body    "OK"})))

backend/src/app/http/errors.clj

@@ -55,13 +55,16 @@
    ::yres/body (ex-data err)})
 
 (defmethod handle-error :restriction
-  [err _ _]
+  [err request _]
   (let [{:keys [code] :as data} (ex-data err)]
     (if (= code :method-not-allowed)
       {::yres/status 405
        ::yres/body data}
-      {::yres/status 400
-       ::yres/body data})))
+
+      (binding [l/*context* (request->context request)]
+        (l/err :hint "restriction error" :data data)
+        {::yres/status 400
+         ::yres/body data}))))
 
 (defmethod handle-error :rate-limit
   [err _ _]

backend/src/app/http/sse.clj

@@ -64,7 +64,8 @@
                           (catch Throwable cause
                             (events/tap :error (errors/handle' cause request))
                             (when-not (ex/instance? java.io.EOFException cause)
-                              (l/err :hint "unexpected error on processing sse response" :cause cause)))
+                              (binding [l/*context* (errors/request->context request)]
+                                (l/err :hint "unexpected error on processing sse response" :cause cause))))
                           (finally
                             (sp/close! events/*channel*)
                             (px/await! listener)))))))}))

backend/src/app/loggers/database.clj

@@ -59,7 +59,7 @@
         :props   (pp/pprint-str props :length 50)
         :hint    (or (ex-message cause) @message)
         :trace   (or (::trace record)
-                     (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))}
+                     (some-> cause (ex/format-throwable :data? false :explain? false :header? false :summary? false)))}
 
        (when-let [params (or (:request/params context) (:params context))]
          {:params (pp/pprint-str params :length 30 :level 13)})
@@ -74,9 +74,8 @@
          {:explain explain})))))
 
 (defn error-record?
-  [{:keys [::l/level ::l/cause]}]
-  (and (= :error level)
-       (ex/exception? cause)))
+  [{:keys [::l/level]}]
+  (= :error level))
 
 (defn- handle-event
   [{:keys [::db/pool]} {:keys [::l/id] :as record}]

backend/src/app/main.clj

@@ -25,7 +25,6 @@
    [app.loggers.webhooks :as-alias webhooks]
    [app.metrics :as-alias mtx]
    [app.metrics.definition :as-alias mdef]
-   [app.migrations.v2 :as migrations.v2]
    [app.msgbus :as-alias mbus]
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
@@ -349,7 +348,6 @@
      :file-gc            (ig/ref :app.tasks.file-gc/handler)
      :file-gc-scheduler  (ig/ref :app.tasks.file-gc-scheduler/handler)
      :offload-file-data  (ig/ref :app.tasks.offload-file-data/handler)
-     :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
      :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
      :telemetry          (ig/ref :app.tasks.telemetry/handler)
      :storage-gc-deleted (ig/ref ::sto.gc-deleted/handler)
@@ -405,10 +403,6 @@
    {::db/pool     (ig/ref ::db/pool)
     ::sto/storage (ig/ref ::sto/storage)}
 
-   :app.tasks.file-xlog-gc/handler
-   {::db/pool     (ig/ref ::db/pool)
-    ::sto/storage (ig/ref ::sto/storage)}
-
    :app.tasks.telemetry/handler
    {::db/pool            (ig/ref ::db/pool)
     ::http.client/client (ig/ref ::http.client/client)
@@ -614,11 +608,6 @@
         (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
 
       (start)
-
-      (when (contains? cf/flags :v2-migration)
-        (px/sleep 5000)
-        (migrations.v2/migrate app.main/system))
-
       (deref p))
     (catch Throwable cause
       (ex/print-throwable cause)

backend/src/app/migrations.clj

@@ -426,7 +426,19 @@
     :fn (mg/resource "app/migrations/sql/0134-mod-file-change-table.sql")}
 
    {:name "0135-mod-team-invitation-table.sql"
-    :fn (mg/resource "app/migrations/sql/0135-mod-team-invitation-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0135-mod-team-invitation-table.sql")}
+
+   {:name "0136-mod-comments-mentions.sql"
+    :fn (mg/resource "app/migrations/sql/0136-mod-comments-mentions.sql")}
+
+   {:name "0137-add-file-migration-table.sql"
+    :fn (mg/resource "app/migrations/sql/0137-add-file-migration-table.sql")}
+
+   {:name "0138-mod-file-data-fragment-table.sql"
+    :fn (mg/resource "app/migrations/sql/0138-mod-file-data-fragment-table.sql")}
+
+   {:name "0139-mod-file-change-table.sql"
+    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/media_refs.clj

@@ -0,0 +1,49 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.migrations.media-refs
+  "A media refs migration fixer script"
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.pprint]
+   [app.srepl.fixes.media-refs :refer [process-file]]
+   [app.srepl.main :as srepl]
+   [clojure.edn :as edn]))
+
+(def ^:private required-services
+  [:app.storage.s3/backend
+   :app.storage.fs/backend
+   :app.storage/storage
+   :app.metrics/metrics
+   :app.db/pool
+   :app.worker/executor])
+
+(defn -main
+  [& [options]]
+  (try
+    (let [config-var (requiring-resolve 'app.main/system-config)
+          start-var  (requiring-resolve 'app.main/start-custom)
+          stop-var   (requiring-resolve 'app.main/stop)
+          config     (select-keys @config-var required-services)]
+
+      (start-var config)
+
+      (let [options (if (string? options)
+                      (ex/ignoring (edn/read-string options))
+                      {})]
+
+        (l/inf :hint "executing media-refs migration" :options options)
+        (srepl/process-files! process-file options))
+
+      (stop-var)
+      (System/exit 0))
+    (catch Throwable cause
+      (ex/print-throwable cause)
+      (flush)
+      (System/exit -1))))
+
+

backend/src/app/migrations/sql/0136-mod-comments-mentions.sql

@@ -0,0 +1,3 @@
+ALTER TABLE comment ADD COLUMN mentions uuid[] NULL DEFAULT '{}';
+
+ALTER TABLE comment_thread ADD COLUMN mentions uuid[] NULL DEFAULT '{}';

backend/src/app/migrations/sql/0137-add-file-migration-table.sql

@@ -0,0 +1,7 @@
+CREATE TABLE file_migration (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  name text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  PRIMARY KEY(file_id, name)
+);

backend/src/app/migrations/sql/0138-mod-file-data-fragment-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_data_fragment
+ALTER CONSTRAINT file_data_fragment_file_id_fkey DEFERRABLE INITIALLY DEFERRED;

backend/src/app/migrations/sql/0139-mod-file-change-table.sql

@@ -0,0 +1,5 @@
+ALTER TABLE file_change
+ DROP CONSTRAINT file_change_file_id_fkey,
+ DROP CONSTRAINT file_change_profile_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE,
+  ADD FOREIGN KEY (profile_id) REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;

backend/src/app/migrations/v2.clj

@@ -1,103 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.migrations.v2
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.logging :as l]
-   [app.db :as db]
-   [app.features.components-v2 :as feat]
-   [app.setup :as setup]
-   [app.util.time :as dt]))
-
-(def ^:private sql:get-teams
-  "SELECT id, features,
-          row_number() OVER (ORDER BY created_at DESC) AS rown
-    FROM team
-   WHERE deleted_at IS NULL
-     AND (not (features @> '{components/v2}') OR features IS NULL)
-   ORDER BY created_at DESC")
-
-(defn- get-teams
-  [conn]
-  (->> (db/cursor conn [sql:get-teams] {:chunk-size 1})
-       (map feat/decode-row)))
-
-(defn- migrate-teams
-  [{:keys [::db/conn] :as system}]
-  ;; Allow long running transaction for this connection
-  (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-
-  ;; Do not allow other migration running in the same time
-  (db/xact-lock! conn 0)
-
-  ;; Run teams migration
-  (run! (fn [{:keys [id rown]}]
-          (try
-            (-> (assoc system ::db/rollback false)
-                (feat/migrate-team! id
-                                    :rown rown
-                                    :label "v2-migration"
-                                    :validate? false
-                                    :skip-on-graphics-error? true))
-            (catch Throwable _
-              (swap! feat/*stats* update :errors (fnil inc 0))
-              (l/wrn :hint "error on migrating team (skiping)"))))
-        (get-teams conn))
-
-  (setup/set-prop! system :v2-migrated true))
-
-(defn migrate
-  [system]
-  (let [tpoint    (dt/tpoint)
-        stats     (atom {})
-        migrated? (setup/get-prop system :v2-migrated false)]
-
-    (when-not migrated?
-      (l/inf :hint "v2 migration started")
-      (try
-        (binding [feat/*stats* stats]
-          (db/tx-run! system migrate-teams))
-
-        (let [stats   (deref stats)
-              elapsed (dt/format-duration (tpoint))]
-          (l/inf :hint "v2 migration finished"
-                 :files (:processed-files stats)
-                 :teams (:processed-teams stats)
-                 :errors (:errors stats)
-                 :elapsed elapsed))
-
-        (catch Throwable cause
-          (l/err :hint "error on aplying v2 migration" :cause cause))))))
-
-(def ^:private required-services
-  [[:app.main/assets :app.storage.s3/backend]
-   [:app.main/assets :app.storage.fs/backend]
-   :app.storage/storage
-   :app.db/pool
-   :app.setup/props
-   :app.svgo/optimizer
-   :app.metrics/metrics
-   :app.migrations/migrations
-   :app.http.client/client])
-
-(defn -main
-  [& _args]
-  (try
-    (let [config-var (requiring-resolve 'app.main/system-config)
-          start-var  (requiring-resolve 'app.main/start-custom)
-          stop-var   (requiring-resolve 'app.main/stop)
-          system-var (requiring-resolve 'app.main/system)
-          config     (select-keys @config-var required-services)]
-
-      (start-var config)
-      (migrate @system-var)
-      (stop-var)
-      (System/exit 0))
-    (catch Throwable cause
-      (ex/print-throwable cause)
-      (flush)
-      (System/exit -1))))

backend/src/app/rpc/commands/auth.clj

@@ -273,7 +273,8 @@
                       (merge {:viewed-tutorial? false
                               :viewed-walkthrough? false
                               :nudge {:big 10 :small 1}
-                              :v2-info-shown true})
+                              :v2-info-shown true
+                              :release-notes-viewed (:main cf/version)})
                       (db/tjson))
 
         password  (or (:password params) "!")

backend/src/app/rpc/commands/binfile.clj

@@ -7,10 +7,13 @@
 (ns app.rpc.commands.binfile
   (:refer-clojure :exclude [assert])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
@@ -19,6 +22,7 @@
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.tasks.file-gc]
    [app.util.services :as sv]
@@ -46,9 +50,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v1/ids #{file-id})
-           (assoc ::bf.v1/embed-assets embed-assets)
-           (assoc ::bf.v1/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v1/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -61,9 +65,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v3/ids #{file-id})
-           (assoc ::bf.v3/embed-assets embed-assets)
-           (assoc ::bf.v3/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v3/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -90,41 +94,30 @@
 
 ;; --- Command: import-binfile
 
-(defn- import-binfile-v1
-  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
-  (let [cfg (-> cfg
-                (assoc ::bf.v1/project-id project-id)
-                (assoc ::bf.v1/profile-id profile-id)
-                (assoc ::bf.v1/name name)
-                (assoc ::bf.v1/input (:path file)))]
-
-    ;; NOTE: the importation process performs some operations that are
-    ;; not very friendly with virtual threads, and for avoid
-    ;; unexpected blocking of other concurrent operations we dispatch
-    ;; that operation to a dedicated executor.
-    (px/invoke! executor (partial bf.v1/import-files! cfg))))
-
-(defn- import-binfile-v3
-  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
-  (let [cfg (-> cfg
-                (assoc ::bf.v3/project-id project-id)
-                (assoc ::bf.v3/profile-id profile-id)
-                (assoc ::bf.v3/name name)
-                (assoc ::bf.v3/input (:path file)))]
-    ;; NOTE: the importation process performs some operations that are
-    ;; not very friendly with virtual threads, and for avoid
-    ;; unexpected blocking of other concurrent operations we dispatch
-    ;; that operation to a dedicated executor.
-    (px/invoke! executor (partial bf.v3/import-files! cfg))))
-
 (defn- import-binfile
-  [{:keys [::db/pool] :as cfg} {:keys [project-id version] :as params}]
-  (let [result (case (int version)
-                 1 (import-binfile-v1 cfg params)
-                 3 (import-binfile-v3 cfg params))]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
+  (let [team   (teams/get-team pool
+                               :profile-id profile-id
+                               :project-id project-id)
+        cfg    (-> cfg
+                   (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team))
+                   (assoc ::bfc/project-id project-id)
+                   (assoc ::bfc/profile-id profile-id)
+                   (assoc ::bfc/name name)
+                   (assoc ::bfc/input (:path file)))
+
+        ;; NOTE: the importation process performs some operations that are
+        ;; not very friendly with virtual threads, and for avoid
+        ;; unexpected blocking of other concurrent operations we dispatch
+        ;; that operation to a dedicated executor.
+        result (case (int version)
+                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
+                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
+
     (db/update! pool :project
                 {:modified-at (dt/now)}
                 {:id project-id})
+
     result))
 
 (def ^:private schema:import-binfile

backend/src/app/rpc/commands/comments.clj

@@ -6,45 +6,172 @@
 
 (ns app.rpc.commands.comments
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
    [app.common.schema :as sm]
+   [app.common.uri :as uri]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.email :as eml]
    [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.quotes :as quotes]
    [app.rpc.retry :as rtry]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [app.util.time :as dt]))
+   [app.util.time :as dt]
+   [clojure.set :as set]
+   [cuerdas.core :as str]))
 
 ;; --- GENERAL PURPOSE INTERNAL HELPERS
 
+(def r-mentions-split #"@\[[^\]]*\]\([^\)]*\)")
+(def r-mentions #"@\[([^\]]*)\]\(([^\)]*)\)")
+
+(def comment-max-length 750)
+
+(defn- format-comment
+  [{:keys [content]}]
+  (->> (d/interleave-all
+        (str/split content r-mentions-split)
+        (->> (re-seq r-mentions content)
+             (map (fn [[_ user _]] user))))
+       (str/join "")))
+
+(defn- format-comment-url
+  [thread file]
+  (str/ffmt "%/#/workspace?%"
+            (cf/get :public-uri)
+            (uri/map->query-string
+             {:file-id (:id file)
+              :page-id (:page-id file)
+              :team-id (:team-id file)
+              :comment-id (:id thread)})))
+
+(defn- format-comment-ref
+  [thread file]
+  (str/ffmt "#%, %, %"
+            (:seqn thread)
+            (:name file)
+            (:page-name file)))
+
+(defn- get-team-users
+  [conn team-id]
+  (->> (teams/get-users+props conn team-id)
+       (map profile/decode-row)
+       (d/index-by :id)))
+
+(defn- notification-email?
+  [profile-id owner-id props]
+  (if (= profile-id owner-id)
+    (not= :none (-> props :notifications :email-comments))
+    (= :all (-> props :notifications :email-comments))))
+
+(defn- mention-email?
+  [props]
+  (not= :none (-> props :notifications :email-comments)))
+
+(defn send-comment-emails!
+  [conn profile comment thread file]
+  (let [team-users        (get-team-users conn (:team-id file))
+        comment-reference (format-comment-ref thread file)
+        comment-content   (format-comment comment)
+        comment-url       (format-comment-url thread file)
+        profile-id        (get profile :id)
+
+        ;; Users mentioned in this comment
+        comment-mentions
+        (-> (:mentions comment)
+            (disj profile-id))
+
+        ;; Users mentioned in this thread
+        thread-mentions
+        (-> (:mentions thread)
+            ;; Remove the mentions in the comment because we're already sending a
+            ;; notification
+            (set/difference comment-mentions)
+            (disj profile-id))
+
+        ;; All users
+        notificate-users-ids
+        (-> (set (keys team-users))
+            (set/difference comment-mentions)
+            (set/difference thread-mentions)
+            (disj profile-id))]
+
+    (doseq [mention comment-mentions]
+      (let [{:keys [fullname email props]} (get team-users mention)]
+        (when (mention-email? props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-mention
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))
+
+    ;; Send to the thread users
+    (doseq [mention thread-mentions]
+      (let [{:keys [fullname email props]} (get team-users mention)]
+        (when (mention-email? props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-thread
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))
+
+    ;; Send to users with the "all" flag activated
+    (doseq [user-id notificate-users-ids]
+      (let [{:keys [id fullname email props]} (get team-users user-id)]
+        (when (notification-email? id (:owner-id thread) props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-notification
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))))
+
 (defn- decode-row
-  [{:keys [participants position] :as row}]
+  [{:keys [participants position mentions] :as row}]
   (cond-> row
     (db/pgpoint? position) (assoc :position (db/decode-pgpoint position))
-    (db/pgobject? participants) (assoc :participants (db/decode-transit-pgobject participants))))
+    (db/pgobject? participants) (assoc :participants (db/decode-transit-pgobject participants))
+    (db/pgarray? mentions) (assoc :mentions (db/decode-pgarray mentions #{}))))
 
 (def xf-decode-row
   (map decode-row))
 
-(def ^:privateqpage-name
+(def ^:private
   sql:get-file
-  "select f.id, f.modified_at, f.revn, f.features,
-          f.project_id, p.team_id, f.data
-     from file as f
-     join project as p on (p.id = f.project_id)
-    where f.id = ?
-      and f.deleted_at is null")
+  "SELECT f.id, f.modified_at, f.revn, f.features, f.name,
+          f.project_id, p.team_id, f.data,
+          f.data_ref_id, f.data_backend
+     FROM file as f
+    INNER JOIN project as p on (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
 
 (defn- get-file
   "A specialized version of get-file for comments module."
@@ -56,12 +183,16 @@
                 :hint "file not found"))
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-      (let [{:keys [data] :as file} (files/decode-row file)]
+      (let [file (->> file
+                      (files/decode-row)
+                      (feat.fdata/resolve-file-data cfg))
+            data (get file :data)]
         (-> file
             (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
             (assoc :page-id page-id)
             (dissoc :data))))))
 
+;; FIXME: rename
 (defn- get-comment-thread
   [conn thread-id & {:as opts}]
   (-> (db/get-by-id conn :comment-thread thread-id opts)
@@ -91,7 +222,7 @@
 
 (defn upsert-comment-thread-status!
   ([conn profile-id thread-id]
-   (upsert-comment-thread-status! conn profile-id thread-id (dt/now)))
+   (upsert-comment-thread-status! conn profile-id thread-id (dt/in-future "1s")))
   ([conn profile-id thread-id mod-at]
    (db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id mod-at mod-at])))
 
@@ -116,36 +247,42 @@
   {::doc/added "1.15"
    ::sm/params schema:get-comment-threads}
   [cfg {:keys [::rpc/profile-id file-id share-id] :as params}]
-
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
 (def ^:private sql:comment-threads
-  "select distinct on (ct.id)
+  "SELECT DISTINCT ON (ct.id)
           ct.*,
-          f.name as file_name,
-          f.project_id as project_id,
-          first_value(c.content) over w as content,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id) as count_comments,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id
-              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
-     from comment_thread as ct
-    inner join comment as c on (c.thread_id = ct.id)
-    inner join file as f on (f.id = ct.file_id)
-     left join comment_thread_status as cts
-            on (cts.thread_id = ct.id and
-                cts.profile_id = ?)
-    where ct.file_id = ?
-   window w as (partition by c.thread_id order by c.created_at asc)")
+          pf.fullname AS owner_fullname,
+          pf.email AS owner_email,
+          pf.photo_id AS owner_photo_id,
+          p.team_id AS team_id,
+          f.name AS file_name,
+          f.project_id AS project_id,
+          first_value(c.content) OVER w AS content,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id) AS count_comments,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id
+              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+     FROM comment_thread AS ct
+    INNER JOIN comment AS c ON (c.thread_id = ct.id)
+    INNER JOIN file AS f ON (f.id = ct.file_id)
+    INNER JOIN project AS p ON (p.id = f.project_id)
+     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
+
+(def ^:private sql:comment-threads-by-file-id
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
-  (->> (db/exec! conn [sql:comment-threads profile-id file-id])
+  (->> (db/exec! conn [sql:comment-threads-by-file-id profile-id file-id])
        (into [] xf-decode-row)))
 
 ;; --- COMMAND: Get Unread Comment Threads
@@ -161,41 +298,41 @@
   {::doc/added "1.15"
    ::sm/params schema:get-unread-comment-threads}
   [cfg {:keys [::rpc/profile-id team-id] :as params}]
-  (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (teams/check-read-permissions! conn profile-id team-id)
-                 (get-unread-comment-threads conn profile-id team-id))))
+  (db/run!
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (teams/check-read-permissions! conn profile-id team-id)
+     (get-unread-comment-threads conn profile-id team-id))))
 
-(def sql:comment-threads-by-team
-  "select distinct on (ct.id)
-          ct.*,
-          f.name as file_name,
-          f.project_id as project_id,
-          first_value(c.content) over w as content,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id) as count_comments,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id
-              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
-     from comment_thread as ct
-    inner join comment as c on (c.thread_id = ct.id)
-    inner join file as f on (f.id = ct.file_id)
-    inner join project as p on (p.id = f.project_id)
-     left join comment_thread_status as cts
-            on (cts.thread_id = ct.id and
-                cts.profile_id = ?)
-    where p.team_id = ?
-   window w as (partition by c.thread_id order by c.created_at asc)")
+(def sql:unread-all-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
 
-(def sql:unread-comment-threads-by-team
-  (str "with threads as (" sql:comment-threads-by-team ")"
-       "select * from threads where count_unread_comments > 0"))
+;; The partial configuration will retrieve only comments created by the user and
+;; threads that have a mention to the user.
+(def sql:unread-partial-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads
+         WHERE count_unread_comments > 0
+           AND team_id = ?
+           AND (owner_id = ? OR ? = ANY(mentions))"))
 
 (defn- get-unread-comment-threads
   [conn profile-id team-id]
-  (->> (db/exec! conn [sql:unread-comment-threads-by-team profile-id team-id])
-       (into [] xf-decode-row)))
+  (let [profile (-> (db/get conn :profile {:id profile-id})
+                    (profile/decode-row))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
+
+    (case notify
+      :all
+      (->> (db/exec! conn [sql:unread-all-comment-threads-by-team profile-id team-id])
+           (into [] xf-decode-row))
+
+      :partial
+      (->> (db/exec! conn [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+           (into [] xf-decode-row))
+
+      [])))
 
 ;; --- COMMAND: Get Single Comment Thread
 
@@ -212,9 +349,9 @@
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (let [sql (str "with threads as (" sql:comment-threads ")"
-                                "select * from threads where id = ?")]
-                   (-> (db/exec-one! conn [sql profile-id file-id id])
+                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
+                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                   (-> (db/exec-one! conn [sql profile-id id file-id])
                        (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
@@ -232,15 +369,25 @@
    ::sm/params schema:get-comments}
   [cfg {:keys [::rpc/profile-id thread-id share-id]}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (let [{:keys [file-id] :as thread} (get-comment-thread conn thread-id)]
+                 (let [{:keys [file-id]} (get-comment-thread conn thread-id)]
                    (files/check-comment-permissions! conn profile-id file-id share-id)
                    (get-comments conn thread-id)))))
 
+(def sql:get-comments
+  "SELECT c.*,
+          ct.file_id AS file_id,
+          pf.fullname AS owner_fullname,
+          pf.email AS owner_email,
+          pf.photo_id AS owner_photo_id
+     FROM comment AS c
+    INNER JOIN comment_thread AS ct ON (ct.id = c.thread_id)
+     LEFT JOIN profile AS pf ON (c.owner_id = pf.id)
+    WHERE c.thread_id = ?
+    ORDER BY c.created_at ASC")
+
 (defn- get-comments
   [conn thread-id]
-  (->> (db/query conn :comment
-                 {:thread-id thread-id}
-                 {:order-by [[:created-at :asc]]})
+  (->> (db/exec! conn [sql:get-comments thread-id])
        (into [] xf-decode-row)))
 
 ;; --- COMMAND: Get file comments users
@@ -297,10 +444,25 @@
   [:map {:title "create-comment-thread"}
    [:file-id ::sm/uuid]
    [:position ::gpt/point]
-   [:content [:string {:max 750}]]
+   [:content [:string {:max comment-max-length}]]
    [:page-id ::sm/uuid]
    [:frame-id ::sm/uuid]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
+
+(defn- update-thread-seqn
+  [conn file-id seqn]
+  (db/update! conn :file
+              {:comment-thread-seqn seqn}
+              {:id file-id}
+              {::db/return-keys false}))
+
+(defn add-owner
+  [thread-or-comment profile]
+  (-> thread-or-comment
+      (assoc :owner-fullname (:fullname profile))
+      (assoc :owner-email (:email profile))
+      (assoc :owner-photo-id (:photo-id profile))))
 
 (sv/defmethod ::create-comment-thread
   {::doc/added "1.15"
@@ -308,11 +470,10 @@
    ::rtry/enabled true
    ::rtry/when rtry/conflict-exception?
    ::sm/params schema:create-comment-thread}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
+  [cfg {:keys [::rpc/profile-id file-id page-id share-id] :as params}]
   (files/check-comment-permissions! cfg profile-id file-id share-id)
 
-  (let [{:keys [team-id project-id page-name]} (get-file cfg file-id page-id)]
-
+  (let [{:keys [team-id project-id] :as file} (get-file cfg file-id page-id)]
     (-> cfg
         (assoc ::quotes/profile-id profile-id)
         (assoc ::quotes/team-id team-id)
@@ -321,21 +482,14 @@
         (quotes/check! {::quotes/id ::quotes/comment-threads-per-file}
                        {::quotes/id ::quotes/comments-per-file}))
 
-    (let [params {:created-at request-at
-                  :profile-id profile-id
-                  :file-id file-id
-                  :page-id page-id
-                  :page-name page-name
-                  :position position
-                  :content content
-                  :frame-id frame-id}
+    (let [params (assoc params ::file file)
           thread (db/tx-run! cfg create-comment-thread params)]
 
       (vary-meta thread assoc ::audit/props thread))))
 
 (defn- create-comment-thread
   [{:keys [::db/conn] :as cfg}
-   {:keys [profile-id file-id page-id page-name created-at position content frame-id]}]
+   {:keys [::rpc/profile-id ::rpc/request-at ::file position content mentions frame-id] :as params}]
 
   (let [;; NOTE: we take the next seq number from a separate query
         ;; because we need to lock the file for avoid race conditions
@@ -346,39 +500,47 @@
         ;; different storage (example: redis) for alivate the update
         ;; pression on the file table
 
-        seqn      (get-next-seqn conn file-id)
+        profile   (profile/get-profile conn profile-id)
+        seqn      (get-next-seqn conn (:id file))
+
+        file-id   (get file :id)
         thread-id (uuid/next)
-        thread    (db/insert! conn :comment-thread
-                              {:id thread-id
-                               :file-id file-id
-                               :owner-id profile-id
-                               :participants (db/tjson #{profile-id})
-                               :page-name page-name
-                               :page-id page-id
-                               :created-at created-at
-                               :modified-at created-at
-                               :seqn seqn
-                               :position (db/pgpoint position)
-                               :frame-id frame-id})
-        comment   (db/insert! conn :comment
-                              {:id (uuid/next)
-                               :thread-id thread-id
-                               :owner-id profile-id
-                               :created-at created-at
-                               :modified-at created-at
-                               :content content})]
+
+        thread    (-> (db/insert! conn :comment-thread
+                                  {:id thread-id
+                                   :file-id file-id
+                                   :page-name (:page-name file)
+                                   :page-id (:page-id file)
+                                   :owner-id profile-id
+                                   :participants (db/tjson #{profile-id})
+                                   :created-at request-at
+                                   :modified-at request-at
+                                   :seqn seqn
+                                   :frame-id frame-id
+                                   :position (db/pgpoint position)
+                                   :mentions (db/encode-pgarray mentions conn "uuid")})
+                      (decode-row))
+        comment   (-> (db/insert! conn :comment
+                                  {:id (uuid/next)
+                                   :thread-id thread-id
+                                   :owner-id profile-id
+                                   :created-at request-at
+                                   :modified-at request-at
+                                   :mentions (db/encode-pgarray mentions conn "uuid")
+                                   :content content})
+                      (decode-row))]
 
     ;; Make the current thread as read.
-    (upsert-comment-thread-status! conn profile-id thread-id created-at)
+    (upsert-comment-thread-status! conn profile-id thread-id request-at)
 
     ;; Optimistic update of current seq number on file.
-    (db/update! conn :file
-                {:comment-thread-seqn seqn}
-                {:id file-id}
-                {::db/return-keys false})
+    (update-thread-seqn conn file-id seqn)
+
+    ;; Send mentions emails
+    (send-comment-emails! conn profile comment thread file)
 
     (-> thread
-        (select-keys [:id :file-id :page-id])
+        (add-owner profile)
         (assoc :comment-id (:id comment)))))
 
 ;; --- COMMAND: Update Comment Thread Status
@@ -391,12 +553,12 @@
 
 (sv/defmethod ::update-comment-thread-status
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-status}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (upsert-comment-thread-status! conn profile-id id)))))
+   ::sm/params schema:update-comment-thread-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (upsert-comment-thread-status! conn profile-id id)))
 
 ;; --- COMMAND: Update Comment Thread
 
@@ -409,76 +571,85 @@
 
 (sv/defmethod ::update-comment-thread
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread}
-  [cfg {:keys [::rpc/profile-id id is-resolved share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:is-resolved is-resolved}
-                                  {:id id})
-                      nil))))
-
+   ::sm/params schema:update-comment-thread
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id is-resolved share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:is-resolved is-resolved}
+                {:id id})
+    nil))
 
 ;; --- COMMAND: Add Comment
 
-(declare ^:private get-comment-thread)
-
 (def ^:private
   schema:create-comment
   [:map {:title "create-comment"}
    [:thread-id ::sm/uuid]
-   [:content [:string {:max 250}]]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:content [:string {:max comment-max-length}]]
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
 
 (sv/defmethod ::create-comment
   {::doc/added "1.15"
    ::webhooks/event? true
-   ::sm/params schema:create-comment}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)
-                      {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
+   ::sm/params schema:create-comment
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content mentions]}]
+  (let [{:keys [file-id page-id] :as thread}
+        (get-comment-thread conn thread-id ::sql/for-update true)
 
-                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                  (quotes/check! cfg {::quotes/id ::quotes/comments-per-file
-                                      ::quotes/profile-id profile-id
-                                      ::quotes/team-id team-id
-                                      ::quotes/project-id project-id
-                                      ::quotes/file-id file-id})
+        {:keys [team-id project-id] :as file}
+        (get-file cfg file-id page-id)]
 
-                  ;; Update the page-name cached attribute on comment thread table.
-                  (when (not= page-name (:page-name thread))
-                    (db/update! conn :comment-thread
-                                {:page-name page-name}
-                                {:id thread-id}))
+    (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (let [comment (db/insert! conn :comment
-                                            {:id (uuid/next)
-                                             :created-at request-at
-                                             :modified-at request-at
-                                             :thread-id thread-id
-                                             :owner-id profile-id
-                                             :content content})
-                        props    {:file-id file-id
-                                  :share-id nil}]
+    (quotes/check! cfg {::quotes/id ::quotes/comments-per-file
+                        ::quotes/profile-id profile-id
+                        ::quotes/team-id team-id
+                        ::quotes/project-id project-id
+                        ::quotes/file-id file-id})
 
-                    ;; Update thread modified-at attribute and assoc the current
-                    ;; profile to the participant set.
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :participants (-> (:participants thread #{})
-                                                   (conj profile-id)
-                                                   (db/tjson))}
-                                {:id thread-id})
+    (let [profile  (profile/get-profile conn profile-id)
+          mentions (into #{} mentions)
+          params   {:id (uuid/next)
+                    :created-at request-at
+                    :modified-at request-at
+                    :thread-id thread-id
+                    :owner-id profile-id
+                    :content content
+                    :mentions (db/encode-pgarray mentions conn "uuid")}
 
-                    ;; Update the current profile status in relation to the
-                    ;; current thread.
-                    (upsert-comment-thread-status! conn profile-id thread-id request-at)
+          comment (-> (db/insert! conn :comment params)
+                      (decode-row)
+                      (assoc :file-id file-id)
+                      (add-owner profile))]
 
-                    (vary-meta comment assoc ::audit/props props))))))
+      ;; Update thread modified-at attribute and assoc the current
+      ;; profile to the participant set.
+      (let [mentions     (into (:mentions thread) mentions)
+            participants (-> (:participants thread #{})
+                             (conj profile-id))
+            params       {:modified-at request-at
+                          :participants (db/tjson participants)
+                          :mentions (db/encode-pgarray mentions conn "uuid")}
 
+            ;; Update the page-name cached attribute on comment thread table.
+            params       (cond-> params
+                           (not= (:page-name file) (:page-name thread))
+                           (assoc :page-name (:page-name file)))]
+
+        (db/update! conn :comment-thread params
+                    {:id thread-id}
+                    {::db/return-keys false}))
+
+      ;; Update the current profile status in relation to the current thread
+      (upsert-comment-thread-status! conn profile-id thread-id)
+
+      (send-comment-emails! conn profile comment thread file)
+
+      (vary-meta comment assoc ::audit/props comment))))
 
 ;; --- COMMAND: Update Comment
 
@@ -486,36 +657,47 @@
   schema:update-comment
   [:map {:title "update-comment"}
    [:id ::sm/uuid]
-   [:content [:string {:max 250}]]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:content [:string {:max comment-max-length}]]
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
+
+;; TODO: Check if there are new mentions, if there are send the new emails.
 
 (sv/defmethod ::update-comment
   {::doc/added "1.15"
-   ::sm/params schema:update-comment}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::sql/for-update true)
-                      {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)]
+   ::sm/params schema:update-comment
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content mentions]}]
+  (let [{:keys [thread-id owner-id] :as comment}
+        (get-comment conn id ::sql/for-update true)
 
-                  (files/check-comment-permissions! conn profile-id file-id share-id)
+        {:keys [file-id page-id] :as thread}
+        (get-comment-thread conn thread-id ::sql/for-update true)]
 
-                  ;; Don't allow edit comments to not owners
-                  (when-not (= owner-id profile-id)
-                    (ex/raise :type :validation
-                              :code :not-allowed))
+    (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (let [{:keys [page-name]} (get-file cfg file-id page-id)]
-                    (db/update! conn :comment
-                                {:content content
-                                 :modified-at request-at}
-                                {:id id})
+    ;; Don't allow edit comments to not owners
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
 
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :page-name page-name}
-                                {:id thread-id})
-                    nil)))))
+    (let [{:keys [page-name]} (get-file cfg file-id page-id)]
+      (db/update! conn :comment
+                  {:content content
+                   :modified-at request-at
+                   :mentions (db/encode-pgarray mentions conn "uuid")}
+                  {:id id})
+
+      (db/update! conn :comment-thread
+                  {:modified-at request-at
+                   :page-name page-name
+                   :mentions
+                   (-> (:mentions thread)
+                       (into mentions)
+                       (db/encode-pgarray conn "uuid"))}
+                  {:id thread-id}
+                  {::db/return-keys false})
+      nil)))
 
 ;; --- COMMAND: Delete Comment Thread
 
@@ -527,17 +709,18 @@
 
 (sv/defmethod ::delete-comment-thread
   {::doc/added "1.15"
-   ::sm/params schema:delete-comment-thread}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (when-not (= owner-id profile-id)
-                        (ex/raise :type :validation
-                                  :code :not-allowed))
+   ::sm/params schema:delete-comment-thread
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
 
-                      (db/delete! conn :comment-thread {:id id})
-                      nil))))
+    (db/delete! conn :comment-thread {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Delete comment
 
@@ -549,17 +732,23 @@
 
 (sv/defmethod ::delete-comment
   {::doc/added "1.15"
-   ::sm/params schema:delete-comment}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [owner-id thread-id] :as comment} (get-comment conn id ::sql/for-update true)
-                          {:keys [file-id] :as thread} (get-comment-thread conn thread-id)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (when-not (= owner-id profile-id)
-                        (ex/raise :type :validation
-                                  :code :not-allowed))
-                      (db/delete! conn :comment {:id id})
-                      nil))))
+   ::sm/params schema:delete-comment
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [owner-id thread-id] :as comment}
+        (get-comment conn id ::sql/for-update true)
+
+        {:keys [file-id]}
+        (get-comment-thread conn thread-id)]
+
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
+
+    (db/delete! conn :comment {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Update comment thread position
 
@@ -573,17 +762,18 @@
 
 (sv/defmethod ::update-comment-thread-position
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-position}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:modified-at request-at
-                                   :position (db/pgpoint position)
-                                   :frame-id frame-id}
-                                  {:id (:id thread)})
-                      nil))))
+   ::sm/params schema:update-comment-thread-position
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:modified-at request-at
+                 :position (db/pgpoint position)
+                 :frame-id frame-id}
+                {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Update comment frame
 
@@ -596,13 +786,14 @@
 
 (sv/defmethod ::update-comment-thread-frame
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-frame}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:modified-at request-at
-                                   :frame-id frame-id}
-                                  {:id id})
-                      nil))))
+   ::sm/params schema:update-comment-thread-frame
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:modified-at request-at
+                 :frame-id frame-id}
+                {:id id}
+                {::db/return-keys false})
+    nil))

backend/src/app/rpc/commands/feedback.clj

@@ -17,7 +17,7 @@
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]))
 
-(declare ^:private send-feedback!)
+(declare ^:private send-user-feedback!)
 
 (def ^:private schema:send-user-feedback
   [:map {:title "send-user-feedback"}
@@ -34,14 +34,16 @@
               :hint "feedback not enabled"))
 
   (let [profile (profile/get-profile pool profile-id)]
-    (send-feedback! pool profile params)
+    (send-user-feedback! pool profile params)
     nil))
 
-(defn- send-feedback!
+(defn- send-user-feedback!
   [pool profile params]
-  (let [dest (cf/get :feedback-destination)]
+  (let [dest (or (cf/get :user-feedback-destination)
+                 ;; LEGACY
+                 (cf/get :feedback-destination))]
     (eml/send! {::eml/conn pool
-                ::eml/factory eml/feedback
+                ::eml/factory eml/user-feedback
                 :from     dest
                 :to       dest
                 :profile  profile

backend/src/app/rpc/commands/files.clj

@@ -22,6 +22,7 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -243,7 +244,8 @@
       (when (contains? (:features file) "fdata/pointer-map")
         (feat.fdata/persist-pointers! cfg id))
 
-      file)))
+      (feat.fmigr/upsert-migrations! conn file)
+      (feat.fmigr/resolve-applied-migrations cfg file))))
 
 (defn get-file
   [{:keys [::db/conn ::wrk/executor] :as cfg} id
@@ -264,6 +266,7 @@
                             {::db/check-deleted (not include-deleted?)
                              ::db/remove-deleted (not include-deleted?)
                              ::sql/for-update lock-for-update?})
+                    (feat.fmigr/resolve-applied-migrations cfg)
                     (feat.fdata/resolve-file-data cfg))
 
         ;; NOTE: we perform the file decoding in a separate thread
@@ -320,6 +323,7 @@
 
           file (-> (get-file cfg id :project-id project-id)
                    (assoc :permissions perms)
+                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
@@ -381,8 +385,10 @@
           f.revn,
           f.vern,
           f.is_shared,
-          ft.media_id AS thumbnail_id
+          ft.media_id AS thumbnail_id,
+          p.team_id
      from file as f
+     inner join project as p on (p.id = f.project_id)
      left join file_thumbnail as ft on (ft.file_id = f.id
                                         and ft.revn = f.revn
                                         and ft.deleted_at is null)
@@ -536,7 +542,8 @@
           f.modified_at,
           f.name,
           f.is_shared,
-          ft.media_id
+          ft.media_id,
+          p.team_id
      from file as f
     inner join project as p on (p.id = f.project_id)
      left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn and ft.deleted_at is null)
@@ -546,7 +553,6 @@
       and p.team_id = ?
     order by f.modified_at desc")
 
-
 (defn- get-library-summary
   [cfg {:keys [id data] :as file}]
   (letfn [(assets-sample [assets limit]
@@ -575,7 +581,7 @@
                          (if-let [media-id (:media-id row)]
                            (-> row
                                (dissoc :media-id)
-                               (assoc :thumbnail-uri (resolve-public-uri media-id)))
+                               (assoc :thumbnail-id media-id))
                            (dissoc row :media-id))))
                   (map #(assoc % :library-summary (get-library-summary cfg %)))
                   (map #(dissoc % :data))))))
@@ -608,14 +614,17 @@
    SELECT l.id,
           l.features,
           l.project_id,
+          p.team_id,
           l.created_at,
           l.modified_at,
           l.deleted_at,
           l.name,
           l.revn,
           l.vern,
-          l.synced_at
+          l.synced_at,
+          l.is_shared
      FROM libs AS l
+    INNER JOIN project AS p ON (p.id = l.project_id)
     WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
 
 (defn get-file-libraries
@@ -682,7 +691,8 @@
             f.name,
             f.is_shared,
             ft.media_id AS thumbnail_id,
-            row_number() over w as row_num
+            row_number() over w as row_num,
+            p.team_id
        from file as f
       inner join project as p on (p.id = f.project_id)
        left join file_thumbnail as ft on (ft.file_id = f.id
@@ -698,11 +708,7 @@
 
 (defn get-team-recent-files
   [conn team-id]
-  (->> (db/exec! conn [sql:team-recent-files team-id])
-       (mapv (fn [row]
-               (if-let [media-id (:thumbnail-id row)]
-                 (assoc row :thumbnail-uri (resolve-public-uri media-id))
-                 (dissoc row :media-id))))))
+  (db/exec! conn [sql:team-recent-files team-id]))
 
 (def ^:private schema:get-team-recent-files
   [:map {:title "get-team-recent-files"}

backend/src/app/rpc/commands/files_create.clj

@@ -6,13 +6,13 @@
 
 (ns app.rpc.commands.files-create
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
    [app.common.types.file :as ctf]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -21,7 +21,6 @@
    [app.rpc.doc :as-alias doc]
    [app.rpc.permissions :as perms]
    [app.rpc.quotes :as quotes]
-   [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -48,34 +47,19 @@
 
   (binding [pmap/*tracked* (pmap/create-tracked)
             cfeat/*current* features]
-    (let [file     (ctf/make-file {:id id
-                                   :project-id project-id
-                                   :name name
-                                   :revn revn
-                                   :is-shared is-shared
-                                   :features features
-                                   :ignore-sync-until ignore-sync-until
-                                   :modified-at modified-at
-                                   :deleted-at deleted-at
-                                   :create-page create-page
-                                   :page-id page-id})
-
-          file     (if (contains? features "fdata/objects-map")
-                     (feat.fdata/enable-objects-map file)
-                     file)
-
-          file     (if (contains? features "fdata/pointer-map")
-                     (feat.fdata/enable-pointer-map file)
-                     file)]
-
-      (db/insert! conn :file
-                  (-> file
-                      (update :data blob/encode)
-                      (update :features db/encode-pgarray conn "text"))
-                  {::db/return-keys false})
-
-      (when (contains? features "fdata/pointer-map")
-        (feat.fdata/persist-pointers! cfg (:id file)))
+    (let [file (ctf/make-file {:id id
+                               :project-id project-id
+                               :name name
+                               :revn revn
+                               :is-shared is-shared
+                               :features features
+                               :ignore-sync-until ignore-sync-until
+                               :modified-at modified-at
+                               :deleted-at deleted-at
+                               :create-page create-page
+                               :page-id page-id})
+          file (-> (bfc/insert-file! cfg file)
+                   (bfc/decode-row))]
 
       (->> (assoc params :file-id (:id file) :role :owner)
            (create-file-role! conn))
@@ -118,11 +102,12 @@
         ;; feature on frontend and make it permanent on file
         features (-> (:features params #{})
                      (set/intersection cfeat/no-migration-features)
+                     (set/difference cfeat/frontend-only-features)
                      (set/union features))
 
         params   (-> params
                      (assoc :profile-id profile-id)
-                     (assoc :features features))]
+                     (assoc :features (set/difference features cfeat/frontend-only-features)))]
 
     (quotes/check! cfg {::quotes/id ::quotes/files-per-project
                         ::quotes/team-id team-id

backend/src/app/rpc/commands/files_snapshot.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files-snapshot
   (:require
+   [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
@@ -22,19 +23,24 @@
    [app.rpc.quotes :as quotes]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
 
 (def sql:get-file-snapshots
-  "SELECT id, label, revn, created_at, created_by, profile_id
-     FROM file_change
-    WHERE file_id = ?
-      AND data IS NOT NULL
-      AND (deleted_at IS NULL OR deleted_at > now())
-    ORDER BY created_at DESC
-    LIMIT 20")
+  "WITH changes AS (
+      SELECT id, label, revn, created_at, created_by, profile_id
+        FROM file_change
+       WHERE file_id = ?
+         AND data IS NOT NULL
+         AND (deleted_at IS NULL OR deleted_at > now())
+   ), versions AS (
+      (SELECT * FROM changes WHERE created_by = 'system' LIMIT 1000)
+      UNION ALL
+      (SELECT * FROM changes WHERE created_by != 'system' LIMIT 1000)
+   )
+   SELECT * FROM versions
+    ORDER BY created_at DESC;")
 
 (defn get-file-snapshots
   [conn file-id]
@@ -52,26 +58,6 @@
                  (files/check-read-permissions! conn profile-id file-id)
                  (get-file-snapshots conn file-id))))
 
-(def ^:private sql:get-file
-  "SELECT f.*,
-          p.id AS project_id,
-          p.team_id AS team_id
-     FROM file AS f
-    INNER JOIN project AS p ON (p.id = f.project_id)
-    WHERE f.id = ?")
-
-(defn- get-file
-  [cfg file-id]
-  (let [file (->> (db/exec-one! cfg [sql:get-file file-id])
-                  (feat.fdata/resolve-file-data cfg))]
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-      (-> file
-          (update :data blob/decode)
-          (update :data feat.fdata/process-pointers deref)
-          (update :data feat.fdata/process-objects (partial into {}))
-          (update :data assoc ::id file-id)
-          (update :data blob/encode)))))
-
 (defn- generate-snapshot-label
   []
   (let [ts (-> (dt/now)
@@ -81,49 +67,53 @@
     (str "snapshot-" ts)))
 
 (defn create-file-snapshot!
-  [cfg profile-id file-id label]
-  (let [file (get-file cfg file-id)
+  [cfg file & {:keys [label created-by deleted-at profile-id]
+               :or {deleted-at :default
+                    created-by :system}}]
+
+  (assert (#{:system :user :admin} created-by)
+          "expected valid keyword for created-by")
+
+  (let [conn
+        (db/get-connection cfg)
 
-        ;; NOTE: final user never can provide label as `:system`
-        ;; keyword because the validator implies label always as
-        ;; string; keyword is used for signal a special case
         created-by
-        (if (= label :system)
-          "system"
-          "user")
+        (name created-by)
 
         deleted-at
-        (if (= label :system)
+        (cond
+          (= deleted-at :default)
           (dt/plus (dt/now) (cf/get-deletion-delay))
+
+          (dt/instant? deleted-at)
+          deleted-at
+
+          :else
           nil)
 
         label
-        (if (= label :system)
-          (str "internal/snapshot/" (:revn file))
-          (or label (generate-snapshot-label)))
+        (or label (generate-snapshot-label))
 
         snapshot-id
-        (uuid/next)]
+        (uuid/next)
 
-    (-> cfg
-        (assoc ::quotes/profile-id profile-id)
-        (assoc ::quotes/project-id (:project-id file))
-        (assoc ::quotes/team-id (:team-id file))
-        (assoc ::quotes/file-id (:id file))
-        (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
-                       {::quotes/id ::quotes/snapshots-per-team}))
+        data
+        (blob/encode (:data file))
+
+        features
+        (db/encode-pgarray (:features file) conn "text")]
 
     (l/debug :hint "creating file snapshot"
-             :file-id (str file-id)
+             :file-id (str (:id file))
              :id (str snapshot-id)
              :label label)
 
     (db/insert! cfg :file-change
                 {:id snapshot-id
                  :revn (:revn file)
-                 :data (:data file)
+                 :data data
                  :version (:version file)
-                 :features (:features file)
+                 :features features
                  :profile-id profile-id
                  :file-id (:id file)
                  :label label
@@ -140,12 +130,25 @@
 
 (sv/defmethod ::create-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:create-file-snapshot}
-  [cfg {:keys [::rpc/profile-id file-id label]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (files/check-edition-permissions! conn profile-id file-id)
-                (create-file-snapshot! cfg profile-id file-id label))))
+   ::sm/params schema:create-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id label]}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (let [file    (bfc/get-file cfg file-id)
+        project (db/get-by-id cfg :project (:project-id file))]
+
+    (-> cfg
+        (assoc ::quotes/profile-id profile-id)
+        (assoc ::quotes/project-id (:project-id file))
+        (assoc ::quotes/team-id (:team-id project))
+        (assoc ::quotes/file-id (:id file))
+        (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
+                       {::quotes/id ::quotes/snapshots-per-team}))
+
+    (create-file-snapshot! cfg file
+                           {:label label
+                            :profile-id profile-id
+                            :created-by :user})))
 
 (defn restore-file-snapshot!
   [{:keys [::db/conn ::mbus/msgbus] :as cfg} file-id snapshot-id]
@@ -231,8 +234,11 @@
   (db/tx-run! cfg
               (fn [{:keys [::db/conn] :as cfg}]
                 (files/check-edition-permissions! conn profile-id file-id)
-                (create-file-snapshot! cfg profile-id file-id :system)
-                (restore-file-snapshot! cfg file-id id))))
+                (let [file (bfc/get-file cfg file-id)]
+                  (create-file-snapshot! cfg file
+                                         {:profile-id profile-id
+                                          :created-by :system})
+                  (restore-file-snapshot! cfg file-id id)))))
 
 (def ^:private schema:update-file-snapshot
   [:map {:title "update-file-snapshot"}

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -50,8 +50,7 @@
              " where file_id=? and tag=? and deleted_at is null")
         res (db/exec! conn [sql file-id tag])]
     (->> res
-         (d/index-by :object-id (fn [row]
-                                  (files/resolve-public-uri (:media-id row))))
+         (d/index-by :object-id :media-id)
          (d/without-nils))))
 
 (defn- get-object-thumbnails
@@ -62,8 +61,7 @@
               " where file_id=? and deleted_at is null")
          res (db/exec! conn [sql file-id])]
      (->> res
-          (d/index-by :object-id (fn [row]
-                                   (files/resolve-public-uri (:media-id row))))
+          (d/index-by :object-id :media-id)
           (d/without-nils))))
 
   ([conn file-id object-ids]
@@ -75,8 +73,7 @@
          res (db/exec! conn [sql file-id ids])]
 
      (->> res
-          (d/index-by :object-id (fn [row]
-                                   (files/resolve-public-uri (:media-id row))))
+          (d/index-by :object-id :media-id)
           (d/without-nils)))))
 
 (sv/defmethod ::get-file-object-thumbnails
@@ -127,8 +124,11 @@
               (if-let [frame  (-> frames first)]
                 (let [frame-id  (:id frame)
                       object-id (thc/fmt-object-id (:id file) page-id frame-id "frame")
-                      frame     (if-let [thumb (get thumbnails object-id)]
-                                  (assoc frame :thumbnail thumb :shapes [])
+
+                      frame     (if-let [media-id (get thumbnails object-id)]
+                                  (-> frame
+                                      (assoc :thumbnail-id media-id)
+                                      (assoc :shapes []))
                                   (dissoc frame :thumbnail))
 
                       children-ids
@@ -402,7 +402,10 @@
 
   [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (files/check-edition-permissions! conn profile-id file-id)
+                    ;; TODO For now we check read permissions instead of write,
+                    ;; to allow viewer users to update thumbnails. We might
+                    ;; review this approach on the future.
+                    (files/check-read-permissions! conn profile-id file-id)
                     (when-not (db/read-only? conn)
                       (let [media (create-file-thumbnail! cfg params)]
                         {:uri (files/resolve-public-uri (:id media))

backend/src/app/rpc/commands/files_update.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files-update
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
@@ -18,6 +19,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
@@ -118,6 +120,7 @@
 (sv/defmethod ::update-file
   {::climit/id [[:update-file/by-profile ::rpc/profile-id]
                 [:update-file/global]]
+
    ::webhooks/event? true
    ::webhooks/batch-timeout (dt/duration "2m")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
@@ -147,7 +150,7 @@
 
                           params   (-> params
                                        (assoc :profile-id profile-id)
-                                       (assoc :features features)
+                                       (assoc :features (set/difference features cfeat/frontend-only-features))
                                        (assoc :team team)
                                        (assoc :file file)
                                        (assoc :changes changes))
@@ -202,45 +205,27 @@
    {:keys [profile-id file features changes session-id skip-validate] :as params}]
 
   (let [;; Retrieve the file data
-        file (feat.fdata/resolve-file-data cfg file)
+        file  (feat.fmigr/resolve-applied-migrations cfg file)
+        file  (feat.fdata/resolve-file-data cfg file)
+        file  (assoc file :features
+                     (-> features
+                         (set/difference cfeat/frontend-only-features)
+                         (set/union (:features file))))]
 
-        file (assoc file :features
-                    (-> features
-                        (set/difference cfeat/frontend-only-features)
-                        (set/union (:features file))))
+    ;; We create a new lexycal scope for clearly delimit the result of
+    ;; executing this update file operation and all its side effects
+    (let [file (px/invoke! executor
+                           (fn []
+                             ;; Process the file data on separated thread for avoid to do
+                             ;; the CPU intensive operation on vthread.
+                             (binding [cfeat/*current*  features
+                                       cfeat/*previous* (:features file)]
+                               (update-file-data! cfg file
+                                                  process-changes-and-validate
+                                                  changes skip-validate))))]
 
-        ;; Process the file data on separated thread for avoid to do
-        ;; the CPU intensive operation on vthread.
-        file  (px/invoke! executor
-                          (fn []
-                            (binding [cfeat/*current*  features
-                                      cfeat/*previous* (:features file)]
-                              (update-file-data! cfg file
-                                                 process-changes-and-validate
-                                                 changes skip-validate))))]
-
-    (when (feat.fdata/offloaded? file)
-      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-        (some->> (:data-ref-id file) (sto/touch-object! storage))))
-
-    (-> cfg
-        (assoc ::wrk/task :file-xlog-gc)
-        (assoc ::wrk/label (str "xlog:" (:id file)))
-        (assoc ::wrk/params {:file-id (:id file)})
-        (assoc ::wrk/delay (dt/duration "5m"))
-        (assoc ::wrk/dedupe true)
-        (assoc ::wrk/priority 1)
-        (wrk/submit!))
-
-    (persist-file! cfg file)
-
-    (let [params   (assoc params :file file)
-          response {:revn (:revn file)
-                    :lagged (get-lagged-changes conn params)}
-          features (db/create-array conn "text" (:features file))
-          deleted-at (if (::snapshot-data file)
-                       (dt/plus timestamp (cf/get-deletion-delay))
-                       (dt/plus timestamp (dt/duration {:hours 1})))]
+      (feat.fmigr/upsert-migrations! conn file)
+      (persist-file! cfg file)
 
       ;; Insert change (xlog) with deleted_at in a future data for
       ;; make them automatically eleggible for GC once they expires
@@ -250,19 +235,27 @@
                    :profile-id profile-id
                    :created-at timestamp
                    :updated-at timestamp
-                   :deleted-at deleted-at
+                   :deleted-at (if (::snapshot-data file)
+                                 (dt/plus timestamp (cf/get-deletion-delay))
+                                 (dt/plus timestamp (dt/duration {:hours 1})))
                    :file-id (:id file)
                    :revn (:revn file)
                    :version (:version file)
-                   :features features
+                   :features (:features file)
                    :label (::snapshot-label file)
                    :data (::snapshot-data file)
                    :changes (blob/encode changes)}
                   {::db/return-keys false})
 
       ;; Send asynchronous notifications
-      (send-notifications! cfg params)
+      (send-notifications! cfg params file))
 
+    (when (feat.fdata/offloaded? file)
+      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
+        (some->> (:data-ref-id file) (sto/touch-object! storage))))
+
+    (let [response {:revn (:revn file)
+                    :lagged (get-lagged-changes conn params)}]
       (vary-meta response assoc ::audit/replace-props
                  {:id         (:id file)
                   :name       (:name file)
@@ -272,9 +265,10 @@
 
 (defn update-file!
   "A public api that allows apply a transformation to a file with all context setup."
-  [cfg file-id update-fn & args]
+  [{:keys [::db/conn] :as cfg} file-id update-fn & args]
   (let [file (get-file cfg file-id)
         file (apply update-file-data! cfg file update-fn args)]
+    (feat.fmigr/upsert-migrations! conn file)
     (persist-file! cfg file)))
 
 (def ^:private sql:get-file
@@ -302,8 +296,7 @@
 
   It also updates the project modified-at attr."
   [{:keys [::db/conn ::timestamp]} file]
-  (let [features (db/create-array conn "text" (:features file))
-        ;; The timestamp can be nil because this function is also
+  (let [;; The timestamp can be nil because this function is also
         ;; intended to be used outside of this module
         modified-at (or timestamp (dt/now))]
 
@@ -316,7 +309,7 @@
                 {:revn (:revn file)
                  :data (:data file)
                  :version (:version file)
-                 :features features
+                 :features (:features file)
                  :data-backend nil
                  :data-ref-id nil
                  :modified-at modified-at
@@ -375,38 +368,16 @@
                    (-> file
                        (assoc ::snapshot-data snapshot)
                        (assoc ::snapshot-label label)))
-                 file)
+                 file)]
 
-          file (cond-> file
-                 (contains? cfeat/*current* "fdata/objects-map")
-                 (feat.fdata/enable-objects-map)
-
-                 (contains? cfeat/*current* "fdata/pointer-map")
-                 (feat.fdata/enable-pointer-map)
-
-                 :always
-                 (update :data blob/encode))]
-
-      (feat.fdata/persist-pointers! cfg id)
-
-      file)))
+      (bfc/encode-file cfg file))))
 
 (defn- get-file-libraries
   "A helper for preload file libraries, mainly used for perform file
   semantical and structural validation"
   [{:keys [::db/conn] :as cfg} file]
   (->> (files/get-file-libraries conn (:id file))
-       (into [file] (map (fn [{:keys [id]}]
-                           (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-                                     pmap/*tracked* nil]
-                             ;; We do not resolve the objects maps here
-                             ;; because there is a lower probability that all
-                             ;; shapes needed to be loded into memory, so we
-                             ;; leeave it on lazy status
-                             (-> (files/get-file cfg id :migrate? false)
-                                 (update :data feat.fdata/process-pointers deref) ; ensure all pointers resolved
-                                 (update :data feat.fdata/process-objects (partial into {}))
-                                 (fmg/migrate-file))))))
+       (into [file] (map #(bfc/get-file cfg (:id %))))
        (d/index-by :id)))
 
 (defn- soft-validate-file-schema!
@@ -424,19 +395,37 @@
       (l/error :hint "file validation error"
                :cause cause))))
 
+
 (defn- process-changes-and-validate
   [cfg file changes skip-validate]
   (let [;; WARNING: this ruins performance; maybe we need to find
         ;; some other way to do general validation
-        libs (when (and (or (contains? cf/flags :file-validation)
-                            (contains? cf/flags :soft-file-validation))
-                        (not skip-validate))
-               (get-file-libraries cfg file))
+        libs
+        (when (and (or (contains? cf/flags :file-validation)
+                       (contains? cf/flags :soft-file-validation))
+                   (not skip-validate))
+          (get-file-libraries cfg file))
 
-        file (-> (files/check-version! file)
-                 (update :revn inc)
-                 (update :data cpc/process-changes changes)
-                 (update :data d/without-nils))]
+
+        ;; The main purpose of this atom is provide a contextual state
+        ;; for the changes subsystem where optionally some hints can
+        ;; be provided for the changes processing. Right now we are
+        ;; using it for notify about the existence of media refs when
+        ;; a new shape is added.
+        state
+        (atom {})
+
+        file
+        (binding [cpc/*state* state]
+          (-> (files/check-version! file)
+              (update :revn inc)
+              (update :data cpc/process-changes changes)
+              (update :data d/without-nils)))
+
+        file
+        (if-let [media-refs (-> @state :media-refs not-empty)]
+          (bfc/update-media-references! cfg file media-refs)
+          file)]
 
     (binding [pmap/*tracked* nil]
       (when (contains? cf/flags :soft-file-validation)
@@ -483,7 +472,7 @@
        (vec)))
 
 (defn- send-notifications!
-  [cfg {:keys [file team changes session-id] :as params}]
+  [cfg {:keys [team changes session-id] :as params} file]
   (let [lchanges (filter library-change? changes)
         msgbus   (::mbus/msgbus cfg)]
 

backend/src/app/rpc/commands/management.clj

@@ -9,6 +9,7 @@
   (:require
    [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
@@ -25,6 +26,7 @@
    [app.rpc.doc :as-alias doc]
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
+   [app.storage.tmp :as tmp]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
@@ -54,8 +56,8 @@
     (vswap! bfc/*state* update :index bfc/update-index fmeds :id)
 
     ;; Process and persist file
-    (let [file (->> (bfc/process-file file)
-                    (bfc/persist-file! cfg))]
+    (let [file (bfc/process-file file)]
+      (bfc/insert-file! cfg file ::db/return-keys false)
 
       ;; The file profile creation is optional, so when no profile is
       ;; present (when this function is called from profile less
@@ -67,7 +69,7 @@
                      :is-owner true
                      :is-admin true
                      :can-edit true}
-                    {::db/return-keys? false}))
+                    {::db/return-keys false}))
 
       (doseq [params (sequence (comp
                                 (map #(bfc/remap-id % :file-id))
@@ -400,11 +402,24 @@
                     ;; that are not very friendly with virtual threads, and for
                     ;; avoid unexpected blocking of other concurrent operations
                     ;; we dispatch that operation to a dedicated executor.
-                    (let [cfg    (-> cfg
-                                     (assoc ::bf.v1/project-id project-id)
-                                     (assoc ::bf.v1/profile-id profile-id)
-                                     (assoc ::bf.v1/input template))
-                          result (px/invoke! executor (partial bf.v1/import-files! cfg))]
+                    (let [template (tmp/tempfile-from template
+                                                      :prefix "penpot.template."
+                                                      :suffix ""
+                                                      :min-age "30m")
+
+                          format   (bfc/parse-file-format template)
+                          team     (teams/get-team conn
+                                                   :profile-id profile-id
+                                                   :project-id project-id)
+                          cfg      (-> cfg
+                                       (assoc ::bfc/project-id project-id)
+                                       (assoc ::bfc/profile-id profile-id)
+                                       (assoc ::bfc/input template)
+                                       (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
+
+                          result   (if (= format :binfile-v3)
+                                     (px/invoke! executor (partial bf.v3/import-files! cfg))
+                                     (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
                       (db/update! conn :project
                                   {:modified-at (dt/now)}

backend/src/app/rpc/commands/media.clj

@@ -60,15 +60,25 @@
   (media/validate-media-type! content)
   (media/validate-media-size! content)
 
-  (db/run! cfg (fn [cfg]
-                 (let [object (create-file-media-object cfg params)
-                       props  {:name (:name params)
-                               :file-id file-id
-                               :is-local (:is-local params)
-                               :size (:size content)
-                               :mtype (:mtype content)}]
-                   (with-meta object
-                     {::audit/replace-props props})))))
+  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                 ;; We get the minimal file for proper checking if
+                 ;; file is not already deleted
+                 (let [_     (files/get-minimal-file conn file-id)
+                       mobj  (create-file-media-object cfg params)]
+
+                   (db/update! conn :file
+                               {:modified-at (dt/now)
+                                :has-media-trimmed false}
+                               {:id file-id}
+                               {::db/return-keys false})
+
+                   (with-meta mobj
+                     {::audit/replace-props
+                      {:name (:name params)
+                       :file-id file-id
+                       :is-local (:is-local params)
+                       :size (:size content)
+                       :mtype (:mtype content)}})))))
 
 (defn- big-enough-for-thumbnail?
   "Checks if the provided image info is big enough for
@@ -142,20 +152,14 @@
       :always
       (assoc ::image (process-main-image info)))))
 
-(defn create-file-media-object
-  [{:keys [::sto/storage ::db/conn ::wrk/executor]}
+(defn- create-file-media-object
+  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
    {:keys [id file-id is-local name content]}]
-
   (let [result (px/invoke! executor (partial process-image content))
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]
 
-    (db/update! conn :file
-                {:modified-at (dt/now)
-                 :has-media-trimmed false}
-                {:id file-id})
-
     (db/exec-one! conn [sql:create-file-media-object
                         (or id (uuid/next))
                         file-id is-local name
@@ -182,7 +186,18 @@
    ::sm/params schema:create-file-media-object-from-url}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (files/check-edition-permissions! pool profile-id file-id)
-  (create-file-media-object-from-url cfg (assoc params :profile-id profile-id)))
+  ;; We get the minimal file for proper checking if file is not
+  ;; already deleted
+  (let [_    (files/get-minimal-file cfg file-id)
+        mobj (create-file-media-object-from-url cfg (assoc params :profile-id profile-id))]
+
+    (db/update! pool :file
+                {:modified-at (dt/now)
+                 :has-media-trimmed false}
+                {:id file-id}
+                {::db/return-keys false})
+
+    mobj))
 
 (defn download-image
   [{:keys [::http/client]} uri]

backend/src/app/rpc/commands/profile.clj

@@ -41,6 +41,12 @@
 (declare strip-private-attrs)
 (declare verify-password)
 
+(def schema:props-notifications
+  [:map {:title "props-notifications"}
+   [:dashboard-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-invites [::sm/one-of #{:all :none}]]])
+
 (def schema:props
   [:map {:title "ProfileProps"}
    [:plugins {:optional true} schema:plugin-registry]
@@ -51,7 +57,9 @@
    [:v2-info-shown {:optional true} ::sm/boolean]
    [:welcome-file-id {:optional true} [:maybe ::sm/boolean]]
    [:release-notes-viewed {:optional true}
-    [::sm/text {:max 100}]]])
+    [::sm/text {:max 100}]]
+   [:notifications {:optional true} schema:props-notifications]
+   [:workspace-visited {:optional true} ::sm/boolean]])
 
 (def schema:profile
   [:map {:title "Profile"}
@@ -200,6 +208,44 @@
                 {:id id})
     nil))
 
+
+;; --- MUTATION: Update notifications
+
+(def ^:private
+  schema:update-profile-notifications
+  [:map {:title "update-profile-notifications"}
+   [:dashboard-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-invites [::sm/one-of #{:all :none}]]])
+
+(declare update-notifications!)
+
+(sv/defmethod ::update-profile-notifications
+  {::doc/added "2.4.0"
+   ::sm/params schema:update-profile-notifications
+   ::climit/id :auth/global}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg update-notifications! (assoc params :profile-id profile-id)))
+
+(defn- update-notifications!
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
+  (let [profile (get-profile conn profile-id)
+
+        notifications
+        {:dashboard-comments dashboard-comments
+         :email-comments email-comments
+         :email-invites email-invites}]
+
+    (db/update!
+     conn :profile
+     {:props
+      (-> (:props profile)
+          (assoc :notifications notifications)
+          (db/tjson))}
+     {:id (:id profile)})
+
+    nil))
+
 ;; --- MUTATION: Update Photo
 
 (declare upload-photo)
@@ -422,7 +468,9 @@
                                   :deleted-at deleted-at
                                   :id profile-id}})
 
-      (rph/with-transform {} (session/delete-fn cfg)))))
+
+      (-> (rph/wrap nil)
+          (rph/with-transform (session/delete-fn cfg))))))
 
 
 ;; --- HELPERS
@@ -431,8 +479,11 @@
   "WITH owner_teams AS (
       SELECT tpr.team_id AS id
         FROM team_profile_rel AS tpr
+        JOIN team AS t ON (t.id = tpr.team_id)
        WHERE tpr.is_owner IS TRUE
          AND tpr.profile_id = ?
+         AND (t.deleted_at IS NULL OR
+              t.deleted_at > now())
    )
    SELECT tpr.team_id AS id,
           count(tpr.profile_id) - 1 AS participants

backend/src/app/rpc/commands/search.clj

@@ -9,7 +9,6 @@
    [app.common.schema :as sm]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :refer [resolve-public-uri]]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]))
 
@@ -61,7 +60,7 @@
                (if-let [media-id (:media-id row)]
                  (-> row
                      (dissoc :media-id)
-                     (assoc :thumbnail-uri (resolve-public-uri media-id)))
+                     (assoc :thumbnail-id media-id))
                  (dissoc row :media-id))))))
 
 (def ^:private schema:search-files

backend/src/app/rpc/commands/teams.clj

@@ -30,7 +30,8 @@
    [app.storage :as sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as wrk]))
+   [app.worker :as wrk]
+   [clojure.set :as set]))
 
 ;; --- Helpers & Specs
 
@@ -285,18 +286,18 @@
 ;; implemented in UI)
 
 (def sql:team-users
-  "select pf.id, pf.fullname, pf.photo_id
+  "select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join team_profile_rel as tpr on (tpr.profile_id = pf.id)
     where tpr.team_id = ?
     union
-   select pf.id, pf.fullname, pf.photo_id
+   select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join project_profile_rel as ppr on (ppr.profile_id = pf.id)
     inner join project as p on (ppr.project_id = p.id)
     where p.team_id = ?
    union
-   select pf.id, pf.fullname, pf.photo_id
+   select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join file_profile_rel as fpr on (fpr.profile_id = pf.id)
     inner join file as f on (fpr.file_id = f.id)
@@ -307,6 +308,30 @@
   [conn team-id]
   (db/exec! conn [sql:team-users team-id team-id team-id]))
 
+;; Get the users but add the props property
+(def sql:team-users+props
+  "select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join team_profile_rel as tpr on (tpr.profile_id = pf.id)
+    where tpr.team_id = ?
+    union
+   select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join project_profile_rel as ppr on (ppr.profile_id = pf.id)
+    inner join project as p on (ppr.project_id = p.id)
+    where p.team_id = ?
+   union
+   select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join file_profile_rel as fpr on (fpr.profile_id = pf.id)
+    inner join file as f on (fpr.file_id = f.id)
+    inner join project as p on (f.project_id = p.id)
+    where p.team_id = ?")
+
+(defn get-users+props
+  [conn team-id]
+  (db/exec! conn [sql:team-users+props team-id team-id team-id]))
+
 (def sql:get-team-by-file
   "SELECT t.*
      FROM team AS t
@@ -416,6 +441,7 @@
                       ::quotes/profile-id profile-id})
 
   (let [features (-> (cfeat/get-enabled-features cf/flags)
+                     (set/difference cfeat/frontend-only-features)
                      (cfeat/check-client-features! (:features params)))
         params   (-> params
                      (assoc :profile-id profile-id)

backend/src/app/rpc/commands/teams_invitations.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.teams-invitations
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
@@ -15,7 +16,6 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.db.sql :as sql]
    [app.email :as eml]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
@@ -34,7 +34,6 @@
 
 ;; --- Mutation: Create Team Invitation
 
-
 (def sql:upsert-team-invitation
   "insert into team_invitation(id, team_id, email_to, created_by, role, valid_until)
    values (?, ?, ?, ?, ?, ?)
@@ -79,27 +78,23 @@
    [:role ::types.team/role]
    [:email ::sm/email]])
 
-(def ^:private check-create-invitation-params!
+(def ^:private check-create-invitation-params
   (sm/check-fn schema:create-invitation))
 
+(defn- allow-invitation-emails?
+  [member]
+  (let [notifications (dm/get-in member [:props :notifications])]
+    (not= :none (:email-invites notifications))))
+
 (defn- create-invitation
   [{:keys [::db/conn] :as cfg} {:keys [team profile role email] :as params}]
 
-  (dm/assert!
-   "expected valid connection on cfg parameter"
-   (db/connection? conn))
-
-  (dm/assert!
-   "expected valid params for `create-invitation` fn"
-   (check-create-invitation-params! params))
+  (assert (db/connection? conn) "expected valid connection on cfg parameter")
+  (assert (check-create-invitation-params params))
 
   (let [email  (profile/clean-email email)
         member (profile/get-profile-by-email conn email)]
 
-    (teams/check-profile-muted conn member)
-    (teams/check-email-bounce conn email true)
-    (teams/check-email-spam conn email true)
-
     ;; When we have email verification disabled and invitation user is
     ;; already present in the database, we proceed to add it to the
     ;; team as-is, without email roundtrip.
@@ -125,62 +120,65 @@
 
         nil)
 
-      (let [id         (uuid/next)
-            expire     (dt/in-future "168h") ;; 7 days
-            invitation (db/exec-one! conn [sql:upsert-team-invitation id
-                                           (:id team) (str/lower email)
-                                           (:id profile)
-                                           (name role) expire
-                                           (name role) expire])
-            updated?   (not= id (:id invitation))
-            profile-id (:id profile)
-            tprops     {:profile-id profile-id
-                        :invitation-id (:id invitation)
-                        :valid-until expire
-                        :team-id (:id team)
-                        :member-email (:email-to invitation)
-                        :member-id (:id member)
-                        :role role}
-            itoken     (create-invitation-token cfg tprops)
-            ptoken     (create-profile-identity-token cfg profile-id)]
+      (do
+        (some->> member (teams/check-profile-muted conn))
+        (teams/check-email-bounce conn email true)
+        (teams/check-email-spam conn email true)
 
-        (when (contains? cf/flags :log-invitation-tokens)
-          (l/info :hint "invitation token" :token itoken))
+        (let [id         (uuid/next)
+              expire     (dt/in-future "168h") ;; 7 days
+              invitation (db/exec-one! conn [sql:upsert-team-invitation id
+                                             (:id team) (str/lower email)
+                                             (:id profile)
+                                             (name role) expire
+                                             (name role) expire])
+              updated?   (not= id (:id invitation))
+              profile-id (:id profile)
+              tprops     {:profile-id profile-id
+                          :invitation-id (:id invitation)
+                          :valid-until expire
+                          :team-id (:id team)
+                          :member-email (:email-to invitation)
+                          :member-id (:id member)
+                          :role role}
+              itoken     (create-invitation-token cfg tprops)
+              ptoken     (create-profile-identity-token cfg profile-id)]
 
-        (let [props  (-> (dissoc tprops :profile-id)
-                         (audit/clean-props))
-              evname (if updated?
-                       "update-team-invitation"
-                       "create-team-invitation")
-              event (-> (audit/event-from-rpc-params params)
-                        (assoc ::audit/name evname)
-                        (assoc ::audit/props props))]
-          (audit/submit! cfg event))
+          (when (contains? cf/flags :log-invitation-tokens)
+            (l/info :hint "invitation token" :token itoken))
 
-        (eml/send! {::eml/conn conn
-                    ::eml/factory eml/invite-to-team
-                    :public-uri (cf/get :public-uri)
-                    :to email
-                    :invited-by (:fullname profile)
-                    :team (:name team)
-                    :token itoken
-                    :extra-data ptoken})
+          (let [props  (-> (dissoc tprops :profile-id)
+                           (audit/clean-props))
+                evname (if updated?
+                         "update-team-invitation"
+                         "create-team-invitation")
+                event (-> (audit/event-from-rpc-params params)
+                          (assoc ::audit/name evname)
+                          (assoc ::audit/props props))]
+            (audit/submit! cfg event))
 
-        itoken))))
+          (when (allow-invitation-emails? member)
+            (eml/send! {::eml/conn conn
+                        ::eml/factory eml/invite-to-team
+                        :public-uri (cf/get :public-uri)
+                        :to email
+                        :invited-by (:fullname profile)
+                        :team (:name team)
+                        :token itoken
+                        :extra-data ptoken}))
 
-(defn- add-user-to-team
-  [conn profile team role email]
+          itoken)))))
+
+(defn- add-member-to-team
+  [conn profile team role member]
 
   (let [team-id (:id team)
-        member  (db/get* conn :profile
-                         {:email (str/lower email)}
-                         {::sql/columns [:id :email]})
         params  (merge
                  {:team-id team-id
                   :profile-id (:id member)}
                  (get types.team/permissions-for-role role))]
 
-      ;; Do not allow blocked users to join teams.
+    ;; Do not allow blocked users to join teams.
     (when (:is-blocked member)
       (ex/raise :type :restriction
                 :code :profile-blocked))
@@ -205,29 +203,33 @@
     (eml/send! {::eml/conn conn
                 ::eml/factory eml/join-team
                 :public-uri (cf/get :public-uri)
-                :to email
+                :to (:email member)
                 :invited-by (:fullname profile)
                 :team (:name team)
                 :team-id (:id team)})))
 
-(def sql:valid-requests-email
-  "SELECT p.email
+(def ^:private sql:valid-access-request-profiles
+  "SELECT p.id, p.email, p.is_blocked
      FROM team_access_request AS tr
      JOIN profile AS p ON (tr.requester_id = p.id)
     WHERE tr.team_id = ?
-      AND tr.auto_join_until > now()")
+      AND tr.auto_join_until > now()
+      AND (p.deleted_at IS NULL OR
+           p.deleted_at > now())")
 
-(defn- get-valid-requests-email
+(defn- get-valid-access-request-profiles
   [conn team-id]
-  (db/exec! conn [sql:valid-requests-email team-id]))
+  (db/exec! conn [sql:valid-access-request-profiles team-id]))
 
-(def ^:private xf:map-email
-  (map :email))
+(def ^:private xf:map-email (map :email))
 
 (defn- create-team-invitations
   [{:keys [::db/conn] :as cfg} {:keys [profile team role emails] :as params}]
-  (let [join-requests    (into #{} xf:map-email
-                               (get-valid-requests-email conn (:id team)))
+  (let [emails           (set emails)
+
+        join-requests    (->> (get-valid-access-request-profiles conn (:id team))
+                              (d/index-by :email))
+
         team-members     (into #{} xf:map-email
                                (teams/get-team-members conn (:id team)))
 
@@ -245,8 +247,10 @@
 
     ;; For requested invitations, do not send invitation emails, add
     ;; the user directly to the team
-    (->> (filter join-requests emails)
-         (run! (partial add-user-to-team conn profile team role)))
+    (->> join-requests
+         (filter #(contains? emails (key %)))
+         (map val)
+         (run! (partial add-member-to-team conn profile team role)))
 
     invitations))
 
@@ -532,17 +536,20 @@
         team-owner (get-team-owner conn team-id)
 
         file       (when (some? file-id)
-                     (get-file-for-team-access-request cfg file-id))
+                     (get-file-for-team-access-request cfg file-id))]
 
-        request    (upsert-team-access-request conn team-id profile-id)]
-
-    ;; FIXME missing quotes
+    (-> cfg
+        (assoc ::quotes/profile-id profile-id)
+        (assoc ::quotes/team-id team-id)
+        (quotes/check! {::quotes/id ::quotes/team-access-requests-per-team}
+                       {::quotes/id ::quotes/team-access-requests-per-requester}))
 
     (teams/check-profile-muted conn requester)
     (teams/check-email-bounce conn (:email team-owner) false)
     (teams/check-email-spam conn (:email team-owner) true)
 
-    (let [factory (cond
+    (let [request (upsert-team-access-request conn team-id profile-id)
+          factory (cond
                     (and (some? file) (:is-default team) is-viewer)
                     eml/request-file-access-yourpenpot-view
 
@@ -565,9 +572,7 @@
                   :team-id team-id
                   :file-name (:name file)
                   :file-id file-id
-                  :page-id (:page-id file)}))
-
-    (with-meta {:request request}
-      {::audit/props {:request 1}})))
-
+                  :page-id (:page-id file)})
 
+      (with-meta {:request request}
+        {::audit/props {:request 1}}))))

backend/src/app/rpc/commands/verify_token.clj

@@ -166,23 +166,26 @@
         ;; invited team.
         (let [props {:team-id (:team-id claims)
                      :role (:role claims)
-                     :invitation-id (:id invitation)}
+                     :invitation-id (:id invitation)}]
 
-              accept-invitation-event
-              (-> (audit/event-from-rpc-params params)
-                  (assoc ::audit/name "accept-team-invitation")
-                  (assoc ::audit/props props))
+          (audit/submit!
+           cfg
+           (-> (audit/event-from-rpc-params params)
+               (assoc ::audit/name "accept-team-invitation")
+               (assoc ::audit/props props)))
 
-              accept-invitation-from-event
-              (-> (audit/event-from-rpc-params params)
-                  (assoc ::audit/profile-id (:created-by invitation))
-                  (assoc ::audit/name "accept-team-invitation-from")
-                  (assoc ::audit/props (assoc props
-                                              :profile-id (:id profile)
-                                              :email (:email profile))))]
-
-          (audit/submit! cfg accept-invitation-event)
-          (audit/submit! cfg accept-invitation-from-event)
+          ;; NOTE: Backward compatibility; old invitations can
+          ;; have the `created-by` to be nil; so in this case we
+          ;; don't submit this event to the audit-log
+          (when-let [created-by (:created-by invitation)]
+            (audit/submit!
+             cfg
+             (-> (audit/event-from-rpc-params params)
+                 (assoc ::audit/profile-id created-by)
+                 (assoc ::audit/name "accept-team-invitation-from")
+                 (assoc ::audit/props (assoc props
+                                             :profile-id (:id profile)
+                                             :email (:email profile))))))
 
           (accept-invitation cfg claims invitation profile)
           (assoc claims :state :created))

backend/src/app/rpc/commands/viewer.clj

@@ -12,12 +12,12 @@
    [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.comments :as comments]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [cuerdas.core :as str]))
 
 ;; --- QUERY: View Only Bundle
 
@@ -27,6 +27,27 @@
       (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
       (update :pages-index select-keys allowed)))
 
+(defn obfuscate-email
+  [email]
+  (let [[name domain]
+        (str/split email "@" 2)
+
+        [_ rest]
+        (str/split domain "." 2)
+
+        name
+        (if (> (count name) 3)
+          (str (subs name 0 1) (apply str (take (dec (count name)) (repeat "*"))))
+          "****")]
+
+    (str name "@****." rest)))
+
+(defn anonymize-member
+  [member]
+  (-> (select-keys member [:id :email :name :fullname :photo-id])
+      (update :email obfuscate-email)
+      (assoc :can-read true)))
+
 (defn- get-view-only-bundle
   [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
   (let [file    (files/get-file cfg file-id)
@@ -38,10 +59,13 @@
         team    (-> (db/get conn :team {:id (:team-id project)})
                     (teams/decode-row))
 
-        members (into #{} (->> (teams/get-team-members conn (:team-id project))
-                               (map :id)))
+        members    (cond->> (teams/get-team-members conn (:team-id project))
+                     (= :share-link (:type perms))
+                     (mapv anonymize-member))
 
-        perms   (assoc perms :in-team (contains? members profile-id))
+        member-ids (into #{} (map :id) members)
+
+        perms   (assoc perms :in-team (contains? member-ids profile-id))
 
         _       (-> (cfeat/get-team-enabled-features cf/flags team)
                     (cfeat/check-client-features! (:features params))
@@ -55,7 +79,6 @@
                   (update :data select-keys [:id :options :pages :pages-index :components]))
 
         libs    (files/get-file-libraries conn file-id)
-        users   (comments/get-file-comments-users conn file-id profile-id)
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]
                              (-> row
@@ -71,13 +94,14 @@
                           {:team-id (:id team)
                            :deleted-at nil})]
 
-    {:users users
+    {:users members
+     :profiles members
      :fonts fonts
      :project project
      :share-links links
      :libraries libs
      :file file
-     :team team
+     :team (assoc team :permissions perms)
      :permissions perms}))
 
 (def schema:get-view-only-bundle

backend/src/app/rpc/doc.clj

@@ -87,6 +87,7 @@
       (let [params  (:query-params request)
             pstyle  (:type params "js")
             context (assoc context :param-style pstyle)]
+
         {::yres/status 200
          ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
                          (tmpl/render context))}))
@@ -207,7 +208,7 @@
   (assert (sm/valid? ::rpc/methods (::rpc/methods params)) "expected valid methods"))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [methods] :as cfg}]
+  [_ {:keys [::rpc/methods] :as cfg}]
   [(let [context (prepare-doc-context methods)]
      [["/_doc"
        {:handler (doc-handler context)

backend/src/app/rpc/quotes.clj

@@ -442,7 +442,7 @@
 
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; QUOTE: SNAPSHOTS-PER-FILE
+;; QUOTE: SNAPSHOTS-PER-TEAM
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private schema:snapshots-per-team
@@ -472,6 +472,57 @@
       (assoc ::count-sql [sql:get-snapshots-per-team team-id])
       (generic-check!)))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; QUOTE: TEAM-ACCESS-REQUESTS-PER-TEAM
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def ^:private schema:team-access-requests-per-team
+  [:map
+   [::profile-id ::sm/uuid]
+   [::team-id ::sm/uuid]])
+
+(def ^:private valid-team-access-requests-per-team-quote?
+  (sm/lazy-validator schema:team-access-requests-per-team))
+
+(def ^:private sql:get-team-access-requests-per-team
+  "SELECT count(*) AS total
+     FROM team_access_request AS tar
+    WHERE tar.team_id = ?")
+
+(defmethod check-quote ::team-access-requests-per-team
+  [{:keys [::profile-id ::team-id ::target] :as quote}]
+  (assert (valid-team-access-requests-per-team-quote? quote) "invalid quote parameters")
+  (-> quote
+      (assoc ::default (cf/get :quotes-team-access-requests-per-team Integer/MAX_VALUE))
+      (assoc ::quote-sql [sql:get-quotes-2 target team-id profile-id profile-id])
+      (assoc ::count-sql [sql:get-team-access-requests-per-team team-id])
+      (generic-check!)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; QUOTE: TEAM-ACCESS-REQUESTS-PER-REQUESTER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def ^:private schema:team-access-requests-per-requester
+  [:map
+   [::profile-id ::sm/uuid]])
+
+(def ^:private valid-team-access-requests-per-requester-quote?
+  (sm/lazy-validator schema:team-access-requests-per-requester))
+
+(def ^:private sql:get-team-access-requests-per-requester
+  "SELECT count(*) AS total
+     FROM team_access_request AS tar
+    WHERE tar.requester_id = ?")
+
+(defmethod check-quote ::team-access-requests-per-requester
+  [{:keys [::profile-id ::target] :as quote}]
+  (assert (valid-team-access-requests-per-requester-quote? quote) "invalid quote parameters")
+  (-> quote
+      (assoc ::default (cf/get :quotes-team-access-requests-per-requester Integer/MAX_VALUE))
+      (assoc ::quote-sql [sql:get-quotes-1 target profile-id])
+      (assoc ::count-sql [sql:get-team-access-requests-per-requester profile-id])
+      (generic-check!)))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; QUOTE: DEFAULT
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/setup.clj

@@ -74,8 +74,7 @@
 
 (defmethod ig/assert-key ::props
   [_ params]
-  (assert (db/pool? (::db/pool params)) "expected valid database pool")
-  (assert (string? (::key params)) "expected valid key string"))
+  (assert (db/pool? (::db/pool params)) "expected valid database pool"))
 
 (defmethod ig/init-key ::props
   [_ {:keys [::db/pool ::key] :as cfg}]
@@ -83,7 +82,7 @@
     (db/xact-lock! conn 0)
     (when-not key
       (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
-                         "all sessions on each restart, it is hightly recommended setting up the "
+                         "all sessions on each restart, it is highly recommended setting up the "
                          "PENPOT_SECRET_KEY environment variable")))
 
     (let [secret (or key (generate-random-key))]

backend/src/app/setup/templates.clj

@@ -54,6 +54,7 @@
                               (::setup/templates cfg))]
     (let [dest (fs/join fs/*cwd* "builtin-templates")
           path (or (:path template) (fs/join dest template-id))]
+
       (if (fs/exists? path)
         (io/input-stream path)
         (let [resp (http/req! cfg

backend/src/app/srepl/fixes.clj

@@ -53,9 +53,9 @@
   fixes all not propertly referenced file-media-object for a file"
   [{:keys [id data] :as file} & _]
   (let [conn  (db/get-connection h/*system*)
-        used  (bfc/collect-used-media data)
+        used  (cfh/collect-used-media data)
         ids   (db/create-array conn "uuid" used)
-        sql   (str "SELECT * FROM file_media_object WHERE id = ANY(?)")
+        sql   "SELECT * FROM file_media_object WHERE id = ANY(?)"
         rows  (db/exec! conn [sql ids])
         index (reduce (fn [index media]
                         (if (not= (:file-id media) id)
@@ -272,6 +272,7 @@
                (reduce +)))
 
         num-missing-slots (count-slots-data (:data file))]
+
     (when (pos? num-missing-slots)
       (l/trc :info (str "Shapes with children with the same swap slot: " num-missing-slots) :file-id (str (:id file))))
     file))

backend/src/app/srepl/fixes/media_refs.clj

@@ -0,0 +1,43 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.srepl.fixes.media-refs
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.files.helpers :as cfh]
+   [app.srepl.helpers :as h]))
+
+(defn- collect-media-refs
+  "Given a file data map, returns all media references used on pages of
+  the file data; components and other parts of the file data are not analized"
+  [data]
+  (let [xform-collect
+        (comp
+         (map val)
+         (mapcat (fn [object]
+                   (->> (cfh/collect-shape-media-refs object)
+                        (map (fn [id]
+                               {:shape-id (:id object)
+                                :id id}))))))
+        process-page
+        (fn [result page-id container]
+          (let [xform (comp xform-collect (map #(assoc % :page-id page-id)))]
+            (into result xform (:objects container))))]
+
+    (reduce-kv process-page [] (:pages-index data))))
+
+(defn update-all-media-references
+  "Check if all file media object references are in plance and create
+  new ones if some of them are missing; this prevents strange bugs on
+  having the same media object associated with two different files;"
+  [cfg file]
+  (let [media-refs (collect-media-refs (:data file))]
+    (bfc/update-media-references! cfg file media-refs)))
+
+(defn process-file
+  [file _opts]
+  (let [system (h/get-current-system)]
+    (update-all-media-references system file)))

backend/src/app/srepl/helpers.clj

@@ -8,17 +8,15 @@
   "A  main namespace for server repl."
   (:refer-clojure :exclude [parse-uuid])
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
-   [app.common.files.migrations :as fmg]
    [app.common.files.validate :as cfv]
    [app.db :as db]
    [app.features.components-v2 :as feat.comp-v2]
-   [app.features.fdata :as feat.fdata]
    [app.main :as main]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.files-snapshot :as fsnap]
-   [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]))
+   [app.util.time :as dt]))
 
 (def ^:dynamic *system* nil)
 
@@ -27,6 +25,10 @@
   (locking println
     (apply println params)))
 
+(defn get-current-system
+  []
+  *system*)
+
 (defn parse-uuid
   [v]
   (if (string? v)
@@ -35,49 +37,18 @@
 
 (defn get-file
   "Get the migrated data of one file."
-  ([id] (get-file (or *system* main/system) id nil))
-  ([system id & {:keys [raw?] :as opts}]
+  ([id]
+   (get-file (or *system* main/system) id))
+  ([system id]
+   (db/run! system bfc/get-file id)))
+
+(defn get-raw-file
+  "Get the migrated data of one file."
+  ([id] (get-raw-file (or *system* main/system) id))
+  ([system id]
    (db/run! system
             (fn [system]
-              (let [file (files/get-file system id :migrate? false)]
-                (if raw?
-                  file
-                  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
-                    (-> file
-                        (update :data feat.fdata/process-pointers deref)
-                        (update :data feat.fdata/process-objects (partial into {}))
-                        (fmg/migrate-file)))))))))
-
-(defn update-file!
-  [system {:keys [id] :as file}]
-  (let [conn (db/get-connection system)
-        file (if (contains? (:features file) "fdata/objects-map")
-               (feat.fdata/enable-objects-map file)
-               file)
-
-        file (if (contains? (:features file) "fdata/pointer-map")
-               (binding [pmap/*tracked* (pmap/create-tracked)]
-                 (let [file (feat.fdata/enable-pointer-map file)]
-                   (feat.fdata/persist-pointers! system id)
-                   file))
-               file)
-
-        file (-> file
-                 (update :features db/encode-pgarray conn "text")
-                 (update :data blob/encode))]
-
-    (db/update! conn :file
-                {:revn (:revn file)
-                 :data (:data file)
-                 :version (:version file)
-                 :features (:features file)
-                 :deleted-at (:deleted-at file)
-                 :created-at (:created-at file)
-                 :modified-at (:modified-at file)
-                 :data-backend nil
-                 :data-ref-id nil
-                 :has-media-trimmed false}
-                {:id (:id file)})))
+              (files/get-file system id :migrate? false)))))
 
 (defn update-team!
   [system {:keys [id] :as team}]
@@ -90,14 +61,6 @@
                 {:id id})
     team))
 
-(defn get-raw-file
-  "Get the migrated data of one file."
-  ([id] (get-raw-file (or *system* main/system) id))
-  ([system id]
-   (db/run! system
-            (fn [system]
-              (files/get-file system id :migrate? false)))))
-
 (defn reset-file-data!
   "Hardcode replace of the data of one file."
   [system id data]
@@ -134,8 +97,11 @@
   (let [conn (db/get-connection system)]
     (->> (feat.comp-v2/get-and-lock-team-files conn team-id)
          (reduce (fn [result file-id]
-                   (fsnap/create-file-snapshot! system nil file-id label)
-                   (inc result))
+                   (let [file (fsnap/get-file-snapshots system file-id)]
+                     (fsnap/create-file-snapshot! system file
+                                                  {:label label
+                                                   :created-by :admin})
+                     (inc result)))
                  0))))
 
 (defn restore-team-snapshot!
@@ -162,25 +128,30 @@
 
 (defn process-file!
   [system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}]
-
-  (when (string? label)
-    (fsnap/create-file-snapshot! system nil file-id label))
-
   (let [conn  (db/get-connection system)
-        file  (get-file system file-id opts)
+        file  (bfc/get-file system file-id ::db/for-update true)
         libs  (when with-libraries?
                 (->> (files/get-file-libraries conn file-id)
                      (into [file] (map (fn [{:keys [id]}]
-                                         (get-file system id))))
+                                         (bfc/get-file system id))))
                      (d/index-by :id)))
 
-        file' (if with-libraries?
-                (update-fn file libs opts)
-                (update-fn file opts))]
+        file' (when file
+                (if with-libraries?
+                  (update-fn file libs opts)
+                  (update-fn file opts)))]
 
     (when (and (some? file')
                (not (identical? file file')))
-      (when validate? (cfv/validate-file-schema! file'))
+      (when validate?
+        (cfv/validate-file-schema! file'))
+
+      (when (string? label)
+        (fsnap/create-file-snapshot! system file
+                                     {:label label
+                                      :deleted-at (dt/in-future {:days 30})
+                                      :created-by :admin}))
+
       (let [file' (update file' :revn inc)]
-        (update-file! system file')
+        (bfc/update-file! system file')
         true))))

backend/src/app/srepl/main.clj

@@ -431,25 +431,40 @@
 
         process-file
         (fn [file-id idx tpoint]
-          (try
-            (l/trc :hint "process:file:start" :file-id (str file-id) :index idx)
-            (let [system (assoc main/system ::db/rollback rollback?)]
-              (db/tx-run! system (fn [system]
-                                   (binding [h/*system* system]
-                                     (h/process-file! system file-id update-fn opts)))))
-
-            (catch Throwable cause
-              (l/wrn :hint "unexpected error on processing file (skiping)"
+          (let [thread-id (px/get-thread-id)]
+            (try
+              (l/trc :hint "process:file:start"
+                     :tid thread-id
                      :file-id (str file-id)
-                     :index idx
-                     :cause cause))
-            (finally
-              (ps/release! sjobs)
-              (let [elapsed (dt/format-duration (tpoint))]
-                (l/trc :hint "process:file:end"
+                     :index idx)
+              (let [system (assoc main/system ::db/rollback rollback?)]
+                (db/tx-run! system (fn [system]
+                                     (binding [h/*system* system]
+                                       (h/process-file! system file-id update-fn opts)))))
+
+              (catch Throwable cause
+                (l/wrn :hint "unexpected error on processing file (skiping)"
+                       :tid thread-id
                        :file-id (str file-id)
                        :index idx
-                       :elapsed elapsed)))))
+                       :cause cause))
+              (finally
+                (when-let [pause (:pause opts)]
+                  (Thread/sleep (int pause)))
+
+                (ps/release! sjobs)
+                (let [elapsed (dt/format-duration (tpoint))]
+                  (l/trc :hint "process:file:end"
+                         :tid thread-id
+                         :file-id (str file-id)
+                         :index idx
+                         :elapsed elapsed))))))
+
+        process-file*
+        (fn [idx file-id]
+          (ps/acquire! sjobs)
+          (px/run! executor (partial process-file file-id idx (dt/tpoint)))
+          (inc idx))
 
         process-files
         (fn [{:keys [::db/conn] :as system}]
@@ -457,14 +472,12 @@
           (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"])
 
           (try
-            (reduce (fn [idx file-id]
-                      (ps/acquire! sjobs)
-                      (px/run! executor (partial process-file file-id idx (dt/tpoint)))
-                      (inc idx))
-                    0
-                    (->> (db/cursor conn [query] {:chunk-size 1})
-                         (take max-items)
-                         (map :id)))
+            (->> (db/plan conn [query])
+                 (transduce (comp
+                             (take max-items)
+                             (map :id))
+                            (completing process-file*)
+                            0))
             (finally
               ;; Close and await tasks
               (pu/close! executor))))]

backend/src/app/storage/tmp.clj

@@ -16,10 +16,13 @@
    [app.util.time :as dt]
    [app.worker :as wrk]
    [datoteka.fs :as fs]
+   [datoteka.io :as io]
    [integrant.core :as ig]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp])
   (:import
+   java.io.InputStream
+   java.io.OutputStream
    java.nio.file.Files))
 
 (def default-tmp-dir "/tmp/penpot")
@@ -86,3 +89,12 @@
     (fs/delete-on-exit! path)
     (sp/offer! queue [path (some-> min-age dt/duration)])
     path))
+
+(defn tempfile-from
+  "Create a new tempfile from from consuming the stream"
+  [input & {:as options}]
+  (let [path (tempfile options)]
+    (with-open [^InputStream input (io/input-stream input)]
+      (with-open [^OutputStream output (io/output-stream path)]
+        (io/copy input output)))
+    path))

backend/src/app/tasks/delete_object.clj

@@ -40,6 +40,11 @@
                   :file-id id
                   :cause cause))))
 
+    ;; Mark file change to be deleted
+    (db/update! conn :file-change
+                {:deleted-at deleted-at}
+                {:file-id id})
+
     ;; Mark file media objects to be deleted
     (db/update! conn :file-media-object
                 {:deleted-at deleted-at}

backend/src/app/tasks/file_gc.clj

@@ -11,7 +11,7 @@
   inactivity (the default threshold is 72h)."
   (:require
    [app.binfile.common :as bfc]
-   [app.common.files.migrations :as fmg]
+   [app.common.files.helpers :as cfh]
    [app.common.files.validate :as cfv]
    [app.common.logging :as l]
    [app.common.thumbnails :as thc]
@@ -22,29 +22,25 @@
    [app.db :as db]
    [app.features.fdata :as feat.fdata]
    [app.storage :as sto]
-   [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as wrk]
-   [clojure.set :as set]
    [integrant.core :as ig]))
 
-(declare ^:private get-file)
-(declare ^:private decode-file)
-(declare ^:private persist-file!)
+(declare get-file)
 
-(def ^:private sql:get-snapshots
-  "SELECT f.file_id AS id,
-          f.data,
-          f.revn,
-          f.version,
-          f.features,
-          f.data_backend,
-          f.data_ref_id
-     FROM file_change AS f
-    WHERE f.file_id = ?
-      AND f.data IS NOT NULL
-    ORDER BY f.created_at ASC")
+(def sql:get-snapshots
+  "SELECT fc.file_id AS id,
+          fc.id AS snapshot_id,
+          fc.data,
+          fc.revn,
+          fc.version,
+          fc.features,
+          fc.data_backend,
+          fc.data_ref_id
+     FROM file_change AS fc
+    WHERE fc.file_id = ?
+      AND fc.data IS NOT NULL
+    ORDER BY fc.created_at ASC")
 
 (def ^:private sql:mark-file-media-object-deleted
   "UPDATE file_media_object
@@ -52,17 +48,22 @@
     WHERE file_id = ? AND id != ALL(?::uuid[])
    RETURNING id")
 
-(def ^:private xf:collect-used-media
-  (comp (map :data) (mapcat bfc/collect-used-media)))
+(def xf:collect-used-media
+  (comp
+   (map :data)
+   (mapcat cfh/collect-used-media)))
 
 (defn- clean-file-media!
   "Performs the garbage collection of file media objects."
   [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [used   (into #{}
-                     xf:collect-used-media
-                     (cons file
-                           (->> (db/cursor conn [sql:get-snapshots id])
-                                (map (partial decode-file cfg)))))
+  (let [xform  (comp
+                (map (partial bfc/decode-file cfg))
+                xf:collect-used-media)
+
+        used   (->> (db/plan conn [sql:get-snapshots id] {:fetch-size 1})
+                    (transduce xform conj #{}))
+        used   (into used xf:collect-used-media [file])
+
         ids    (db/create-array conn "uuid" used)
         unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids])
                     (into #{} (map :id)))]
@@ -145,51 +146,45 @@
       AND f.deleted_at IS null
     ORDER BY f.modified_at ASC")
 
+(defn- get-used-components
+  "Given a file and a set of components marked for deletion, return a
+  filtered set of component ids that are still un use"
+  [components library-id {:keys [data]}]
+  (filter #(ctf/used-in? data library-id % :component) components))
+
 (defn- clean-deleted-components!
   "Performs the garbage collection of unreferenced deleted components."
   [{:keys [::db/conn] :as cfg} {:keys [data] :as file}]
   (let [file-id (:id file)
 
-        get-used-components
-        (fn [data components]
-          ;; Find which of the components are used in the file.
-          (into #{}
-                (filter #(ctf/used-in? data file-id % :component))
-                components))
+        deleted-components
+        (ctkl/deleted-components-seq data)
 
-        get-unused-components
-        (fn [components files]
-          ;; Find and return a set of unused components (on all files).
-          (reduce (fn [components {:keys [data]}]
-                    (if (seq components)
-                      (->> (get-used-components data components)
-                           (set/difference components))
-                      (reduced components)))
+        xform
+        (mapcat (partial get-used-components deleted-components file-id))
 
-                  components
-                  files))
+        used-remote
+        (->> (db/plan conn [sql:get-files-for-library file-id] {:fetch-size 1})
+             (transduce (comp (map (partial bfc/decode-file cfg)) xform) conj #{}))
 
-        process-fdata
-        (fn [data unused]
-          (reduce (fn [data id]
-                    (l/trc :hint "delete component"
-                           :component-id (str id)
-                           :file-id (str file-id))
-                    (ctkl/delete-component data id))
-                  data
-                  unused))
+        used-local
+        (into #{} xform [file])
 
-        deleted (into #{} (ctkl/deleted-components-seq data))
-
-        unused  (->> (db/cursor conn [sql:get-files-for-library file-id] {:chunk-size 1})
-                     (map (partial decode-file cfg))
-                     (cons file)
-                     (get-unused-components deleted)
-                     (mapv :id)
-                     (set))
-
-        file    (update file :data process-fdata unused)]
+        unused
+        (transduce bfc/xf-map-id disj
+                   (into #{} bfc/xf-map-id deleted-components)
+                   (concat used-remote used-local))
 
+        file
+        (update file :data
+                (fn [data]
+                  (reduce (fn [data id]
+                            (l/trc :hint "delete component"
+                                   :component-id (str id)
+                                   :file-id (str file-id))
+                            (ctkl/delete-component data id))
+                          data
+                          unused)))]
 
     (l/dbg :hint "clean" :rel "components" :file-id (str file-id) :total (count unused))
     file))
@@ -204,31 +199,32 @@
 
 (def ^:private xf:collect-pointers
   (comp (map :data)
-        (map blob/decode)
         (mapcat feat.fdata/get-used-pointer-ids)))
 
-(defn- clean-data-fragments!
+(defn- clean-fragments!
   [{:keys [::db/conn]} {:keys [id] :as file}]
   (let [used   (into #{} xf:collect-pointers [file])
 
-        unused (let [ids (db/create-array conn "uuid" used)]
-                 (->> (db/exec! conn [sql:mark-deleted-data-fragments id ids])
-                      (into #{} (map :id))))]
+        unused (->> (db/exec! conn [sql:mark-deleted-data-fragments id
+                                    (db/create-array conn "uuid" used)])
+                    (into #{} bfc/xf-map-id))]
 
     (l/dbg :hint "clean" :rel "file-data-fragment" :file-id (str id) :total (count unused))
     (doseq [id unused]
       (l/trc :hint "mark deleted"
              :rel "file-data-fragment"
              :id (str id)
-             :file-id (str id)))))
+             :file-id (str id)))
+
+    file))
 
 (defn- clean-media!
   [cfg file]
   (let [file (->> file
+                  (clean-deleted-components! cfg)
                   (clean-file-media! cfg)
                   (clean-file-thumbnails! cfg)
-                  (clean-file-object-thumbnails! cfg)
-                  (clean-deleted-components! cfg))]
+                  (clean-file-object-thumbnails! cfg))]
     (cfv/validate-file-schema! file)
     file))
 
@@ -249,65 +245,27 @@
       FOR UPDATE
      SKIP LOCKED")
 
-(defn- get-file
-  [{:keys [::db/conn ::min-age ::file-id]}]
-  (->> (db/exec! conn [sql:get-file min-age file-id])
-       (first)))
-
-(defn- decode-file
-  [cfg {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-    (-> (feat.fdata/resolve-file-data cfg file)
-        (update :features db/decode-pgarray #{})
-        (update :data blob/decode)
-        (update :data feat.fdata/process-pointers deref)
-        (update :data feat.fdata/process-objects (partial into {}))
-        (update :data assoc :id id)
-        (fmg/migrate-file))))
-
-(defn- persist-file!
-  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file}]
-  (let [file (if (contains? (:features file) "fdata/objects-map")
-               (feat.fdata/enable-objects-map file)
-               file)
-
-        file (if (contains? (:features file) "fdata/pointer-map")
-               (binding [pmap/*tracked* (pmap/create-tracked)]
-                 (let [file (feat.fdata/enable-pointer-map file)]
-                   (feat.fdata/persist-pointers! cfg id)
-                   file))
-               file)
-
-        file (-> file
-                 (update :features db/encode-pgarray conn "text")
-                 (update :data blob/encode))]
-
-    ;; If file was already offloaded, we touch the underlying storage
-    ;; object for properly trigger storage-gc-touched task
-    (when (feat.fdata/offloaded? file)
-      (some->> (:data-ref-id file) (sto/touch-object! storage)))
-
-    (db/update! conn :file
-                {:has-media-trimmed true
-                 :features (:features file)
-                 :version (:version file)
-                 :data (:data file)
-                 :data-backend nil
-                 :data-ref-id nil}
-                {:id id}
-                {::db/return-keys true})))
+(defn get-file
+  [{:keys [::db/conn ::min-age]} file-id]
+  (let [min-age (if min-age
+                  (db/interval min-age)
+                  (db/interval 0))]
+    (->> (db/exec! conn [sql:get-file min-age file-id])
+         (first))))
 
 (defn- process-file!
-  [cfg]
-  (if-let [file (get-file cfg)]
-    (let [file (decode-file cfg file)
-          file (clean-media! cfg file)
-          file (persist-file! cfg file)]
-      (clean-data-fragments! cfg file)
+  [cfg file-id]
+  (if-let [file (get-file cfg file-id)]
+    (let [file (->> file
+                    (bfc/decode-file cfg)
+                    (clean-media! cfg)
+                    (clean-fragments! cfg))
+          file (assoc file :has-media-trimmed true)]
+      (bfc/update-file! cfg file)
       true)
 
     (do
-      (l/dbg :hint "skip" :file-id (str (::file-id cfg)))
+      (l/dbg :hint "skip" :file-id (str file-id))
       false)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -324,15 +282,15 @@
   (fn [{:keys [props] :as task}]
     (let [min-age (dt/duration (or (:min-age props)
                                    (cf/get-deletion-delay)))
+          file-id (get props :file-id)
           cfg     (-> cfg
                       (assoc ::db/rollback (:rollback? props))
-                      (assoc ::file-id (:file-id props))
-                      (assoc ::min-age (db/interval min-age)))]
+                      (assoc ::min-age min-age))]
 
       (try
         (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
                           (let [cfg        (update cfg ::sto/storage sto/configure conn)
-                                processed? (process-file! cfg)]
+                                processed? (process-file! cfg file-id)]
                             (when (and processed? (contains? cf/flags :tiered-file-data-storage))
                               (wrk/submit! (-> cfg
                                                (assoc ::wrk/task :offload-file-data)

backend/src/app/tasks/file_xlog_gc.clj

@@ -1,64 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.tasks.file-xlog-gc
-  (:require
-   [app.common.logging :as l]
-   [app.config :as cf]
-   [app.db :as db]
-   [integrant.core :as ig]))
-
-;; Get the latest available snapshots without exceeding the total
-;; snapshot limit
-(def ^:private sql:get-latest-snapshots
-  "SELECT fch.id, fch.created_at
-     FROM file_change AS fch
-    WHERE fch.file_id = ?
-      AND fch.created_by = 'system'
-      AND fch.data IS NOT NULL
-      AND fch.deleted_at > now()
-    ORDER BY fch.created_at DESC
-    LIMIT ?")
-
-;; Mark all snapshots that are outside the allowed total threshold
-;; available for the GC
-(def ^:private sql:delete-snapshots
-  "UPDATE file_change
-      SET deleted_at = now()
-    WHERE file_id = ?
-      AND deleted_at > now()
-      AND data IS NOT NULL
-      AND created_by = 'system'
-      AND created_at < ?")
-
-(defn- get-alive-snapshots
-  [conn file-id]
-  (let [total     (cf/get :auto-file-snapshot-total 10)
-        snapshots (db/exec! conn [sql:get-latest-snapshots file-id total])]
-    (not-empty snapshots)))
-
-(defn- delete-old-snapshots!
-  [{:keys [::db/conn] :as cfg} file-id]
-  (when-let [snapshots (get-alive-snapshots conn file-id)]
-    (let [last-date (-> snapshots peek :created-at)
-          result    (db/exec-one! conn [sql:delete-snapshots file-id last-date])]
-      (l/inf :hint "delete old file snapshots"
-             :file-id (str file-id)
-             :current (count snapshots)
-             :deleted (db/get-update-count result)))))
-
-(defmethod ig/assert-key ::handler
-  [_ params]
-  (assert (db/pool? (::db/pool params)) "expected a valid database pool"))
-
-(defmethod ig/init-key ::handler
-  [_ cfg]
-  (fn [{:keys [props] :as task}]
-    (let [file-id (:file-id props)]
-      (assert (uuid? file-id) "expected file-id on props")
-      (-> cfg
-          (assoc ::db/rollback (:rollback props false))
-          (db/tx-run! delete-old-snapshots! file-id)))))

backend/src/app/tasks/objects_gc.clj

@@ -26,7 +26,7 @@
 
 (defn- delete-profiles!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-profiles min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id]}]
                  (l/trc :hint "permanently delete" :rel "profile" :id (str id))
 
@@ -49,7 +49,7 @@
 
 (defn- delete-teams!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-teams min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "team"
@@ -77,7 +77,7 @@
 
 (defn- delete-fonts!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-fonts min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at] :as font}]
                  (l/trc :hint "permanently delete"
                         :rel "team-font-variant"
@@ -109,7 +109,7 @@
 
 (defn- delete-projects!
   [{:keys [::db/conn ::min-age ::chunk-size] :as cfg}]
-  (->> (db/cursor conn [sql:get-projects min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "project"
@@ -135,7 +135,7 @@
 
 (defn- delete-files!
   [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
-  (->> (db/cursor conn [sql:get-files min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id deleted-at project-id] :as file}]
                  (l/trc :hint "permanently delete"
                         :rel "file"
@@ -164,7 +164,7 @@
 
 (defn delete-file-thumbnails!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-file-thumbnails min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-thumbnail"
@@ -193,7 +193,7 @@
 
 (defn delete-file-object-thumbnails!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-file-object-thumbnails min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-tagged-object-thumbnail"
@@ -222,7 +222,7 @@
 
 (defn- delete-file-data-fragments!
   [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
-  (->> (db/cursor conn [sql:get-file-data-fragments min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-data-fragment"
@@ -248,7 +248,7 @@
 
 (defn- delete-file-media-objects!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-file-media-objects min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
                  (l/trc :hint "permanently delete"
                         :rel "file-media-object"
@@ -275,9 +275,9 @@
       FOR UPDATE
      SKIP LOCKED")
 
-(defn- delete-file-change!
+(defn- delete-file-changes!
   [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/cursor conn [sql:get-file-change min-age chunk-size] {:chunk-size 5})
+  (->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
                  (l/trc :hint "permanently delete"
                         :rel "file-change"
@@ -299,11 +299,11 @@
    #'delete-file-data-fragments!
    #'delete-file-object-thumbnails!
    #'delete-file-thumbnails!
+   #'delete-file-changes!
    #'delete-files!
    #'delete-projects!
    #'delete-fonts!
-   #'delete-teams!
-   #'delete-file-change!])
+   #'delete-teams!])
 
 (defn- execute-proc!
   "A generic function that executes the specified proc iterativelly
@@ -326,7 +326,7 @@
   [k v]
   {k (assoc v
             ::min-age (cf/get-deletion-delay)
-            ::chunk-size 50)})
+            ::chunk-size 100)})
 
 (defmethod ig/init-key ::handler
   [_ cfg]

backend/test/backend_tests/binfile_test.clj

@@ -7,6 +7,7 @@
 (ns backend-tests.binfile-test
   "Internal binfile test, no RPC involved"
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v3 :as v3]
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
@@ -93,15 +94,15 @@
 
     (v3/export-files!
      (-> th/*system*
-         (assoc ::v3/ids #{(:id file)})
-         (assoc ::v3/embed-assets false)
-         (assoc ::v3/include-libraries false))
+         (assoc ::bfc/ids #{(:id file)})
+         (assoc ::bfc/embed-assets false)
+         (assoc ::bfc/include-libraries false))
      (io/output-stream output))
 
     (let [result (-> th/*system*
-                     (assoc ::v3/project-id (:default-project-id profile))
-                     (assoc ::v3/profile-id (:id profile))
-                     (assoc ::v3/input output)
+                     (assoc ::bfc/project-id (:default-project-id profile))
+                     (assoc ::bfc/profile-id (:id profile))
+                     (assoc ::bfc/input output)
                      (v3/import-files!))]
       (t/is (= (count result) 1))
       (t/is (every? uuid? result)))))

backend/test/backend_tests/helpers.clj

@@ -62,7 +62,7 @@
 (def default
   {:database-uri "postgresql://postgres/penpot_test"
    :redis-uri "redis://redis/1"
-   :file-snapshot-every 1})
+   :auto-file-snapshot-every 1})
 
 (def config
   (cf/read-config :prefix "penpot-test"
@@ -210,14 +210,14 @@
   ([system i {:keys [profile-id project-id] :as params}]
    (dm/assert! "expected uuid" (uuid? profile-id))
    (dm/assert! "expected uuid" (uuid? project-id))
-   (db/run! system
-            (fn [system]
-              (let [features (cfeat/get-enabled-features cf/flags)]
-                (files.create/create-file system
-                                          (merge {:id (mk-uuid "file" i)
-                                                  :name (str "file" i)
-                                                  :features features}
-                                                 params)))))))
+   (db/tx-run! system
+               (fn [system]
+                 (let [features (cfeat/get-enabled-features cf/flags)]
+                   (files.create/create-file system
+                                             (merge {:id (mk-uuid "file" i)
+                                                     :name (str "file" i)
+                                                     :features features}
+                                                    params)))))))
 
 (defn mark-file-deleted*
   ([params]

backend/test/backend_tests/rpc_comment_test.clj

@@ -177,7 +177,7 @@
             ;; (th/print-result! out)
             (t/is (th/success? out))
             (let [[thread :as result] (:result out)]
-              (t/is (= 1 (count result)))))
+              (t/is (= 0 (count result)))))
 
           (let [data {::th/type :update-comment-thread-status
                       ::rpc/profile-id (:id profile-1)

backend/test/backend_tests/rpc_file_snapshot_test.clj

@@ -97,6 +97,7 @@
             (th/db-query :file-change
                          {:file-id (:id file)}
                          {:order-by [:created-at]})]
+
         (t/is (= 2 (count rows)))
         (t/is (= "user" (:created-by row1)))
         (t/is (= "system" (:created-by row2)))))

backend/test/backend_tests/rpc_file_test.clj

@@ -16,6 +16,7 @@
    [app.db.sql :as sql]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
    [app.storage :as sto]
    [app.util.time :as dt]
    [backend-tests.helpers :as th]
@@ -383,8 +384,19 @@
       ;; as deleted.
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
+      ;; This only clears fragments, the file media objects still referenced because
+      ;; snapshots are preserved
       (let [res (th/run-task! :objects-gc {:min-age 0})]
-        (t/is (= 3 (:processed res))))
+        (t/is (= 2 (:processed res))))
+
+      ;; Mark all snapshots to be a non-snapshot file change
+      (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)])
+      (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
+
+      ;; Rerun the file-gc and objects-gc
+      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (let [res (th/run-task! :objects-gc {:min-age 0})]
+        (t/is (= 2 (:processed res))))
 
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
@@ -417,20 +429,6 @@
 
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
-              (:result out)))
-
-          (update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-            (let [params {::th/type :update-file
-                          ::rpc/profile-id profile-id
-                          :id file-id
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :components-v2 true
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
               (:result out)))]
 
     (let [storage (:app.storage/storage th/*system*)
@@ -550,8 +548,20 @@
       ;; as deleted.
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
+      ;; This only removes unused fragments, file media are still
+      ;; referenced on snapshots.
       (let [res (th/run-task! :objects-gc {:min-age 0})]
-        (t/is (= 7 (:processed res))))
+        (t/is (= 2 (:processed res))))
+
+      ;; Mark all snapshots to be a non-snapshot file change
+      (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
+      (th/db-exec! ["update file_change set data = null where file_id = ?" (:id file)])
+
+      ;; Rerun file-gc and objects-gc task for the same file once all snapshots are
+      ;; "expired/deleted"
+      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (let [res (th/run-task! :objects-gc {:min-age 0})]
+        (t/is (= 6 (:processed res))))
 
       (let [rows (th/db-query :file-data-fragment {:file-id (:id file)
                                                    :deleted-at nil})]
@@ -591,20 +601,7 @@
 
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
-              (:result out)))
-
-          #_(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-                          (let [params {::th/type :update-file
-                                        ::rpc/profile-id profile-id
-                                        :id file-id
-                                        :session-id (uuid/random)
-                                        :revn revn
-                                        :features cfeat/supported-features
-                                        :changes changes}
-                                out    (th/command! params)]
-              ;; (th/print-result! out)
-                            (t/is (nil? (:error out)))
-                            (:result out)))]
+              (:result out)))]
 
     (let [storage  (:app.storage/storage th/*system*)
           profile  (th/create-profile* 1)
@@ -1090,8 +1087,7 @@
         (t/is (contains? result :file-id))
 
         (t/is (= (:id file) (:file-id result)))
-        (t/is (str/starts-with? (get-in result [:page :objects frame1-id :thumbnail])
-                                "http://localhost:3449/assets/by-id/"))
+        (t/is (uuid? (get-in result [:page :objects frame1-id :thumbnail-id])))
         (t/is (= [] (get-in result [:page :objects frame1-id :shapes]))))
 
       ;; Delete thumbnail data
@@ -1337,3 +1333,498 @@
       (t/is (every? #(bytes? (:data %)) rows))
       (t/is (every? #(nil? (:data-ref-id %)) rows))
       (t/is (every? #(nil? (:data-backend %)) rows)))))
+
+(t/deftest file-gc-with-components-1
+  (let [storage (:app.storage/storage th/*system*)
+        profile (th/create-profile* 1)
+        file    (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
+
+        s-id-1  (uuid/random)
+        s-id-2  (uuid/random)
+        c-id    (uuid/random)
+
+        page-id (first (get-in file [:data :pages]))]
+
+    (let [rows (th/db-query :file-data-fragment {:file-id (:id file)
+                                                 :deleted-at nil})]
+      (t/is (= (count rows) 1)))
+
+    ;; Update file inserting new component
+    (update-file!
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id page-id
+       :id s-id-1
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-1
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :main-instance true
+              :component-root true
+              :component-file (:id file)
+              :component-id c-id})}
+
+      {:type :add-obj
+       :page-id page-id
+       :id s-id-2
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-2
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :main-instance false
+              :component-root true
+              :component-file (:id file)
+              :component-id c-id})}
+
+      {:type :add-component
+       :path ""
+       :name "Board"
+       :main-instance-id s-id-1
+       :main-instance-page page-id
+       :id c-id
+       :anotation nil}])
+
+    ;; Run the file-gc task immediately without forced min-age
+    (t/is (false? (th/run-task! :file-gc {:file-id (:id file)})))
+
+    ;; Run the task again
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+
+    ;; Retrieve file and check trimmed attribute
+    (let [row (th/db-get :file {:id (:id file)})]
+      (t/is (true? (:has-media-trimmed row))))
+
+    ;; Check that component exists
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result     (:result out)
+            component (get-in result [:data :components c-id])]
+
+        (t/is (some? component))
+        (t/is (nil? (:objects component)))))
+
+    ;; Now proceed to delete a component
+    (update-file!
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :del-component
+       :id c-id}
+      {:type :del-obj
+       :page-id page-id
+       :id s-id-1
+       :ignore-touched true}])
+
+    ;; ;; Check that component is marked as deleted
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result (:result out)
+            component (get-in result [:data :components c-id])]
+        (t/is (true? (:deleted component)))
+        (t/is (some? (not-empty (:objects component))))))
+
+    ;; Re-run the file-gc task
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+    (let [row (th/db-get :file {:id (:id file)})]
+      (t/is (true? (:has-media-trimmed row))))
+
+    ;; Check that component is still there after file-gc task
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result (:result out)
+            component (get-in result [:data :components c-id])]
+        (t/is (true? (:deleted component)))
+        (t/is (some? (not-empty (:objects component))))))
+
+    ;; Now delete the last instance using deleted component
+    (update-file!
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :del-obj
+       :page-id page-id
+       :id s-id-2
+       :ignore-touched true}])
+
+    ;; Now, we have deleted the usage of component if we pass file-gc,
+    ;; that component should be deleted
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+
+    ;; Check that component is properly removed
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result     (:result out)
+            components (get-in result [:data :components])]
+        (t/is (not (contains? components c-id)))))))
+
+(t/deftest file-gc-with-components-2
+  (let [storage (:app.storage/storage th/*system*)
+        profile (th/create-profile* 1)
+        file-1  (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared true})
+
+        file-2  (th/create-file* 2 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
+
+        rel     (th/link-file-to-library*
+                 {:file-id (:id file-2)
+                  :library-id (:id file-1)})
+
+        s-id-1  (uuid/random)
+        s-id-2  (uuid/random)
+        c-id    (uuid/random)
+
+        f1-page-id (first (get-in file-1 [:data :pages]))
+        f2-page-id (first (get-in file-2 [:data :pages]))]
+
+    ;; Update file library inserting new component
+    (update-file!
+     :file-id (:id file-1)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id f1-page-id
+       :id s-id-1
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-1
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :main-instance true
+              :component-root true
+              :component-file (:id file-1)
+              :component-id c-id})}
+      {:type :add-component
+       :path ""
+       :name "Board"
+       :main-instance-id s-id-1
+       :main-instance-page f1-page-id
+       :id c-id
+       :anotation nil}])
+
+    ;; Instanciate a component in a different file
+    (update-file!
+     :file-id (:id file-2)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id f2-page-id
+       :id s-id-2
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-2
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :main-instance false
+              :component-root true
+              :component-file (:id file-1)
+              :component-id c-id})}])
+
+    ;; Run the file-gc on file and library
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+
+    ;; Check that component exists
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-1)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result     (:result out)
+            component (get-in result [:data :components c-id])]
+
+        (t/is (some? component))
+        (t/is (nil? (:objects component)))))
+
+    ;; Now proceed to delete a component
+    (update-file!
+     :file-id (:id file-1)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :del-component
+       :id c-id}
+      {:type :del-obj
+       :page-id f1-page-id
+       :id s-id-1
+       :ignore-touched true}])
+
+    ;; Check that component is marked as deleted
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-1)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result (:result out)
+            component (get-in result [:data :components c-id])]
+        (t/is (true? (:deleted component)))
+        (t/is (some? (not-empty (:objects component))))))
+
+    ;; Re-run the file-gc task
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (false? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+
+    ;; Check that component is still there after file-gc task
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-1)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result (:result out)
+            component (get-in result [:data :components c-id])]
+        (t/is (true? (:deleted component)))
+        (t/is (some? (not-empty (:objects component))))))
+
+    ;; Now delete the last instance using deleted component
+    (update-file!
+     :file-id (:id file-2)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :del-obj
+       :page-id f2-page-id
+       :id s-id-2
+       :ignore-touched true}])
+
+    ;; Mark
+    (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file-1)])
+
+    ;; Now, we have deleted the usage of component if we pass file-gc,
+    ;; that component should be deleted
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+
+    ;; Check that component is properly removed
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-1)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result     (:result out)
+            components (get-in result [:data :components])]
+        (t/is (not (contains? components c-id)))))))
+
+
+
+
+(defn add-file-media-object
+  [& {:keys [profile-id file-id]}]
+  (let [mfile  {:filename "sample.jpg"
+                :path (th/tempfile "backend_tests/test_files/sample.jpg")
+                :mtype "image/jpeg"
+                :size 312043}
+        params {::th/type :upload-file-media-object
+                ::rpc/profile-id profile-id
+                :file-id file-id
+                :is-local true
+                :name "testfile"
+                :content mfile}
+        out    (th/command! params)]
+
+    ;; (th/print-result! out)
+    (t/is (nil? (:error out)))
+    (:result out)))
+
+
+
+(t/deftest file-gc-with-media-assets-and-absorb-library
+  (let [storage (:app.storage/storage th/*system*)
+        profile (th/create-profile* 1)
+
+        file-1  (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared true})
+
+        file-2  (th/create-file* 2 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
+
+        fmedia  (add-file-media-object :profile-id (:id profile) :file-id (:id file-1))
+
+
+        rel     (th/link-file-to-library*
+                 {:file-id (:id file-2)
+                  :library-id (:id file-1)})
+
+        s-id-1  (uuid/random)
+        s-id-2  (uuid/random)
+        c-id    (uuid/random)
+
+        f1-page-id (first (get-in file-1 [:data :pages]))
+        f2-page-id (first (get-in file-2 [:data :pages]))
+
+        fills
+        [{:fill-image
+          {:id (:id fmedia)
+           :name "test"
+           :width 200
+           :height 200}}]]
+
+    ;; Update file library inserting new component
+    (update-file!
+     :file-id (:id file-1)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id f1-page-id
+       :id s-id-1
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-1
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :fills fills
+              :main-instance true
+              :component-root true
+              :component-file (:id file-1)
+              :component-id c-id})}
+      {:type :add-component
+       :path ""
+       :name "Board"
+       :main-instance-id s-id-1
+       :main-instance-page f1-page-id
+       :id c-id
+       :anotation nil}])
+
+    ;; Instanciate a component in a different file
+    (update-file!
+     :file-id (:id file-2)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id f2-page-id
+       :id s-id-2
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id s-id-2
+              :name "Board"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :frame
+              :fills fills
+              :main-instance false
+              :component-root true
+              :component-file (:id file-1)
+              :component-id c-id})}])
+
+    ;; Check that file media object references are present for both objects
+    ;; the original one and the instance.
+    (let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
+      (t/is (= 2 (count rows)))
+      (t/is (= (:id file-1) (:file-id (get rows 0))))
+      (t/is (= (:id file-2) (:file-id (get rows 1))))
+      (t/is (every? (comp nil? :deleted-at) rows)))
+
+    ;; Check if the underlying media reference on shape is different
+    ;; from the instantiation
+    (let [data {::th/type :get-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-2)}
+          out  (th/command! data)]
+
+      (t/is (th/success? out))
+      (let [result (:result out)
+            fill   (get-in result [:data :pages-index f2-page-id :objects s-id-2 :fills 0 :fill-image])]
+        (t/is (some? fill))
+        (t/is (not= (:id fill) (:id fmedia)))))
+
+    ;; Run the file-gc on file and library
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+
+    ;; Now proceed to delete file and absorb it
+    (let [data {::th/type :delete-file
+                ::rpc/profile-id (:id profile)
+                :id (:id file-1)}
+          out  (th/command! data)]
+      (t/is (th/success? out)))
+
+    (th/run-task! :delete-object
+                  {:object :file
+                   :deleted-at (dt/now)
+                   :id (:id file-1)})
+
+    ;; Check that file media object references are marked all for deletion
+    (let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
+      ;; (pp/pprint rows)
+      (t/is (= 2 (count rows)))
+
+      (t/is (= (:id file-1) (:file-id (get rows 0))))
+      (t/is (some? (:deleted-at (get rows 0))))
+
+      (t/is (= (:id file-2) (:file-id (get rows 1))))
+      (t/is (nil? (:deleted-at (get rows 1)))))
+
+    (th/run-task! :objects-gc
+                  {:min-age 0})
+
+    (let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
+      (t/is (= 1 (count rows)))
+
+      (t/is (= (:id file-2) (:file-id (get rows 0))))
+      (t/is (nil? (:deleted-at (get rows 0)))))))

backend/test/backend_tests/rpc_management_test.clj

@@ -6,6 +6,7 @@
 
 (ns backend-tests.rpc-management-test
   (:require
+   [app.binfile.common :as bfc]
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
    [app.common.types.shape :as cts]
@@ -82,7 +83,6 @@
       ;; Check that result is correct
       (t/is (nil? (:error out)))
       (let [result (:result out)]
-
         ;; Check that the returned result is a file but has different id
         ;; and different name.
         (t/is (= "file 1 (copy)" (:name result)))

backend/test/backend_tests/rpc_media_test.clj

@@ -10,6 +10,7 @@
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.storage :as sto]
+   [app.util.time :as dt]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
    [datoteka.fs :as fs]))
@@ -245,3 +246,35 @@
       (t/is (= "image/jpeg" (:mtype result)))
       (t/is (uuid? (:media-id result)))
       (t/is (uuid? (:thumbnail-id result))))))
+
+
+(t/deftest media-object-upload-command-when-file-is-deleted
+  (let [prof   (th/create-profile* 1)
+        proj   (th/create-project* 1 {:profile-id (:id prof)
+                                      :team-id (:default-team-id prof)})
+        file   (th/create-file* 1 {:profile-id (:id prof)
+                                   :project-id (:default-project-id prof)
+                                   :is-shared false})
+
+        _      (th/db-update! :file
+                              {:deleted-at (dt/now)}
+                              {:id (:id file)})
+
+        mfile  {:filename "sample.jpg"
+                :path (th/tempfile "backend_tests/test_files/sample.jpg")
+                :mtype "image/jpeg"
+                :size 312043}
+
+        params {::th/type :upload-file-media-object
+                ::rpc/profile-id (:id prof)
+                :file-id (:id file)
+                :is-local true
+                :name "testfile"
+                :content mfile}
+
+        out    (th/command! params)]
+
+    (let [error      (:error out)
+          error-data (ex-data error)]
+      (t/is (th/ex-info? error))
+      (t/is (= (:type error-data) :not-found)))))

backend/test/backend_tests/rpc_profile_test.clj

@@ -203,7 +203,24 @@
             edata (ex-data error)]
         (t/is (th/ex-info? error))
         (t/is (= (:type edata) :validation))
-        (t/is (= (:code edata) :owner-teams-with-people))))))
+        (t/is (= (:code edata) :owner-teams-with-people)))
+
+      (let [params {::th/type :delete-team
+                    ::rpc/profile-id (:id prof1)
+                    :id (:id team1)}
+            out    (th/command! params)]
+      ;; (th/print-result! out)
+
+        (let [team (th/db-get :team {:id (:id team1)} {::db/remove-deleted false})]
+          (t/is (dt/instant? (:deleted-at team)))))
+
+    ;; Request profile to be deleted
+      (let [params {::th/type :delete-profile
+                    ::rpc/profile-id (:id prof1)}
+            out    (th/command! params)]
+      ;; (th/print-result! out)
+        (t/is (nil? (:result out)))
+        (t/is (nil? (:error out)))))))
 
 (t/deftest profile-deletion-3
   (let [prof1 (th/create-profile* 1)
@@ -291,7 +308,7 @@
           out    (th/command! params)]
       ;; (th/print-result! out)
 
-      (t/is (= {} (:result out)))
+      (t/is (nil? (:result out)))
       (t/is (nil? (:error out))))
 
     ;; query files after profile soft deletion
@@ -336,7 +353,7 @@
                   ::rpc/profile-id (:id prof1)}
           out    (th/command! params)]
       ;; (th/print-result! out)
-      (t/is (= {} (:result out)))
+      (t/is (nil? (:result out)))
       (t/is (nil? (:error out))))
 
     (th/run-pending-tasks!)

backend/test/backend_tests/rpc_team_test.clj

@@ -37,18 +37,17 @@
                     :role :editor}]
 
       ;; invite external user without complaints
-      (let [data       (assoc data :emails ["foo@bar.com"])
-            out        (th/command! data)
+      (let [data        (assoc data :emails ["foo@bar.com"])
+            out         (th/command! data)
             ;; retrieve the value from the database and check its content
-            invitation (db/exec-one!
-                        th/*pool*
-                        ["select count(*) as num from team_invitation where team_id = ? and email_to = ?"
-                         (:team-id data) "foo@bar.com"])]
+            invitations (th/db-query :team-invitation
+                                     {:team-id (:team-id data)
+                                      :email-to "foo@bar.com"})]
 
         ;; (th/print-result! out)
         (t/is (th/success? out))
         (t/is (= 1 (:call-count (deref mock))))
-        (t/is (= 1 (:num invitation))))
+        (t/is (= 1 (count invitations))))
 
       ;; invite internal user without complaints
       (th/reset-mock! mock)
@@ -102,6 +101,105 @@
           (t/is (= :validation (:type edata)))
           (t/is (= :member-is-muted (:code edata))))))))
 
+(t/deftest create-team-invitations-with-request-access
+  (with-mocks [mock {:target 'app.email/send! :return nil}]
+    (let [profile1  (th/create-profile* 1 {:is-active true})
+          requester (th/create-profile* 2 {:is-active true :email "requester@example.com"})
+
+          team      (th/create-team* 1 {:profile-id (:id profile1)})
+          proj      (th/create-project* 1 {:profile-id (:id profile1)
+                                           :team-id (:id team)})
+          file      (th/create-file* 1 {:profile-id (:id profile1)
+                                        :project-id (:id proj)})]
+      (let [data {::th/type :create-team-access-request
+                  ::rpc/profile-id (:id requester)
+                  :file-id (:id file)}
+            out  (th/command! data)]
+        (t/is (th/success? out))
+        (t/is (= 1 (:call-count @mock))))
+
+      (th/reset-mock! mock)
+
+      (let [data {::th/type :create-team-invitations
+                  ::rpc/profile-id (:id profile1)
+                  :team-id (:id team)
+                  :role :editor
+                  :emails ["requester@example.com"]}
+            out  (th/command! data)]
+        (t/is (th/success? out))
+        (t/is (= 1 (:call-count @mock)))
+
+        ;; Check that request is properly removed
+        (let [requests (th/db-query :team-access-request
+                                    {:requester-id (:id requester)})]
+          (t/is (= 0 (count requests))))
+
+        (let [rows (th/db-query :team-profile-rel {:team-id (:id team)})]
+          (t/is (= 2 (count rows))))))))
+
+
+(t/deftest create-team-invitations-with-request-access-2
+  (with-mocks [mock {:target 'app.email/send! :return nil}]
+    (let [profile1   (th/create-profile* 1 {:is-active true})
+          requester  (th/create-profile* 2 {:is-active true
+                                            :email "requester@example.com"})
+
+          team       (th/create-team* 1 {:profile-id (:id profile1)})
+          proj       (th/create-project* 1 {:profile-id (:id profile1)
+                                            :team-id (:id team)})
+          file       (th/create-file* 1 {:profile-id (:id profile1)
+                                         :project-id (:id proj)})]
+
+      ;; Create the first access request
+      (let [data {::th/type :create-team-access-request
+                  ::rpc/profile-id (:id requester)
+                  :file-id (:id file)}
+            out  (th/command! data)]
+        (t/is (th/success? out))
+        (t/is (= 1 (:call-count @mock))))
+
+      (th/reset-mock! mock)
+
+      ;; Proceed to delete the requester user
+      (th/db-update! :profile
+                     {:deleted-at (dt/in-past "1h")}
+                     {:id (:id requester)})
+
+      ;; Create a new profile with the same email
+      (let [requester' (th/create-profile* 3 {:is-active true :email "requester@example.com"})]
+
+        ;; Create a request access with new requester
+        (let [data {::th/type :create-team-access-request
+                    ::rpc/profile-id (:id requester')
+                    :file-id (:id file)}
+              out  (th/command! data)]
+          (t/is (th/success? out))
+          (t/is (= 1 (:call-count @mock))))
+
+        (th/reset-mock! mock)
+
+        ;; Create an invitation for the requester email
+        (let [data {::th/type :create-team-invitations
+                    ::rpc/profile-id (:id profile1)
+                    :team-id (:id team)
+                    :role :editor
+                    :emails ["requester@example.com"]}
+              out  (th/command! data)]
+          (t/is (th/success? out))
+          (t/is (= 1 (:call-count @mock))))
+
+        ;; Check that request is properly removed
+        (let [requests (th/db-query :team-access-request
+                                    {:requester-id (:id requester')})]
+          (t/is (= 0 (count requests))))
+
+        (let [[r1 r2 :as rows] (th/db-query :team-profile-rel
+                                            {:team-id (:id team)}
+                                            {:order-by [:created-at]})]
+          (t/is (= 2 (count rows)))
+          (t/is (= (:profile-id r1) (:id profile1)))
+          (t/is (= (:profile-id r2) (:id requester'))))))))
+
 
 (t/deftest invitation-tokens
   (with-mocks [mock {:target 'app.email/send! :return nil}]
@@ -486,14 +584,12 @@
       ;; request success
       (let [out        (th/command! data)
             ;; retrieve the value from the database and check its content
-            request    (db/exec-one!
-                        th/*pool*
-                        ["select count(*) as num from team_access_request where team_id = ? and requester_id = ?"
-                         (:id team) (:id requester)])]
-
+            requests   (th/db-query :team-access-request
+                                    {:team-id (:id team)
+                                     :requester-id (:id requester)})]
         (t/is (th/success? out))
         (t/is (= 1 (:call-count @mock)))
-        (t/is (= 1 (:num request))))
+        (t/is (= 1 (count requests))))
 
       ;; request again fails
       (th/reset-mock! mock)
@@ -509,10 +605,10 @@
       ;; request again when is expired success
       (th/reset-mock! mock)
 
-      (db/exec-one!
-       th/*pool*
-       ["update team_access_request set valid_until = ? where team_id = ? and requester_id = ?"
-        (dt/in-past "1h") (:id team) (:id requester)])
+      (th/db-update! :team-access-request
+                     {:valid-until (dt/in-past "1h")}
+                     {:team-id (:id team)
+                      :requester-id (:id requester)})
 
       (t/is (th/success? (th/command! data)))
       (t/is (= 1 (:call-count @mock))))))

common/deps.edn

@@ -1,25 +1,25 @@
 {:deps
  {org.clojure/clojure {:mvn/version "1.12.0"}
-  org.clojure/data.json {:mvn/version "2.5.0"}
+  org.clojure/data.json {:mvn/version "2.5.1"}
   org.clojure/tools.cli {:mvn/version "1.1.230"}
   org.clojure/clojurescript {:mvn/version "1.11.132"}
   org.clojure/test.check {:mvn/version "1.1.1"}
   org.clojure/data.fressian {:mvn/version "1.1.0"}
 
   ;; Logging
-  org.apache.logging.log4j/log4j-api {:mvn/version "2.24.1"}
-  org.apache.logging.log4j/log4j-core {:mvn/version "2.24.1"}
-  org.apache.logging.log4j/log4j-web {:mvn/version "2.24.1"}
-  org.apache.logging.log4j/log4j-jul {:mvn/version "2.24.1"}
-  org.apache.logging.log4j/log4j-slf4j2-impl {:mvn/version "2.24.1"}
+  org.apache.logging.log4j/log4j-api {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-core {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-web {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-jul {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-slf4j2-impl {:mvn/version "2.24.3"}
   org.slf4j/slf4j-api {:mvn/version "2.0.16"}
   pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.32"}
 
   selmer/selmer {:mvn/version "1.12.61"}
   criterium/criterium {:mvn/version "0.4.6"}
 
-  metosin/jsonista {:mvn/version "0.3.11"}
-  metosin/malli {:mvn/version "0.16.4"}
+  metosin/jsonista {:mvn/version "0.3.13"}
+  metosin/malli {:mvn/version "0.17.0"}
 
   expound/expound {:mvn/version "0.9.0"}
   com.cognitect/transit-clj {:mvn/version "1.0.333"}
@@ -47,11 +47,10 @@
   org.la4j/la4j {:mvn/version "0.6.0"}
 
   ;; exception printing
-  fipp/fipp {:mvn/version "0.6.26"}
+  fipp/fipp {:mvn/version "0.6.27"}
+
+  me.flowthing/pp {:mvn/version "2024-11-13.77"}
 
-  io.github.eerohele/pp
-  {:git/tag "2024-09-09.69"
-   :git/sha "de4b20f"}
 
   io.aviso/pretty {:mvn/version "1.4.4"}
   environ/environ {:mvn/version "1.2.0"}}
@@ -60,7 +59,7 @@
  {:dev
   {:extra-deps
    {org.clojure/tools.namespace {:mvn/version "RELEASE"}
-    thheller/shadow-cljs {:mvn/version "2.28.18"}
+    thheller/shadow-cljs {:mvn/version "2.28.20"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
     com.bhauman/rebel-readline {:mvn/version "RELEASE"}
     criterium/criterium {:mvn/version "RELEASE"}
@@ -69,7 +68,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.5" :git/sha "2a21b7a"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.6" :git/sha "52cf7d6"}}
    :ns-default build}
 
   :test