🐳 Add missing package in the exporter Docker image (#7026)

penpot-exporter requires poppler-utils for exporting to a PDF, but it is missing.
Added the package to the Dockerfile in the RUN section where dependencies are
being installed.

Signed-off-by: Brandon Currell <brandon+git@currell.pw>

.circleci/config.yml

@@ -1,5 +1,53 @@
 version: 2.1
+
 jobs:
+  lint:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    steps:
+      - checkout
+
+      - run:
+          name: "fmt check"
+          working_directory: "."
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+
+      - run:
+          name: "lint clj common"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:common
+
+      - run:
+          name: "lint clj frontend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:frontend
+
+      - run:
+          name: "lint clj backend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:backend
+
+      - run:
+          name: "lint clj exporter"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:exporter
+
+      - run:
+          name: "lint clj library"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:library
+
   test-common:
     docker:
       - image: penpotapp/devenv:latest
@@ -17,15 +65,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "common/deps.edn"}}
-
-      - run:
-          name: "fmt check & linter"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
+            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
 
       - run:
           name: "JVM tests"
@@ -37,13 +77,16 @@ jobs:
           name: "NODE tests"
           working_directory: "./common"
           command: |
+            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-          key: v1-dependencies-{{ checksum "common/deps.edn"}}
-
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
 
   test-frontend:
     docker:
@@ -62,37 +105,107 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
       - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
+          name: "install dependencies"
+          working_directory: "./frontend"
+          # We install playwright here because the dependent tasks
+          # uses the same cache as this task so we prepopulate it
           command: |
             yarn install
-            yarn run lint:clj
+            yarn run playwright install chromium
 
       - run:
-          name: "fmt check & linter"
+          name: "lint scss on frontend"
           working_directory: "./frontend"
           command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run fmt:js:check
             yarn run lint:scss
-            yarn run lint:clj
 
       - run:
           name: "unit tests"
           working_directory: "./frontend"
           command: |
-            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
+  test-library:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: Install dependencies and build
+          working_directory: "./library"
+          command: |
+            yarn install
+
+      - run:
+          name: Build and Test
+          working_directory: "./library"
+          command: |
+            ./scripts/build
+            yarn run test
+
+  test-components:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g -Xms2g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: Install dependencies
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run playwright install chromium
+
+      - run:
+          name: Build Storybook
+          working_directory: "./frontend"
+          command: yarn run build:storybook
+
+      - run:
+          name: Serve Storybook and run tests
+          working_directory: "./frontend"
+          command: |
+            npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
+              "npx http-server storybook-static --port 6006 --silent" \
+              "npx wait-on tcp:6006 && yarn test:storybook"
 
   test-integration:
     docker:
@@ -111,7 +224,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
       - run:
           name: "integration tests"
@@ -121,7 +234,7 @@ jobs:
             yarn run build:app:assets
             yarn run build:app
             yarn run build:app:libs
-            yarn run playwright install --with-deps chromium
+            yarn run playwright install chromium
             yarn run test:e2e -x --workers=4
 
   test-backend:
@@ -148,21 +261,6 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "backend/deps.edn" }}
 
-      - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run lint:clj
-
-      - run:
-          name: "fmt check & linter"
-          working_directory: "./backend"
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
-
       - run:
           name: "tests"
           working_directory: "./backend"
@@ -178,43 +276,67 @@ jobs:
       - save_cache:
           paths:
             - ~/.m2
+            - ~/.gitlibs
           key: v1-dependencies-{{ checksum "backend/deps.edn" }}
 
-
-  test-exporter:
+  test-render-wasm:
     docker:
       - image: penpotapp/devenv:latest
 
     working_directory: ~/repo
     resource_class: medium+
-
     environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
 
     steps:
       - checkout
 
       - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
+          name: "fmt check"
+          working_directory: "./render-wasm"
           command: |
-            yarn install
-            yarn run lint:clj
+            cargo fmt --check
 
       - run:
-          name: "fmt check & linter"
-          working_directory: "./exporter"
+          name: "lint"
+          working_directory: "./render-wasm"
           command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
+            ./lint
+
+      - run:
+          name: "cargo tests"
+          working_directory: "./render-wasm"
+          command: |
+            ./test
 
 workflows:
   penpot:
     jobs:
-      - test-frontend
-      - test-integration
-      - test-backend
-      - test-common
-      - test-exporter
+      - lint
+      - test-frontend:
+          requires:
+            - lint: success
+
+      - test-library:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-components:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-integration:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-backend:
+          requires:
+            - lint: success
+
+      - test-common:
+          requires:
+            - lint: success
+
+      - test-render-wasm

.clj-kondo/config.edn

@@ -58,6 +58,12 @@
   :redundant-do
   {:level :off}
 
+  :redundant-ignore
+  {:level :off}
+
+  :redundant-nested-call
+  {:level :off}
+
   :earmuffed-var-not-dynamic
   {:level :off}
 

.editorconfig

@@ -11,3 +11,9 @@ end_of_line = lf
 insert_final_newline = true
 
 trim_trailing_whitespace = true
+
+[*.{rs}]
+indent_size = 4
+indent_style = space
+
+end_of_line = lf

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,29 +1,19 @@
-<!--
+### Related Ticket
 
-Some key notes before you open a PR:
+<!-- Reference the related GitHub/Taiga ticket. -->
 
- 1. Select which branch should this PR be merged in? By default, you should always merge to the develop branch.
- 2. PR name follows [convention](http://karma-runner.github.io/4.0/dev/git-commit-msg.html)
- 3. All tests pass locally, UI and Unit tests
- 4. All business logic and validations must be on the server-side
- 5. Update necessary Documentation
- 6. Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes
+### Summary
 
+### Steps to reproduce 
 
-Also, if you're new here
+### Checklist
 
-- Contribution Guide => https://github.com/uxbox/uxbox/blob/develop/CONTRIBUTING.md
+- [ ] Choose the correct target branch; use `develop` by default.
+- [ ] Provide a brief summary of the changes introduced.
+- [ ] Add a detailed explanation of how to reproduce the issue and/or verify the fix, if applicable.
+- [ ] Include screenshots or videos, if applicable.
+- [ ] Add or modify existing integration tests in case of bugs or new features, if applicable.
+- [ ] Check CI passes successfully.
+- [ ] Update the `CHANGES.md` file, referencing the related GitHub issue, if applicable.
 
--->
-
-> Please provide enough information so that others can review your pull request:
-
-<!-- You can skip this if you're fixing a typo or updating existing documentation -->
-
-> Explain the **details** for making this change. What existing problem does the pull request solve?
-
-<!-- Example: When "Adding a function to do X", explain why it is necessary to have a way to do X. -->
-
-> Screenshots/GIFs
-
-<!-- Add images/recordings to better visualize the change: expected/current behviour -->
+<!-- For more details, check the contribution guidelines: https://github.com/penpot/penpot/blob/develop/CONTRIBUTING.md -->

.github/workflows/build-bundles.yml

@@ -0,0 +1,129 @@
+name: Build and Upload Penpot Bundles non-prod
+
+on:
+  # Create bundler for every tag
+  push:
+    tags:
+      - '**' # Pattern matched against refs/tags
+  # Create bundler every hour between 5:00 and 20:00 on working days
+  schedule:
+    - cron: '0 5-20 * * 1-5'
+  # Create bundler from manual action
+  workflow_dispatch:
+    inputs:
+      zip_mode:
+        # zip_mode defines how the build artifacts are packaged:
+        # - 'individual': creates one ZIP file per component (frontend, backend, exporter)
+        # - 'all': creates a single ZIP containing all components
+        # - null: for the rest of cases (non-manual events)
+        description: 'Bundle packaging mode'
+        required: false
+        default: 'individual'
+        type: choice
+        options:
+          - individual
+          - all
+
+jobs:
+  build-bundles:
+    name: Build and Upload Penpot Bundles
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Extract somer useful variables
+        id: vars
+        run: |
+          echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "gh_branch=${{ github.base_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      # Set up Docker Buildx for multi-arch build
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run manage.sh build-bundle from host
+        run: ./manage.sh build-bundle
+
+      - name: Prepare directories for zipping
+        run: |
+          mkdir zips
+          mv bundles penpot
+
+      - name: Create zip bundles for zip_mode == 'all'
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        run: |
+          echo "📦 Packaging Penpot 'all' bundles..."
+          zip -r zips/penpot-all-bundles.zip penpot
+
+      - name: Create zip bundles for zip_mode != 'all'
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        run: |
+          echo "📦 Packaging Penpot 'individual' bundles..."
+          zip -r zips/penpot-frontend.zip penpot/frontend
+          zip -r zips/penpot-backend.zip penpot/backend
+          zip -r zips/penpot-exporter.zip penpot/exporter
+
+      - name: Upload unified 'all' bundle
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: penpot-all-bundles
+          path: zips/penpot-all-bundles.zip
+
+      - name: Upload individual bundles
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: penpot-individual-bundles
+          path: |
+            zips/penpot-frontend.zip
+            zips/penpot-backend.zip
+            zips/penpot-exporter.zip
+
+      - name: Upload unified 'all' bundle to S3
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        run: |
+          aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.gh_branch}}.zip
+          aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.commit_hash }}.zip
+
+      - name: Upload 'individual' bundles to S3
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        run: |
+            for name in penpot-frontend penpot-backend penpot-exporter; do
+              aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.gh_branch }}-latest.zip
+              aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.commit_hash }}.zip
+            done
+
+      - name: Notify Mattermost about automatic bundles
+        if: github.event_name == 'pull_request'
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          TEXT: |
+            📦 *Penpot bundle automatically generated*
+            📄 PR: ${{ github.event.pull_request.title }}
+            🔁 From: \`${{ github.head_ref }}\` to \`{{ github.base_ref }}\`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Notify Mattermost about manual bundles
+        if: github.event_name == 'workflow_dispatch'
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          TEXT: |
+            📦 *Penpot bundle manually generated*
+            📄 Triggered from branch: `${{ github.ref_name}}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Print artifact summary URL
+        run: |
+          echo "📦 Artifacts available at:"
+          echo "🔗 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/commit-checker.yml

@@ -0,0 +1,50 @@
+name: 'Commit Message Check'
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+  push:
+    branches:
+      - main
+      - develop
+      - staging
+
+jobs:
+  check-commit-message:
+    name: Check Commit Message
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Commit Type
+        uses: gsactions/commit-message-checker@v2
+        with:
+          pattern: '^(Merge|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
+          flags: 'gm'
+          error: 'Commit should match CONTRIBUTING.md guideline'
+          checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
+          accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
+
+      # - name: Check Line Length
+      #   uses: gsactions/commit-message-checker@v2
+      #   with:
+      #     pattern: '^[^#].{74}'
+      #     error: 'The maximum line length of 74 characters is exceeded.'
+      #     excludeDescription: 'true' # optional: this excludes the description body of a pull request
+      #     excludeTitle: 'true' # optional: this excludes the title of a pull request
+      #     checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
+      #     accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is ue
+      # - name: Check for Resolves / Fixes
+      #   uses: gsactions/commit-message-checker@v2
+      #   with:
+      #     pattern: '^.+(Resolves|Fixes): \#[0-9]+$'
+      #     error: 'You need at least one "Resolves|Fixes: #<issue number>" line.'
+
+

.gitignore

@@ -30,6 +30,7 @@
 /*.zip
 /.clj-kondo/.cache
 /_dump
+/notes
 /backend/*.md
 /backend/*.sql
 /backend/*.txt
@@ -40,6 +41,7 @@
 /backend/resources/public/assets
 /backend/resources/public/media
 /backend/target/
+/backend/experiments
 /bundle*
 /cd.md
 /clj-profiler/
@@ -50,9 +52,6 @@
 /exporter/target
 /frontend/.storybook/preview-body.html
 /frontend/.storybook/preview-head.html
-/frontend/cypress/fixtures/validuser.json
-/frontend/cypress/videos/*/
-/frontend/cypress/videos/*/
 /frontend/dist/
 /frontend/npm-debug.log
 /frontend/out/
@@ -68,6 +67,10 @@
 /vendor/**/target
 /vendor/svgclean/bundle*.js
 /web
+/library/target/
+/library/*.zip
+/external
+
 clj-profiler/
 node_modules
 /test-results/

.nvmrc

@@ -1 +1 @@
-v20.11.1
+v22.13.1

CHANGES.md

@@ -1,5 +1,349 @@
 # CHANGELOG
 
+## 2.8.1
+
+### :bug: Bugs fixed
+
+- Fix unexpected exception on processing old texts [Github #6889](https://github.com/penpot/penpot/pull/6889)
+- Fix error on inspect tab when selecting multiple shapes [Taiga #11655](https://tree.taiga.io/project/penpot/issue/11655)
+- Fix missing package for the penport_exporter Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
+
+
+## 2.8.0
+
+### :rocket: Epics and highlights
+
+### :boom: Breaking changes & Deprecations
+
+**Penpot Library**
+
+The initial prototype is completly reworked for provide a more consistent API
+and to have proper validation and params decoding. All the details can be found
+on [its own changelog](library/CHANGES.md)
+
+**Penpot migrate from Redis to Valkey**
+
+As [Valkey](https://valkey.io/) is an opne-souce fork of [Redis](https://redis.io/)
+version 7.2.4, this version of Penpot will be compatible with Redis but may diverge
+in future versions. Therefore, **migration from Redis to ValKey is recommended for all
+on-premises instances** that want to keep up to date.
+
+### :heart: Community contributions (Thank you!)
+- Add Serbian language [GitHub #5002](https://github.com/penpot/penpot/issues/5002) by [crnobog69](https://github.com/crnobog69)
+
+### :sparkles: New features & Enhancements
+
+- Optimize profile setup flow for better user experience [Taiga #10028](https://tree.taiga.io/project/penpot/us/10028)
+- Rewrite path shape data PathData encoding [Taiga #8542](https://tree.taiga.io/project/penpot/us/8542?milestone=441308)
+- Update base image for Docker Backend and Exporter to Ubuntu 24.04
+- Update base image for Docker Frontend to Nginx 1.28.0
+- Allow multi file token import [Github #27](https://github.com/tokens-studio/penpot/issues/27)
+- Create `input*` wrapper component, and `label*`, `input-field*` and `hint-message*` components [Taiga #10713](https://tree.taiga.io/project/penpot/us/10713)
+- Deselect layers (and path nodes) with Ctrl+Shift+Drag [Github #2509](https://github.com/penpot/penpot/issues/2509)
+- Copy to SVG from contextual menu [Github #838](https://github.com/penpot/penpot/issues/838)
+- Add styles for Inkeep Chat at workspace [Taiga #10708](https://tree.taiga.io/project/penpot/us/10708)
+- Add configuration for air gapped installations with Docker
+- Support system color scheme [Github #5030](https://github.com/penpot/penpot/issues/5030)
+- Persist ruler visibility across files and reloads [GitHub #4586](https://github.com/penpot/penpot/issues/4586)
+- Update google fonts (at 2025/05/19) [Taiga 10792](https://tree.taiga.io/project/penpot/us/10792)
+- Add tooltip component to DS [Taiga 9220](https://tree.taiga.io/project/penpot/us/9220)
+- Allow multi file token export [Taiga #10144](https://tree.taiga.io/project/penpot/us/10144)
+- Fix problem when double click on hidden shapes [Taiga #11314](https://tree.taiga.io/project/penpot/issue/11314)
+
+### :bug: Bugs fixed
+
+- Fix getCurrentUser for plugins api [Taiga #11057](https://tree.taiga.io/project/penpot/issue/11057)
+- Fix spacing / sizes of different elements in the measurements section of the design tab [Taiga #11076](https://tree.taiga.io/project/penpot/issue/11076)
+- Fix selection of short paths [Github #4472](https://github.com/penpot/penpot/issues/4472)
+- Fix element positioning on the right side to adjust to grid [#11073](https://tree.taiga.io/project/penpot/issue/11073)
+- Fix palette is over sidebar [#11160](https://tree.taiga.io/project/penpot/issue/11160)
+- Fix font size input not displaying "mixed" when multiple texts are selected [Taiga #11177](https://tree.taiga.io/project/penpot/issue/11177)
+- Misalignments at Create account [Taiga #11315](https://tree.taiga.io/project/penpot/issue/11315)
+- Fix issue with importing files where flex/grid is used [Taiga #11334](https://tree.taiga.io/project/penpot/issue/11334)
+- Fix wrong color in the export progress bar [Taiga #11299](https://tree.taiga.io/project/penpot/issue/11299)
+- Fix right sidebar width overflow on long layer names [Taiga #11212](https://tree.taiga.io/project/penpot/issue/11212)
+- Fix comment icon fill [Taiga #11388](https://tree.taiga.io/project/penpot/issue/11388)
+- Fix gap on radio-buttons component [Taiga #11360](https://tree.taiga.io/project/penpot/issue/11360)
+- Fix button width [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
+- Fix mixed letter spacing and line height [Taiga #11178](https://tree.taiga.io/project/penpot/issue/11178)
+- Fix snap nodes shortcut [Taiga #11054](https://tree.taiga.io/project/penpot/issue/11054)
+- Fix changing a text property in a text layer does not unapply the previously applied token in the same property [Taiga #11337](https://tree.taiga.io/project/penpot/issue/11337)
+- Fix shortcut error pressing G+W from the View Mode [Taiga #11061](https://tree.taiga.io/project/penpot/issue/11061)
+- Fix entering long project name [Taiga #11417](https://tree.taiga.io/project/penpot/issue/11417)
+- Fix slow color picker [Taiga #11019](https://tree.taiga.io/project/penpot/issue/11019)
+- Fix tooltip position after click [Taiga #11405](https://tree.taiga.io/project/penpot/issue/11405)
+- Fix incorrect media translation on paste text with fill images [Github #6845](https://github.com/penpot/penpot/pull/6845)
+
+## 2.7.2
+
+### :bug: Bugs fixed
+
+- Update plugins runtime [Github #6604](https://github.com/penpot/penpot/pull/6604)
+- Backport from develop a minor fix that enables import of files
+  generated by penpot library [Github #6614](https://github.com/penpot/penpot/pull/6614)
+- Fix copy in error message [GitHub #6615](https://github.com/penpot/penpot/pull/6615)
+- Fix url on invitation link [Taiga #11284](https://tree.taiga.io/project/penpot/issue/11284)
+
+
+## 2.7.1
+
+### :bug: Bugs fixed
+
+- Fix incorrect handling of strokes with images on importing files
+- Fix tokens disappearing after manual additions [Taiga #11063](https://tree.taiga.io/project/penpot/issue/11063)
+
+
+## 2.7.0
+
+### :rocket: Epics and highlights
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+- Design improvements to the Invitations page with an empty state [GitHub #2608](https://github.com/penpot/penpot/issues/2608) by [@iprithvitharun](https://github.com/iprithvitharun)
+
+### :sparkles: New features
+
+- Update board presets with a newer devices [Taiga #10610](https://tree.taiga.io/project/penpot/us/10610)
+- Propagate "sharing a prototype" to editors and viewers [Taiga #8853](https://tree.taiga.io/project/penpot/us/8853)
+- Design improvements to the Invitations page with an empty state [Taiga #4554](https://tree.taiga.io/project/penpot/us/4554)
+- Duplicate token sets [Taiga #10694](https://tree.taiga.io/project/penpot/issue/10694)
+- Add set selection in create Token themes flow [Taiga #10746](https://tree.taiga.io/project/penpot/issue/10746)
+- Display indicator on not active sets [Taiga #10668](https://tree.taiga.io/project/penpot/issue/10668)
+
+### :bug: Bugs fixed
+
+- Fix "at" icon to match all icons on app [Taiga #11136](https://tree.taiga.io/project/penpot/issue/11136)
+- Fix problem in viewer with the back button [Taiga #10907](https://tree.taiga.io/project/penpot/issue/10907)
+- Fix resize bar background on tokens panel [Taiga #10811](https://tree.taiga.io/project/penpot/issue/10811)
+- Fix shortcut for history version panel [Taiga #11006](https://tree.taiga.io/project/penpot/issue/11006)
+- Fix positioning of comment drafts when near the right / bottom edges of viewport [Taiga #10534](https://tree.taiga.io/project/penpot/issue/10534)
+- Fix path having a wrong selrect [Taiga #10257](https://tree.taiga.io/project/penpot/issue/10257)
+- Fix SVG `stroke-linecap` property when importing SVGs [Taiga #9489](https://tree.taiga.io/project/penpot/issue/9489)
+- Fix position problems cutting-pasting a component [Taiga #10677](https://tree.taiga.io/project/penpot/issue/10677)
+- Fix design tab has a horizontal scroll [Taiga #10660](https://tree.taiga.io/project/penpot/issue/10660)
+- Fix long file names being clipped when longer than allowed length [Taiga #10662](https://tree.taiga.io/project/penpot/issue/10662)
+- Fix problem with error detail in toast [Taiga #10519](https://tree.taiga.io/project/penpot/issue/10519)
+- Fix view mode error when an external user tries to export something from a prototype using a shared link [Taiga #10251](https://tree.taiga.io/project/penpot/issue/10251)
+- Fix merge path nodes with only one node selected [Taiga #9626](https://tree.taiga.io/project/penpot/issue/9626)
+- Fix problem with import errors [Taiga #10040](https://tree.taiga.io/project/penpot/issue/10040)
+- Fix color gradient on texts [Taiga Issue #7488](https://tree.taiga.io/project/penpot/issue/7488)
+- Add support for self mentions [Taiga #10809](https://tree.taiga.io/project/penpot/issue/10809)
+- Fix team info settings alignment [Taiga #10869](https://tree.taiga.io/project/penpot/issue/10869)
+- Fix left sidebar horizontal scroll on nested layers [Taiga #10791](https://tree.taiga.io/project/penpot/issue/10791)
+- Improve error message details importing tokens [Taiga Issue #10772](https://tree.taiga.io/project/penpot/issue/10772)
+- Fix no selected set after Drag & Drop [Github #71](https://github.com/tokens-studio/penpot/issues/71)
+- Styledictionary v5 Update [Github #6283](https://github.com/penpot/penpot/pull/6283)
+- Fix Rename a set throws an internal error [Github #78](https://github.com/tokens-studio/penpot/issues/78)
+- Fix Out of Sync Token Value & Color Picker [Github #102](https://github.com/tokens-studio/penpot/issues/102)
+- Fix Color should preserve color space [Github #69](https://github.com/tokens-studio/penpot/issues/69)
+- Fix cannot rename Design Token Sets when group of same name exists [Taiga Issue #10773](https://tree.taiga.io/project/penpot/issue/10773)
+- Fix problem when duplicating grid layout [Github #6391](https://github.com/penpot/penpot/issues/6391)
+- Fix issue that makes workspace shortcuts stop working [Taiga #11062](https://tree.taiga.io/project/penpot/issue/11062)
+- Fix problem while syncing library colors and typographies [Taiga #11068](https://tree.taiga.io/project/penpot/issue/11068)
+- Fix problem with path edition of shapes [Taiga #9496](https://tree.taiga.io/project/penpot/issue/9496)
+- Fix exception on paste invalid html [Taiga #11047](https://tree.taiga.io/project/penpot/issue/11047)
+- Fix share button being displayed with no permissions [Taiga #11086](https://tree.taiga.io/project/penpot/issue/11086)
+- Fix inline styles in code tab [Taiga Issue #7583](https://tree.taiga.io/project/penpot/issue/7583)
+- Fix exception on returning openapi.json
+- Fix json encoding of TokensLib [Taiga #10994](https://tree.taiga.io/project/penpot/issue/10994)
+
+## 2.6.2
+
+### :bug: Bugs fixed
+
+- Increase the height of the right sidebar dropdowns [Taiga #10615](https://tree.taiga.io/project/penpot/issue/10615)
+- Fix scroll on token themes modal [Taiga #10745](https://tree.taiga.io/project/penpot/issue/10745)
+- Fix collapsing grouped sets in "edit Theme" closes the dialog [Taiga #10771](https://tree.taiga.io/project/penpot/issue/10771)
+- Fix unexpected exception on path editor on merge segments when undo stack is empty
+- Fix pricing CTA to be under a config flag [Taiga #10808](https://tree.taiga.io/project/penpot/issue/10808)
+- Fix allow moving a main component into another [Taiga #10818](https://tree.taiga.io/project/penpot/issue/10818)
+- Fix several issues with internal srepl helpers
+- Fix unexpected exception on template import from libraries
+- Fix incorrect uuid parsing from different parts of code
+- Fix update layout on component restore [Taiga #10637](https://tree.taiga.io/project/penpot/issue/10637)
+- Fix horizontal scroll in viewer [Github #6290](https://github.com/penpot/penpot/issues/6290)
+- Fix detach component in a particular case [Taiga #10837](https://tree.taiga.io/project/penpot/issue/10837)
+
+## 2.6.1
+
+### :bug: Bugs fixed
+
+- Fix webhooks not shown in list [Taiga #10763](https://tree.taiga.io/project/penpot/issue/10763)
+- Fix colorpicker scroll when dropdown displayed [Taiga #10696](https://tree.taiga.io/project/penpot/issue/10696)
+- Clean internal workspace state on exit or url changed [Taiga #10619](https://tree.taiga.io/project/penpot/issue/10619)
+
+## 2.6.0
+
+### :rocket: Epics and highlights
+
+- Design Tokens
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+### :sparkles: New features
+
+- [COMMENTS] "Mark All as Read" Functionality in Dashboard [Taiga #9235](https://tree.taiga.io/project/penpot/us/9235)
+- [COMMENTS] Bubble Groups [Taiga #9236](https://tree.taiga.io/project/penpot/us/9236)
+- Change templates carrousel [Taiga #9803](https://tree.taiga.io/project/penpot/us/9803)
+- [DESIGN TOKENS] Tokens CRUD. Types added: Color, Opacity, Border radius, Dimension, Sizing, Spacing, Rotation and Stroke.
+- [DESIGN TOKENS] Create references (alias) that point to other tokens.
+- [DESIGN TOKENS] Math operations in token values.
+- [DESIGN TOKENS] Sets CRUD, grouping and reordering.
+- [DESIGN TOKENS] Multidimensional Themes and Sets management.
+- [DESIGN TOKENS] Apply/Remove tokens to/from elements from the Tokens tab.
+- [DESIGN TOKENS] Integration with components.
+- [DESIGN TOKENS] Import and export tokens from a JSON file.
+- [DESIGN TOKENS] Apply Themes and Sets at document level.
+- Add more descriptive tooltip to boards for first time users [Taiga #9426](https://tree.taiga.io/project/penpot/us/9426)
+- First State of a Project Changes Consolidation [Taia #10605](https://tree.taiga.io/project/penpot/us/10605)
+
+### :bug: Bugs fixed
+
+- Fix opacity in frame containers [Github #5858](https://github.com/penpot/penpot/pull/5858)
+- Avoid resizing on click [Taiga #10213](https://tree.taiga.io/project/penpot/issue/10213)
+- Hide horizontal scroll from dashboard sidebar [Taiga #10422](https://tree.taiga.io/project/penpot/issue/10422)
+- Fix cut and paste a copy a cmponent inside its parent [Taiga #10365](https://tree.taiga.io/project/penpot/us/10365)
+- Fix duplicate page with component over frame [Taiga #8151](https://tree.taiga.io/project/penpot/issue/8151) and [Taiga #9698](https://tree.taiga.io/project/penpot/issue/9698)
+- The plugin list in the navigation menu lacks scrolling, some plugins are not visible when a large number are installed [Taiga #9360](https://tree.taiga.io/project/penpot/us/9360)
+- Fix hidden toolbar click event still available [Taiga #10437](https://tree.taiga.io/project/penpot/us/10437)
+- Fix hovering over templates [Taiga #10545](https://tree.taiga.io/project/penpot/issue/10545)
+- Fix problem with default shadows value in plugins [Plugins #191](https://github.com/penpot/penpot-plugins/issues/191)
+- Fix problem with constraints when creating group [Taiga #10455](https://tree.taiga.io/project/penpot/issue/10455)
+- Fix opening pen with shortcut multiple times breaks toolbar [Taiga #10566](https://tree.taiga.io/project/penpot/issue/10566)
+- Fix actions when workspace is visited first time [Taiga #10548](https://tree.taiga.io/project/penpot/issue/10548)
+- Chat icon overlaps "Show" button in carrousel section [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Fix assets name on inspect tab [Taiga #10630](https://tree.taiga.io/project/penpot/issue/10630)
+- Fix chat icon overlaps "Show" button in carrousel section [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Fix incorrect handling of background task result (now task rows are properly marked as completed)
+- Fix available size of resize handler [Taiga #10639](https://tree.taiga.io/project/penpot/issue/10639)
+- Internal error when install a plugin by penpothub - Try plugin [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Add character limitation to asset inputs [Taiga #10669](https://tree.taiga.io/project/penpot/issue/10669)
+- Fix Storybook link 'list of all available icons' wrong path [Taiga #10705](https://tree.taiga.io/project/penpot/issue/10705)
+
+
+## 2.5.4
+
+### :heart: Community contributions (Thank you!)
+
+- Add support for WEBP format on shape export [Github #6053](https://github.com/penpot/penpot/pull/6053) and [Github #6074](https://github.com/penpot/penpot/pull/6074)
+
+### :bug: Bugs fixed
+
+- Fix feature loading on workspace when opening a file in a background
+  tab [Taiga #10377](https://tree.taiga.io/project/penpot/issue/10377)
+- Fix minor inconsistencies on RPC `get-file-libraries` and `get-file`
+  methods (add missing team-id prop)
+- Fix problem with viewer role and inspect mode [Taiga #9751](https://tree.taiga.io/project/penpot/issue/9751)
+- Fix error when clicking on a comment at the viewer's sidebar [Taiga #10465](https://tree.taiga.io/project/penpot/issue/10465)
+
+## 2.5.3
+
+### :bug: Bugs fixed
+
+- Component sync issues with multiple tabs [Taiga #10471](https://tree.taiga.io/project/penpot/issue/10471)
+
+## 2.5.2
+
+### :sparkles: New features
+
+- When the workspace is empty, set default the board creation tool [Taiga #9425](https://tree.taiga.io/project/penpot/us/9425)
+
+### :bug: Bugs fixed
+
+- Fix scroll on storybook docs [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+- Navigate tracking event firing multiple times [Taiga #10415](https://tree.taiga.io/project/penpot/issue/10415)
+- Fix problem with selection colors [Taiga #10376](https://tree.taiga.io/project/penpot/issue/10376)
+- Fix scroll on storybook icons list [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+
+## 2.5.1
+
+### :sparkles: New features
+
+- Improve Nginx entryponit to get the resolvers dinamically by default
+
+## 2.5.0
+
+### :boom: Breaking changes & Deprecations
+
+Although this is not a breaking change, we believe it’s important to highlight it in this
+section:
+
+This release includes a fix for an internal bug in Penpot that caused incorrect handling
+of media assets (e.g., fill images). The issue has been resolved since version 2.4.3, so
+no new incorrect references will be generated. However, existing files may still contain
+incorrect references.
+
+To address this, we’ve provided a script to correct these references in existing files.
+
+While having incorrect references generally doesn’t result in visible issues, there are
+rare cases where it can cause problems. For example, if a component library (containing
+images) is deleted, and that library is being used in other files, running the FileGC task
+(responsible for freeing up space and performing logical deletions) could leave those
+files with broken references to the images.
+
+To execute script:
+
+```bash
+docker exec -ti <container-name-or-id> ./run.sh app.migrations.media-refs '{:max-jobs 1}'
+```
+
+If you have a big database and many cores available, you can reduce the time of processing
+all files by increasing paralelizacion changing the `max-jobs` value from 1 to N (where N
+is a number of cores)
+
+### :sparkles: New features
+
+- [GRADIENTS] New gradients UI with multi-stop support. [Taiga #3418](https://tree.taiga.io/project/penpot/epic/3418)
+- [GRADIENTS] Radial Gradient [Taiga #8768](https://tree.taiga.io/project/penpot/us/8768)
+- Shareable link pointing to an specific board. [Taiga #3219](https://tree.taiga.io/project/penpot/us/3219)
+- Copy styles in CSS [Taiga #9401](https://tree.taiga.io/project/penpot/us/9401)
+- Copy/paste shape styles (fills, strokes, shadows, etc..) [Taiga #8937](https://tree.taiga.io/project/penpot/us/8937)
+- Copy text content to clipboard [Taiga #9970](https://tree.taiga.io/project/penpot/us/9970?milestone=424203)
+- Resize board to fit content option [Taiga #4707](https://tree.taiga.io/project/penpot/us/4707)
+- Rename selected layer via Board name [Taiga #9430](https://tree.taiga.io/project/penpot/us/9430)
+- [COMMENTS] Mention Functionality with and Sidebar Filters [Taiga #9237](https://tree.taiga.io/project/penpot/us/9237)
+- [COMMENTS] Visual Changes in Comments [Taiga #9234](https://tree.taiga.io/project/penpot/us/9234)
+- [COMMENTS] Notifications in Backend, Profile Section, and Mention Email Notification [Taiga #9233](https://tree.taiga.io/project/penpot/us/9233)
+
+### :bug: Bugs fixed
+
+- Fix menu shadow color [Taiga #10102](https://tree.taiga.io/project/penpot/issue/10102)
+- Fix missing state refresh on notifications update [Taiga #10253](https://tree.taiga.io/project/penpot/issue/10253)
+- Fix icon visualization on select component [Taiga #8889](https://tree.taiga.io/project/penpot/issue/8889)
+- Fix typo on integration tests docs [Taiga #10112](https://tree.taiga.io/project/penpot/issue/10112)
+- Fix menu shadow color [Taiga #10102](https://tree.taiga.io/project/penpot/issue/10102)
+- Fix problem with alt key measures being stuck [Taiga #9348](https://tree.taiga.io/project/penpot/issue/9348)
+- Fix error when reseting stroke cap
+- Fix problem with strokes not refreshing in Safari [Taiga #9040](https://tree.taiga.io/project/penpot/issue/9040)
+- Fix problem with multiple color changes [Taiga #9631](https://tree.taiga.io/project/penpot/issue/9631)
+- Fix create new layers in a component copy [Taiga #10037](https://tree.taiga.io/project/penpot/issue/10037)
+- Fix problem in plugins with zoomIntoView [Plugins #189](https://github.com/penpot/penpot-plugins/issues/189)
+- Fix problem in plugins with renaming components [Taiga #10060](https://tree.taiga.io/project/penpot/issue/10060)
+- Added upload svg with images method [#5489](https://github.com/penpot/penpot/issues/5489)
+- Fix problem with root frame parent reference [Taiga #9437](https://tree.taiga.io/project/penpot/issue/9437)
+- Fix change flex direction using plugins API [Taiga #9407](https://tree.taiga.io/project/penpot/issue/9407)
+- Fix problem opening url when page-id didn't exist [Taiga #10157](https://tree.taiga.io/project/penpot/issue/10157)
+- Fix problem with onboarding to a team [Taiga #10143](https://tree.taiga.io/project/penpot/issue/10143)
+- Fix problem with grid layout crashing [Taiga #10127](https://tree.taiga.io/project/penpot/issue/10127)
+- Fix rename locked boards [Taiga #10174](https://tree.taiga.io/project/penpot/issue/10174)
+- Fix update-libraries dialog disappear when clicking outside [Taiga #10238](https://tree.taiga.io/project/penpot/issue/10238)
+- Fix incorrect handling of team access requests with deleted/recreated users
+- Fix incorect handling of profile settings related to invitation notifications [Taiga #10252](https://tree.taiga.io/project/penpot/issue/10252)
+
+## 2.4.3
+
+### :bug: Bugs fixed
+
+- Fix errors from editable select on measures menu [Taiga #9888](https://tree.taiga.io/project/penpot/issue/9888)
+- Fix exception on importing some templates from templates slider
+- Consolidate adding share button to workspace
+- Fix problem when pasting text [Taiga #9929](https://tree.taiga.io/project/penpot/issue/9929)
+- Fix incorrect media reference handling on component instantiation
+
 ## 2.4.2
 
 ### :bug: Bugs fixed
@@ -10,6 +354,7 @@
 - Fix missing methods reference on API Docs
 - Fix memory usage issue on file-gc asynchronous task (related to snapshots feature)
 
+
 ## 2.4.1
 
 ### :bug: Bugs fixed
@@ -17,6 +362,7 @@
 - Fix error when importing files with touched components [Taiga #9625](https://tree.taiga.io/project/penpot/issue/9625)
 - Fix problem when changing color libraries [Plugins #184](https://github.com/penpot/penpot-plugins/issues/184)
 
+
 ## 2.4.0
 
 ### :rocket: Epics and highlights
@@ -28,12 +374,12 @@
   (penpot). Because of that, the default NGINX listen port is now 8080 instead of 80, so
   you will have to modify your infrastructure to apply this change.
 
-- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because, 
-  starting with the next versions, Redis is no longer distributed under an open-source license. 
-  On-premise users are obviously free to upgrade to the version they are using or a more modern one. 
-  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume 
-  associated with the Redis container because the 7.2 storage format may not be compatible with what 
-  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate 
+- Redis 7.2 is explicitly pinned in our example docker-compose.yml file. This is done because,
+  starting with the next versions, Redis is no longer distributed under an open-source license.
+  On-premise users are obviously free to upgrade to the version they are using or a more modern one.
+  Keep in mind that if you were using a version other than 7.2, you may have to recreate the volume
+  associated with the Redis container because the 7.2 storage format may not be compatible with what
+  you already have stored on the volume, and Redis may not start. In the near future, we will evaluate
   whether to move to an open-source version of Redis (such as https://valkey.io/).
 
 ### :heart: Community contributions (Thank you!)

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Penpot's Code of Conduct
+
+Check it at: https://help.penpot.app/contributing-guide/coc/

CONTRIBUTING.md

@@ -1,62 +1,59 @@
 # Contributing Guide #
 
 Thank you for your interest in contributing to Penpot. This is a
-generic guide that details how to contribute to Penpot in a way that
-is efficient for everyone. If you want a specific documentation for
-different parts of the platform, please refer to `docs/` directory.
-
+generic guide that details how to contribute to the project in a way that
+is efficient for everyone. If you are looking for specific documentation on
+different parts of the platform, please refer to the `docs/` directory,
+or the rendered version at the [Help Center](https://help.penpot.app/).
 
 ## Reporting Bugs ##
 
 We are using [GitHub Issues](https://github.com/penpot/penpot/issues)
-for our public bugs. We keep a close eye on this and try to make it
+for our public bugs. We keep a close eye on them and try to make it
 clear when we have an internal fix in progress. Before filing a new
 task, try to make sure your problem doesn't already exist.
 
-If you found a bug, please report it, as far as possible with:
+If you found a bug, please report it, as far as possible, with:
 
 - a detailed explanation of steps to reproduce the error
-- a browser and the browser version used
-- a dev tools console exception stack trace (if it is available)
+- the browser and browser version used
+- a dev tools console exception stack trace (if available)
 
-If you found a bug that you consider better discuss in private (for
-example: security bugs), consider first send an email to
+If you found a bug which you think is better to discuss in private (for
+example, security bugs), consider first sending an email to
 `support@penpot.app`.
 
-**We don't have formal bug bounty program for security reports; this
-is an open source application and your contribution will be recognized
+**We don't have a formal bug bounty program for security reports; this
+is an open source application, and your contribution will be recognized
 in the changelog.**
 
 
-## Pull requests ##
+## Pull Requests ##
 
-If you want propose a change or bug fix with the Pull-Request system
-firstly you should carefully read the **DCO** section and format your
-commits accordingly.
+If you want to propose a change or bug fix via a pull request (PR),
+you should first carefully read the section **Developer's Certificate of
+Origin**. You must also format your code and commits according to the
+instructions below.
 
-If you intend to fix a bug it's fine to submit a pull request right
-away but we still recommend to file an issue detailing what you're
+If you intend to fix a bug, it's fine to submit a pull request right
+away, but we still recommend filing an issue detailing what you're
 fixing. This is helpful in case we don't accept that specific fix but
 want to keep track of the issue.
 
-If you want to implement or start working in a new feature, please
-open a **question** / **discussion** issue for it. No pull-request
-will be accepted without previous chat about the changes,
-independently if it is a new feature, already planned feature or small
-quick win.
+If you want to implement or start working on a new feature, please
+open a **question*- / **discussion*- issue for it. No PR
+will be accepted without a prior discussion about the changes,
+whether it is a new feature, an already planned one, or a quick win.
 
-If is going to be your first pull request, You can learn how from this
-free video series:
-
-https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github
-
-We will use the `easy fix` mark for tag for indicate issues that are
-easy for beginners.
+If it is your first PR, you can learn how to proceed from
+[this free video
+series](https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github)
 
+We use the `easy fix` tag to indicate issues that are appropriate for beginners.
 
 ## Commit Guidelines ##
 
-We have very precise rules over how our git commit messages can be formatted.
+We have very precise rules on how our git commit messages must be formatted.
 
 The commit message format is:
 
@@ -71,33 +68,37 @@ The commit message format is:
 Where type is:
 
 - :bug: `:bug:` a commit that fixes a bug
-- :sparkles: `:sparkles:` a commit that an improvement
-- :tada: `:tada:` a commit with new feature
+- :sparkles: `:sparkles:` a commit that adds an improvement
+- :tada: `:tada:` a commit with a new feature
 - :recycle: `:recycle:` a commit that introduces a refactor
 - :lipstick: `:lipstick:` a commit with cosmetic changes
-- :ambulance: `:ambulance:` a commit that fixes critical bug
+- :ambulance: `:ambulance:` a commit that fixes a critical bug
 - :books: `:books:` a commit that improves or adds documentation
-- :construction: `:construction:`: a wip commit
+- :construction: `:construction:` a WIP commit
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
-- :whale: `:whale:` a commit for docker related stuff
-- :rewind: `:rewind:` a commit that reverts changes
-- :paperclip: `:paperclip:` a commit with other not relevant changes
-- :arrow_up: `:arrow_up:` a commit with dependencies updates
+- :whale: `:whale:` a commit for Docker-related stuff
+- :paperclip: `:paperclip:` a commit with other non-relevant changes
+- :arrow_up: `:arrow_up:` a commit with dependency updates
+- :arrow_down: `:arrow_down:` a commit with dependency downgrades
+- :fire: `:fire:` a commit that removes files or code
+- :globe_with_meridians: `:globe_with_meridians:` a commit that adds or updates
+  translations
 
 More info:
+
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
 Each commit should have:
 
-- A concise subject using imperative mood.
-- The subject should have capitalized the first letter, without period
-  at the end and no larger than 65 characters.
+- A concise subject using the imperative mood.
+- The subject should capitalize the first letter, omit the period
+  at the end, and be no longer than 65 characters.
 - A blank line between the subject line and the body.
-- An entry on the CHANGES.md file if applicable, referencing the
-  github or taiga issue/user-story using the these same rules.
+- An entry in the CHANGES.md file if applicable, referencing the
+  GitHub or Taiga issue/user story using these same rules.
 
 Examples of good commit messages:
 
@@ -110,8 +111,30 @@ Examples of good commit messages:
 - `:ambulance: Fix critical bug on user registration process`
 - `:tada: Add new approach for user registration`
 
+## Formatting and Linting ##
 
-## Code of conduct ##
+You will want to make sure your code is formatted and linted before submitting
+a PR. We use [cljfmt](https://github.com/weavejester/cljfmt) and
+[clj-kondo](https://github.com/clj-kondo/clj-kondo) for this. After installing
+them on your system, you can run them with:
+
+```bash
+# Check formatting
+yarn fmt:clj:check
+
+# Check and fix formatting
+yarn fmt:clj
+
+# Run the linter
+yarn lint:clj
+```
+
+There are more choices in `package.json`.
+
+Ideally, you should run these commands as git pre-commit hooks. A convenient way
+of defining them is to use [Husky](https://typicode.github.io/husky/#/).
+
+## Code of Conduct ##
 
 As contributors and maintainers of this project, we pledge to respect
 all people who contribute through reporting issues, posting feature
@@ -131,11 +154,11 @@ unprofessional conduct.
 
 Project maintainers have the right and responsibility to remove, edit,
 or reject comments, commits, code, wiki edits, issues, and other
-contributions that are not aligned to this Code of Conduct. Project
+contributions that are not aligned with this Code of Conduct. Project
 maintainers who do not follow the Code of Conduct may be removed from
 the project team.
 
-This code of conduct applies both within project spaces and in public
+This Code of Conduct applies both within project spaces and in public
 spaces when an individual is representing the project or its
 community.
 
@@ -144,12 +167,11 @@ may be reported by opening an issue or contacting one or more of the
 project maintainers.
 
 This Code of Conduct is adapted from the Contributor Covenant, version
-1.1.0, available from http://contributor-covenant.org/version/1/1/0/
+1.1.0, available from [http://contributor-covenant.org/version/1/1/0/](http://contributor-covenant.org/version/1/1/0/)
 
+## Developer's Certificate of Origin (DCO)
 
-## Developer's Certificate of Origin (DCO) ##
-
-By submitting code you are agree and can certify the below:
+By submitting code you agree to and can certify the following:
 
     Developer's Certificate of Origin 1.1
 
@@ -177,13 +199,15 @@ By submitting code you are agree and can certify the below:
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
 
-Then, all your code patches (**documentation are excluded**) should
+Then, all your code patches (**documentation is excluded**) should
 contain a sign-off at the end of the patch/commit description body. It
-can be automatically added on adding `-s` parameter to `git commit`.
+can be automatically added by adding the `-s` parameter to `git commit`.
 
-This is an example of the aspect of the line:
+This is an example of what the line should look like:
 
-	Signed-off-by: Andrey Antukh <niwi@niwi.nz>
+```
+Signed-off-by: Andrey Antukh <niwi@niwi.nz>
+```
 
 Please, use your real name (sorry, no pseudonyms or anonymous
 contributions are allowed).

README.md

@@ -16,36 +16,38 @@
 </p>
 
 <p align="center">
-    <a href="https://penpot.app/"><b>Website</b></a>  •  
-    <a href="https://help.penpot.app/technical-guide/getting-started/"><b>Getting Started</b></a>  •  
-    <a href="https://help.penpot.app/user-guide/"><b>User Guide</b></a>  •  
-    <a href="https://help.penpot.app/user-guide/introduction/info/"><b>Tutorials & Info</b></a>  •  
+    <a href="https://penpot.app/"><b>Website</b></a>  •
+    <a href="https://help.penpot.app/user-guide/"><b>User Guide</b></a>  •
+    <a href="https://penpot.app/learning-center"><b>Learning Center</b></a>  •
     <a href="https://community.penpot.app/"><b>Community</b></a>
 </p>
 <p align="center">
-    <a href="https://www.youtube.com/@Penpot"><b>Youtube</b></a>  •  
-    <a href="https://peertube.kaleidos.net/a/penpot_app/video-channels"><b>Peertube</b></a>  •  
-    <a href="https://www.linkedin.com/company/penpot/"><b>Linkedin</b></a>  •  
-    <a href="https://instagram.com/penpot.app"><b>Instagram</b></a>  •  
-    <a href="https://fosstodon.org/@penpot/"><b>Mastodon</b></a>  •  
+    <a href="https://www.youtube.com/@Penpot"><b>Youtube</b></a>  •
+    <a href="https://peertube.kaleidos.net/a/penpot_app/video-channels"><b>Peertube</b></a>  •
+    <a href="https://www.linkedin.com/company/penpot/"><b>Linkedin</b></a>  •
+    <a href="https://instagram.com/penpot.app"><b>Instagram</b></a>  •
+    <a href="https://fosstodon.org/@penpot/"><b>Mastodon</b></a>  •
+    <a href="https://bsky.app/profile/penpot.app"><b>Bluesky</b></a>  •
     <a href="https://twitter.com/penpotapp"><b>X</b></a>
 
 </p>
 
 <br />
 
-[Penpot video](https://github.com/penpot/penpot/assets/5446186/b8ad0764-585e-4ddc-b098-9b4090d337cc)
+[Penpot video](https://github.com/user-attachments/assets/7c67fd7c-04d3-4c9b-88ec-b6f5e23f8332
+)
 
 <br />
 
 Penpot is the first **open-source** design tool for design and code collaboration. Designers can create stunning designs, interactive prototypes, design systems at scale, while developers enjoy ready-to-use code and make their workflow easy and fast. And all of this with no handoff drama.
 
-Penpot is available on browser and [self host](https://penpot.app/self-host). It’s web-based and works with open standards (SVG, CSS and HTML). And last but not least, it’s free! 
+Available on browser or self-hosted, Penpot works with open standards like SVG, CSS, HTML and JSON, and it’s free!
 
-Penpot’s latest [huge release 2.0](https://penpot.app/dev-diaries), takes the platform to a whole new level. This update introduces the ground-breaking [CSS Grid Layout feature](https://penpot.app/penpot-2.0), a complete UI redesign, a new Components system, and much more. Plus, it's faster and more accessible. 
+The latest updates take Penpot even further. It’s the first design tool to integrate native [design tokens](https://penpot.dev/collaboration/design-tokens)—a single source of truth to improve efficiency and collaboration between product design and development.
+With the [huge 2.0 release](https://penpot.app/dev-diaries), Penpot took the platform to a whole new level. This update introduces the ground-breaking [CSS Grid Layout feature](https://penpot.app/penpot-2.0), a complete UI redesign, a new Components system, and much more.
+For organizations that need extra service for its teams, [get in touch](https://cal.com/team/penpot/talk-to-us)
 
-
-🎇 **Penpot Fest** is our design, code & Open Source event. Check out the highlights from [Penpot Fest 2023 edition](https://www.youtube.com/watch?v=sOpLZaK5mDc)!
+🎇 Design, code, and Open Source meet at [Penpot Fest](https://penpot.app/penpotfest)! Be part of the 2025 edition in Madrid, Spain, on October 9-10.
 
 ## Table of contents ##
 
@@ -61,7 +63,7 @@ Penpot’s latest [huge release 2.0](https://penpot.app/dev-diaries), takes the
 Penpot expresses designs as code. Designers can do their best work and see it will be beautifully implemented by developers in a two-way collaboration.
 
 ### Plugin system ###
-[Penpot plugins](https://penpot.app/penpothub/plugins) let you expand the platform's capabilities, give you the flexibility to integrate it with other apps, and design custom solutions. 
+[Penpot plugins](https://penpot.app/penpothub/plugins) let you expand the platform's capabilities, give you the flexibility to integrate it with other apps, and design custom solutions.
 
 ### Designed for developers ###
 Penpot was built to serve both designers and developers and create a fluid design-code process. You have the choice to enjoy real-time collaboration or play "solo".
@@ -78,6 +80,10 @@ Penpot offers integration into the development toolchain, thanks to its support
 ### What’s great for design ###
 With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
 
+### Design Tokens ###
+With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
+
+
 <br />
 
 <p align="center">
@@ -88,10 +94,9 @@ With Penpot you can design libraries to share and reuse; turn design elements in
 
 ## Getting started ##
 
-### Install with Elestio ###
-Penpot is the only design & prototype platform that is deployment agnostic. You can use it or deploy it anywhere.
+Penpot is the only design & prototype platform that is deployment agnostic. You can use it in our [SAAS](https://design.penpot.app) or deploy it anywhere.
 
-Learn how to install it with Elestio and Docker, or other options on [our website](https://penpot.app/self-host).
+Learn how to install it with Docker, Kubernetes, Elestio or other options on [our website](https://penpot.app/self-host).
 <br />
 
 <p align="center">
@@ -123,15 +128,21 @@ You will find the following categories:
 </p>
 <br />
 
+### Code of Conduct ###
+
+Anyone who contributes to Penpot, whether through code, in the community, or at an event, must adhere to the
+[code of conduct](https://help.penpot.app/contributing-guide/coc/) and foster a positive and safe environment.
+
+
 ## Contributing ##
 
-Any contribution will make a difference to improve Penpot. How can you get involved? 
+Any contribution will make a difference to improve Penpot. How can you get involved?
 
-Choose your way: 
+Choose your way:
 
 - Create and [share Libraries & Templates](https://penpot.app/libraries-templates.html) that will be helpful for the community
 - Invite your [team to join](https://design.penpot.app/#/auth/register)
-- Star this repo and follow us on Social Media: [Mastodon](https://fosstodon.org/@penpot/), [Youtube](https://www.youtube.com/c/Penpot), [Instagram](https://instagram.com/penpot.app), [Linkedin](https://www.linkedin.com/company/penpotdesign),  [Peertube](https://peertube.kaleidos.net/a/penpot_app) and [X](https://twitter.com/penpotapp).
+- Give this repo a star and follow us on Social Media: [Mastodon](https://fosstodon.org/@penpot/), [Youtube](https://www.youtube.com/c/Penpot), [Instagram](https://instagram.com/penpot.app), [Linkedin](https://www.linkedin.com/company/penpotdesign),  [Peertube](https://peertube.kaleidos.net/a/penpot_app), [X](https://twitter.com/penpotapp) and [BlueSky](https://bsky.app/profile/penpot.app)
 - Participate in the [Community](https://community.penpot.app/) space by asking and answering questions; reacting to others’ articles;  opening your own conversations and following along on decisions affecting the project.
 - Report bugs with our easy [guide for bugs hunting](https://help.penpot.app/contributing-guide/reporting-bugs/) or [GitHub issues](https://github.com/penpot/penpot/issues)
 - Become a [translator](https://help.penpot.app/contributing-guide/translations)

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.0"}
+  org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.6-6"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,8 +17,15 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.4.0.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.7.0.RELEASE"}
+  ;; Minimal dependencies required by lettuce, we need to include them
+  ;; explicitly because clojure dependency management does not support
+  ;; yet the BOM format.
+  io.micrometer/micrometer-core {:mvn/version "1.14.2"}
+  io.micrometer/micrometer-observation {:mvn/version "1.14.2"}
+
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
+  com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
   {:git/tag "v11.4"
@@ -26,24 +33,24 @@
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
-  com.github.seancorfield/next.jdbc {:mvn/version "1.3.955"}
-  metosin/reitit-core {:mvn/version "0.7.2"}
-  nrepl/nrepl {:mvn/version "1.3.0"}
-  cider/cider-nrepl {:mvn/version "0.50.2"}
+  com.github.seancorfield/next.jdbc
+  {:mvn/version "1.3.1002"}
+  metosin/reitit-core {:mvn/version "0.9.1"}
+  nrepl/nrepl {:mvn/version "1.3.1"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.4"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.46.1.3"}
+  org.postgresql/postgresql {:mvn/version "42.7.6"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "6.0.0"}
+  com.zaxxer/HikariCP {:mvn/version "6.3.0"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.8"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.18.1"}
+  org.jsoup/jsoup {:mvn/version "1.20.1"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -54,12 +61,11 @@
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
   dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.1"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.3"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.28.26"}
-  }
+  software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -74,7 +80,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.5" :git/sha "2a21b7a"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test
@@ -88,8 +94,8 @@
 
   :jmx-remote
   {:jvm-opts ["-Dcom.sun.management.jmxremote"
-              "-Dcom.sun.management.jmxremote.port=9091"
-              "-Dcom.sun.management.jmxremote.rmi.port=9091"
+              "-Dcom.sun.management.jmxremote.port=9090"
+              "-Dcom.sun.management.jmxremote.rmi.port=9090"
               "-Dcom.sun.management.jmxremote.local.only=false"
               "-Dcom.sun.management.jmxremote.authenticate=false"
               "-Dcom.sun.management.jmxremote.ssl=false"

backend/package.json

@@ -4,7 +4,7 @@
   "license": "MPL-2.0",
   "author": "Kaleidos INC",
   "private": true,
-  "packageManager": "yarn@4.3.1",
+  "packageManager": "yarn@4.9.2+sha512.1fc009bc09d13cfd0e19efa44cbfc2b9cf6ca61482725eb35bbc5e257e093ebf4130db6dfe15d604ff4b79efd8e1e8e99b25fa7d0a6197c9f9826358d4d65c3c",
   "repository": {
     "type": "git",
     "url": "https://github.com/penpot/penpot"

backend/resources/app/email/comment-mention/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has mentioned you on a comment at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-mention/en.subj

@@ -0,0 +1 @@
+Mentioned in comment

backend/resources/app/email/comment-mention/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has mentioned you on a comment at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/comment-notification/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has commented at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4#31EFB8" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-notification/en.subj

@@ -0,0 +1 @@
+New comment

backend/resources/app/email/comment-notification/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has commented at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/comment-thread/en.html

@@ -0,0 +1,244 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml"
+  xmlns:o="urn:schemas-microsoft-com:office:office">
+
+<head>
+  <title>
+  </title>
+  <!--[if !mso]><!-- -->
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <!--<![endif]-->
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style type="text/css">
+    #outlook a {
+      padding: 0;
+    }
+
+    body {
+      margin: 0;
+      padding: 0;
+      -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+    }
+
+    table,
+    td {
+      border-collapse: collapse;
+      mso-table-lspace: 0pt;
+      mso-table-rspace: 0pt;
+    }
+
+    img {
+      border: 0;
+      height: auto;
+      line-height: 100%;
+      outline: none;
+      text-decoration: none;
+      -ms-interpolation-mode: bicubic;
+    }
+
+    p {
+      display: block;
+      margin: 13px 0;
+    }
+  </style>
+  <!--[if mso]>
+        <xml>
+        <o:OfficeDocumentSettings>
+          <o:AllowPNG/>
+          <o:PixelsPerInch>96</o:PixelsPerInch>
+        </o:OfficeDocumentSettings>
+        </xml>
+        <![endif]-->
+  <!--[if lte mso 11]>
+        <style type="text/css">
+          .mj-outlook-group-fix { width:100% !important; }
+        </style>
+        <![endif]-->
+  <!--[if !mso]><!-->
+  <link href="https://fonts.googleapis.com/css?family=Source%20Sans%20Pro" rel="stylesheet" type="text/css">
+  <style type="text/css">
+    @import url(https://fonts.googleapis.com/css?family=Source%20Sans%20Pro);
+  </style>
+  <!--<![endif]-->
+  <style type="text/css">
+    @media only screen and (min-width:480px) {
+      .mj-column-per-100 {
+        width: 100% !important;
+        max-width: 100%;
+      }
+
+      .mj-column-px-425 {
+        width: 425px !important;
+        max-width: 425px;
+      }
+    }
+  </style>
+  <style type="text/css">
+    @media only screen and (max-width:480px) {
+      table.mj-full-width-mobile {
+        width: 100% !important;
+      }
+
+      td.mj-full-width-mobile {
+        width: auto !important;
+      }
+    }
+  </style>
+</head>
+
+<body style="background-color:#E5E5E5;">
+  <div style="background-color:#E5E5E5;">
+    <!--[if mso | IE]>
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:16px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:collapse;border-spacing:0px;">
+                        <tbody>
+                          <tr>
+                            <td style="width:97px;">
+                              <img height="32" src="{{ public-uri }}/images/email/uxbox-title.png"
+                                style="border:0;display:block;outline:none;text-decoration:none;height:32px;width:100%;font-size:13px;"
+                                width="97" />
+                            </td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <!--[if mso | IE]>
+          </td>
+        </tr>
+      </table>
+
+      <table
+         align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600"
+      >
+        <tr>
+          <td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;">
+      <![endif]-->
+    <div style="background:#FFFFFF;background-color:#FFFFFF;margin:0px auto;max-width:600px;">
+      <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation"
+        style="background:#FFFFFF;background-color:#FFFFFF;width:100%;">
+        <tbody>
+          <tr>
+            <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
+              <!--[if mso | IE]>
+                  <table role="presentation" border="0" cellpadding="0" cellspacing="0">
+
+        <tr>
+
+            <td
+               class="" style="vertical-align:top;width:600px;"
+            >
+          <![endif]-->
+              <div class="mj-column-per-100 mj-outlook-group-fix"
+                style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
+                <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;"
+                  width="100%">
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:24px;font-weight:600;line-height:150%;text-align:left;color:#000000;">
+                        Hello {{name|abbreviate:25}}!</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        <span style="font-weight:bold;">{{ source-user }}</span> has created a comment in a thread you've been mentioned at "{{ comment-reference }}".</div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;font-style:italic;line-height:150%;text-align:left;color:#212426;
+                                  border-top: 1px solid #bfbfbf; border-bottom: 1px solid #bfbfbf; padding: 32px 0px;">
+                        {{ comment-content }}
+                      </div>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="center" vertical-align="middle"
+                      style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <table border="0" cellpadding="0" cellspacing="0" role="presentation"
+                        style="border-collapse:separate;line-height:100%;">
+                        <tr>
+                          <td align="center" bgcolor="#6911d4#31EFB8" role="presentation"
+                              style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
+                              valign="middle">
+                            <a href="{{ comment-url }}"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> GO TO THE COMMENT </a>
+                          </td>
+                        </tr>
+                      </table>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
+                      <div
+                        style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
+                        The Penpot team.</div>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+              <!--[if mso | IE]>
+            </td>
+
+        </tr>
+
+                  </table>
+                <![endif]-->
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+    {% include "app/email/includes/footer.html" %}
+
+  </div>
+</body>
+
+</html>

backend/resources/app/email/comment-thread/en.subj

@@ -0,0 +1 @@
+New response in comment

backend/resources/app/email/comment-thread/en.txt

@@ -0,0 +1,13 @@
+Hello {{name|abbreviate:25}}!
+
+{{ source-user }} has created a comment in a thread you've been mentioned at "{{ comment-reference }}".
+
+--
+
+{{ comment-content }}
+
+--
+
+{{ comment-url }}
+
+The Penpot team.

backend/resources/app/email/invite-to-team/en.html

@@ -195,12 +195,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/verify-token?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Accept invite </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> ACCEPT INVITE </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/join-team/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/projects"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Go to the Team </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GO TO THE TEAM </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/password-recovery/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/recovery?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Reset password </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> RESET PASSWORD </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/register/en.html

@@ -196,12 +196,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
                             <a href="{{ public-uri }}/#/auth/verify-token?token={{token}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Verify email </a>
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> VERIFY EMAIL </a>
                           </td>
                         </tr>
                       </table>

backend/resources/app/email/request-file-access-yourpenpot-view/en.html

@@ -204,12 +204,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -251,4 +251,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access-yourpenpot-view/en.txt

@@ -6,7 +6,7 @@ Since this file is in your Penpot team, you can provide access by sending a view
 
 To proceed, please click the link below to generate and send the view-only link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 

backend/resources/app/email/request-file-access-yourpenpot/en.html

@@ -227,12 +227,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -274,4 +274,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access-yourpenpot/en.txt

@@ -19,7 +19,7 @@ Alternatively, you can create and share a view-only link to the file. This will
 
 Click the link below to generate and send the link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 

backend/resources/app/email/request-file-access/en.html

@@ -211,12 +211,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape }}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Give access to “{{team-name|abbreviate:25}}” Team </a>
+                            <a href="{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape }}"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GIVE ACCESS TO “{{team-name|abbreviate:25}}” TEAM </a>
                           </td>
                         </tr>
                       </table>
@@ -244,12 +244,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Send a View-Only link </a>
+                            <a href="{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true"
+                               style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                               target="_blank"> SEND A VIEW-ONLY LINK </a>
                           </td>
                         </tr>
                       </table>
@@ -292,4 +292,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-file-access/en.txt

@@ -13,7 +13,7 @@ This will automatically include {{requested-by|abbreviate:25}} in the team, so t
 
 Click the link below to provide team access:
 
-{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 
@@ -23,8 +23,7 @@ Alternatively, you can create and share a view-only link to the file. This will
 
 Click the link below to generate and send the link:
 
-{{ public-uri }}/#/view/{{file-id}}?page-id={{page-id}}&section=interactions&index=0&share=true
-
+{{ public-uri }}/#/view?file-id={{file-id}}&page-id={{page-id}}&section=interactions&index=0&share=true
 
 
 If you do not wish to grant access at this time, you can simply disregard this email.

backend/resources/app/email/request-team-access/en.html

@@ -202,12 +202,12 @@
                       <table border="0" cellpadding="0" cellspacing="0" role="presentation"
                         style="border-collapse:separate;line-height:100%;">
                         <tr>
-                          <td align="center" bgcolor="#31EFB8" role="presentation"
-                            style="border:none;border-radius:3px;cursor:auto;mso-padding-alt:10px 25px;background:#31EFB8;"
+                          <td align="center" bgcolor="#6911d4" role="presentation"
+                            style="border:none;border-radius:8px;cursor:auto;mso-padding-alt:10px 25px;background:#6911d4;"
                             valign="middle">
-                            <a href="{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}"
-                              style="display:inline-block;background:#31EFB8;color:#1F1F1F;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:3px;"
-                              target="_blank"> Give access to “{{team-name|abbreviate:25}}” </a>
+                            <a href="{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}"
+                              style="display:inline-block;background:#6911d4;color:#FFFFFF;font-family:Source Sans Pro, sans-serif;font-size:16px;font-weight:normal;line-height:120%;margin:0;text-decoration:none;text-transform:none;padding:10px 25px;mso-padding-alt:0px;border-radius:8px;"
+                              target="_blank"> GIVE ACCESS TO “{{team-name|abbreviate:25}}” TEAM </a>
                           </td>
                         </tr>
                       </table>
@@ -249,4 +249,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/request-team-access/en.txt

@@ -4,7 +4,7 @@ Hello!
 
 To provide access, please click the link below:
 
-{{ public-uri }}/#/dashboard/team/{{team-id}}/members?invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 If you do not wish to grant access at this time, you can simply disregard this email.

backend/resources/app/onboarding.edn

@@ -1,4 +1,7 @@
-[{:id "wireframing-kit"
+[{:id "tokens-starter-kit"
+  :name "Design tokens starter kit"
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"},
+ {:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}
  {:id "prototype-examples"

backend/resources/app/templates/debug.tmpl

@@ -114,37 +114,13 @@ Debug Main Page
     </fieldset>
     <fieldset>
       <legend>Import binfile:</legend>
-      <desc>Import penpot file in binary
-      format. If <strong>overwrite</strong> is checked, all files will
-      be overwritten using the same ids found in the file instead of
-      generating a new ones.</desc>
+      <desc>Import penpot file in binary format.</desc>
 
       <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
         <div class="row">
           <input type="file" name="file" value="" />
         </div>
 
-        <div class="row">
-          <label>Overwrite?</label>
-          <input type="checkbox" name="overwrite" />
-          <br />
-          <small>
-            Instead of creating a new file with all relations remapped,
-            reuses all ids and updates/overwrites the objects that are
-            already exists on the database.
-            <strong>Warning, this operation should be used with caution.</strong>
-          </small>
-        </div>
-
-        <div class="row">
-          <label>Migrate?</label>
-          <input type="checkbox" name="migrate" />
-          <br />
-          <small>
-            Applies the file migrations on the importation process.
-          </small>
-        </div>
-
         <div class="row">
           <input type="submit" name="upload" value="Upload" />
         </div>
@@ -175,6 +151,78 @@ Debug Main Page
         </div>
 
 
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
+
+  <section class="widget">
+    <h2>Feature Flags</h2>
+    <fieldset>
+      <legend>Enable</legend>
+      <desc>Add a feature flag to a team</desc>
+      <form method="post" action="/dbg/actions/add-team-feature">
+        <div class="row">
+          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
+        </div>
+        <div class="row">
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
+        </div>
+
+        <div class="row">
+          <label for="check-feature">Skip feature check</label>
+          <input id="check-feature" type="checkbox" name="skip-check" />
+          <br />
+          <small>
+            Do not check if the feature is supported
+          </small>
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+    <fieldset>
+      <legend>Disable</legend>
+      <desc>Remove a feature flag from a team</desc>
+      <form method="post" action="/dbg/actions/remove-team-feature">
+        <div class="row">
+          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
+        </div>
+        <div class="row">
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
+        </div>
+
+        <div class="row">
+          <label for="check-feature">Skip feature check</label>
+          <input id="check-feature" type="checkbox" name="skip-check" />
+          <br />
+          <small>
+            Do not check if the feature is supported
+          </small>
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
         <div class="row">
           <input type="submit" value="Submit" />
         </div>

backend/scripts/manage.py

@@ -35,40 +35,35 @@ def get_prepl_conninfo():
 
     return host, port
 
-def send_eval(expr):
+def send(data):
     host, port = get_prepl_conninfo()
+    with socket.create_connection((host, port)) as s:
+        f = s.makefile(mode="rw")
 
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.connect((host, port))
-        s.send(expr.encode("utf-8"))
-        s.send(b":repl/quit\n\n")
+        json.dump(data, f)
+        f.write("\n")
+        f.flush()
 
-        with s.makefile() as f:
-            while True:
-                line = f.readline()
-                result = json.loads(line)
-                tag = result.get("tag", None)
-                if tag == "ret":
-                    return result.get("val", None), result.get("exception", None)
-                elif tag == "out":
-                    print(result.get("val"), end="")
-                else:
-                    raise RuntimeError("unexpected response from PREPL")
+        while True:
+            line = f.readline()
+            result = json.loads(line)
+            tag = result.get("tag", None)
 
-def encode(val):
-    return json.dumps(json.dumps(val))
+            if tag == "ret":
+                return result.get("val", None), result.get("err", None)
+            elif tag == "out":
+                print(result.get("val"), end="")
+            else:
+                raise RuntimeError("unexpected response from PREPL")
 
-def print_error(res):
-    for error in res["via"]:
-        print("ERR:", error["message"])
-        break
+def print_error(error):
+    print("ERR:", error["hint"])
 
 def run_cmd(params):
     try:
-        expr = "(app.srepl.cli/exec {})".format(encode(params))
-        res, failed = send_eval(expr)
-        if failed:
-            print_error(res)
+        res, err = send(params)
+        if err:
+            print_error(err)
             sys.exit(-1)
 
         return res
@@ -96,7 +91,7 @@ def update_profile(email, fullname, password, is_active):
             "email": email,
             "fullname": fullname,
             "password": password,
-            "is_active": is_active
+            "isActive": is_active
         }
     }
 
@@ -138,7 +133,7 @@ def derive_password(password):
     params = {
         "cmd": "derive-password",
         "params": {
-            "password": password,
+            "password": password
         }
     }
 

backend/scripts/repl

@@ -30,7 +30,9 @@ export PENPOT_FLAGS="\
        enable-access-tokens \
        enable-tiered-file-data-storage \
        enable-file-validation \
-       enable-file-schema-validation";
+       enable-file-schema-validation \
+       enable-subscriptions \
+       enable-subscriptions-old";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"
@@ -70,20 +72,21 @@ export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
 export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
 
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full"
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow  \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
-# Enable preview
-export OPTIONS="$OPTIONS -J--enable-preview"
+export OPTIONS="-A:jmx-remote -A:dev"
 
 # Setup HEAP
 # export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
@@ -104,9 +107,6 @@ export OPTIONS="$OPTIONS -J--enable-preview"
 # Setup GC
 # export OPTIONS="$OPTIONS -J-XX:+UseZGC"
 
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 

backend/scripts/run

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-backend-asserts \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-file-snapshot \
+       enable-tiered-file-data-storage";
+
+export JAVA_OPTS="
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints";
+
+export CLOJURE_OPTIONS="-A:dev"
+
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+
+entrypoint=${1:-app.main};
+shift 1;
+set -ex
+
+clojure $CLOJURE_OPTIONS -A:dev -M -m $entrypoint "$@";

backend/scripts/run.template.sh

@@ -18,9 +18,9 @@ if [ -f ./environ ]; then
     source ./environ
 fi
 
-export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow -Dpolyglot.engine.WarnInterpreterOnly=false --enable-preview $JVM_OPTS"
+export JAVA_OPTS="-Dim4java.useV7=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
 
 ENTRYPOINT=${1:-app.main};
 
 set -ex
-exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m $ENTRYPOINT
+exec $JAVA_CMD $JAVA_OPTS -jar penpot.jar -m $ENTRYPOINT

backend/scripts/start-dev

@@ -23,18 +23,9 @@ export PENPOT_FLAGS="\
        enable-access-tokens \
        enable-tiered-file-data-storage \
        enable-file-validation \
-       enable-file-schema-validation";
-
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints"
+       enable-file-schema-validation \
+       enable-subscriptions \
+       enable-subscriptions-old ";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"
@@ -45,9 +36,6 @@ export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
 # Setup default multipart upload size to 300MiB
 export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
 # Initialize MINIO config
 mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
 mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
@@ -65,6 +53,18 @@ export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 
 entrypoint=${1:-app.main};
 
-set -ex
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:-OmitStackTraceInFastThrow  \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
-clojure $OPTIONS -A:dev -M -m $entrypoint;
+export OPTIONS="-A:jmx-remote -A:dev"
+
+set -ex
+clojure $OPTIONS -M -m $entrypoint;

backend/src/app/binfile/cleaner.clj

@@ -0,0 +1,131 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.binfile.cleaner
+  "A collection of helpers for perform cleaning of artifacts; mainly
+  for recently imported shapes."
+  (:require
+   [app.common.data :as d]
+   [app.common.types.shape :as cts]
+   [app.common.uuid :as uuid]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PRE DECODE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- pre-clean-bool-content
+  [shape]
+  (if-let [content (get shape :bool-content)]
+    (-> shape
+        (assoc :content content)
+        (dissoc :bool-content))
+    shape))
+
+(defn- pre-clean-shadow-color
+  [shape]
+  (d/update-when shape :shadow
+                 (fn [shadows]
+                   (mapv (fn [shadow]
+                           (update shadow :color
+                                   (fn [color]
+                                     (let [ref-id   (get color :id)
+                                           ref-file (get color :file-id)]
+                                       (-> (d/without-qualified color)
+                                           (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
+                                           (cond-> ref-id
+                                             (assoc :ref-id ref-id))
+                                           (cond-> ref-file
+                                             (assoc :ref-file ref-file)))))))
+                         shadows))))
+
+(defn clean-shape-pre-decode
+  "Applies a pre-decode phase migration to the shape"
+  [shape]
+  (cond-> shape
+    (= "bool" (:type shape))
+    (pre-clean-bool-content)
+
+    (contains? shape :shadow)
+    (pre-clean-shadow-color)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; POST DECODE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- fix-shape-shadow-color
+  "Some shapes can come with invalid `id` property on shadow colors
+  caused by incorrect uuid parsing bug that should be already fixed;
+  this function removes the invalid id from the data structure."
+  [shape]
+  (let [fix-color
+        (fn [{:keys [id] :as color}]
+          (if (uuid? id)
+            color
+            (if (and (string? id)
+                     (re-matches uuid/regex id))
+              (assoc color :id (uuid/uuid id))
+              (dissoc color :id))))
+
+        fix-shadow
+        (fn [shadow]
+          (d/update-when shadow :color fix-color))
+
+        xform
+        (map fix-shadow)]
+
+    (d/update-when shape :shadow
+                   (fn [shadows]
+                     (into [] xform shadows)))))
+
+(defn- fix-root-shape
+  "Ensure all root objects are well formed shapes"
+  [shape]
+  (if (= (:id shape) uuid/zero)
+    (-> shape
+        (assoc :parent-id uuid/zero)
+        (assoc :frame-id uuid/zero)
+        ;; We explicitly dissoc them and let the shape-setup
+        ;; to regenerate it with valid values.
+        (dissoc :selrect)
+        (dissoc :points)
+        (cts/setup-shape))
+    shape))
+
+(defn- fix-legacy-flex-dir
+  "This operation is only relevant to old data and it is fixed just
+  for convenience."
+  [shape]
+  (d/update-when shape :layout-flex-dir
+                 (fn [dir]
+                   (case dir
+                     :reverse-row :row-reverse
+                     :reverse-column :column-reverse
+                     dir))))
+
+(defn clean-shape-post-decode
+  "A shape procesor that expected to be executed after schema decoding
+  process but before validation."
+  [shape]
+  (-> shape
+      (fix-shape-shadow-color)
+      (fix-root-shape)
+      (fix-legacy-flex-dir)))
+
+(defn- fix-container
+  [container]
+  (-> container
+      ;; Remove possible `nil` keys on objects
+      (d/update-when :objects dissoc nil)
+      (d/update-when :objects d/update-vals clean-shape-post-decode)))
+
+(defn clean-file
+  [file & {:as _opts}]
+  (update file :data
+          (fn [data]
+            (-> data
+                (d/update-when :pages-index d/update-vals fix-container)
+                (d/update-when :components d/update-vals fix-container)
+                (d/without-nils)))))

backend/src/app/binfile/common.clj

@@ -9,9 +9,9 @@
   binfile format implementations and management rpc methods."
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
+   [app.common.files.helpers :as cfh]
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
@@ -20,17 +20,19 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.components-v2 :as feat.compv2]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
-   [clojure.walk :as walk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [datoteka.fs :as fs]
+   [datoteka.io :as io]))
 
 (set! *warn-on-reflection* true)
 
@@ -51,6 +53,31 @@
   (* 1024 1024 100))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(declare get-resolved-file-libraries)
+
+(def file-attrs
+  #{:id
+    :name
+    :migrations
+    :features
+    :project-id
+    :is-shared
+    :version
+    :data})
+
+(defn parse-file-format
+  [template]
+  (assert (fs/path? template) "expected InputStream for `template`")
+
+  (with-open [^java.lang.AutoCloseable input (io/input-stream template)]
+    (let [buffer (byte-array 4)]
+      (io/read-to-buffer input buffer)
+      (if (and (= (aget buffer 0) 80)
+               (= (aget buffer 1) 75)
+               (= (aget buffer 2) 3)
+               (= (aget buffer 3) 4))
+        :binfile-v3
+        :binfile-v1))))
 
 (def xf-map-id
   (map :id))
@@ -117,22 +144,43 @@
    (reduce #(index-object %1 %2 attr) index coll)))
 
 (defn decode-row
-  "A generic decode row helper"
-  [{:keys [data features] :as row}]
-  (cond-> row
-    features (assoc :features (db/decode-pgarray features #{}))
-    data     (assoc :data (blob/decode data))))
+  [{:keys [data changes features] :as row}]
+  (when row
+    (cond-> row
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
+
+
+(defn decode-file
+  "A general purpose file decoding function that resolves all external
+  pointers, run migrations and return plain vanilla file map"
+  [cfg {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+    (let [file (->> file
+                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (feat.fdata/resolve-file-data cfg))
+          libs (delay (get-resolved-file-libraries cfg file))]
+
+      (-> file
+          (update :features db/decode-pgarray #{})
+          (update :data blob/decode)
+          (update :data feat.fdata/process-pointers deref)
+          (update :data feat.fdata/process-objects (partial into {}))
+          (update :data assoc :id id)
+          (fmg/migrate-file libs)))))
 
 (defn get-file
-  [cfg file-id]
+  "Get file, resolve all features and apply migrations.
+
+  Usefull when you have plan to apply massive or not cirurgical
+  operations on file, because it removes the ovehead of lazy fetching
+  and decoding."
+  [cfg file-id & {:as opts}]
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                 (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                   (when-let [file (db/get* conn :file {:id file-id}
-                                            {::db/remove-deleted false})]
-                     (-> file
-                         (decode-row)
-                         (update :data feat.fdata/process-pointers deref)
-                         (update :data feat.fdata/process-objects (partial into {}))))))))
+                 (some->> (db/get* conn :file {:id file-id}
+                                   (assoc opts ::db/remove-deleted false))
+                          (decode-file cfg)))))
 
 (defn clean-file-features
   [file]
@@ -173,10 +221,8 @@
   "Given a set of file-id's, return all matching relations with the libraries"
   [cfg ids]
 
-  (dm/assert!
-   "expected a set of uuids"
-   (and (set? ids)
-        (every? uuid? ids)))
+  (assert (set? ids) "expected a set of uuids")
+  (assert (every? uuid? ids) "expected a set of uuids")
 
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (let [ids (db/create-array conn "uuid" ids)
@@ -225,49 +271,70 @@
             :data nil}
            {::sql/columns [:media-id :file-id :revn]}))
 
+(def ^:private sql:get-missing-media-references
+  "SELECT fmo.*
+     FROM file_media_object AS fmo
+    WHERE fmo.id = ANY(?::uuid[])
+      AND file_id != ?")
 
-(def ^:private
-  xform:collect-media-id
-  (comp
-   (map :objects)
-   (mapcat vals)
-   (mapcat (fn [obj]
-             ;; NOTE: because of some bug, we ended with
-             ;; many shape types having the ability to
-             ;; have fill-image attribute (which initially
-             ;; designed for :path shapes).
-             (sequence
-              (keep :id)
-              (concat [(:fill-image obj)
-                       (:metadata obj)]
-                      (map :fill-image (:fills obj))
-                      (map :stroke-image (:strokes obj))
-                      (->> (:content obj)
-                           (tree-seq map? :children)
-                           (mapcat :fills)
-                           (map :fill-image))))))))
+(defn update-media-references!
+  "Given a file and a coll of media-refs, check if all provided
+  references are correct or fix them in-place"
+  [{:keys [::db/conn] :as cfg} {file-id :id :as file} media-refs]
+  (let [missing-index
+        (reduce (fn [result {:keys [id] :as fmo}]
+                  (assoc result id
+                         (-> fmo
+                             (assoc :id (uuid/next))
+                             (assoc :file-id file-id)
+                             (dissoc :created-at)
+                             (dissoc :deleted-at))))
+                {}
+                (db/exec! conn [sql:get-missing-media-references
+                                (->> (into #{} xf-map-id media-refs)
+                                     (db/create-array conn "uuid"))
+                                file-id]))
 
-(defn collect-used-media
-  "Given a fdata (file data), returns all media references."
-  [data]
-  (-> #{}
-      (into xform:collect-media-id (vals (:pages-index data)))
-      (into xform:collect-media-id (vals (:components data)))
-      (into (keys (:media data)))))
+        lookup-index
+        (fn [id]
+          (if-let [mobj (get missing-index id)]
+            (do
+              (l/trc :hint "lookup index"
+                     :file-id (str file-id)
+                     :snap-id (str (:snapshot-id file))
+                     :id (str id)
+                     :result (str (get mobj :id)))
+              (get mobj :id))
+
+            id))
+
+        update-shapes
+        (fn [data {:keys [page-id shape-id]}]
+          (d/update-in-when data [:pages-index page-id :objects shape-id] cfh/relink-refs lookup-index))
+
+        file
+        (update file :data #(reduce update-shapes % media-refs))]
+
+    (doseq [[old-id item] missing-index]
+      (l/dbg :hint "create missing references"
+             :file-id (str file-id)
+             :snap-id (str (:snapshot-id file))
+             :old-id (str old-id)
+             :id (str (:id item)))
+      (db/insert! conn :file-media-object item
+                  {::db/return-keys false}))
+
+    file))
+
+(def sql:get-file-media
+  "SELECT * FROM file_media_object WHERE id = ANY(?)")
 
 (defn get-file-media
-  [cfg {:keys [data id] :as file}]
+  [cfg {:keys [data] :as file}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (let [ids (collect-used-media data)
-                       ids (db/create-array conn "uuid" ids)
-                       sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")]
-
-                   ;; We assoc the file-id again to the file-media-object row
-                   ;; because there are cases that used objects refer to other
-                   ;; files and we need to ensure in the exportation process that
-                   ;; all ids matches
-                   (->> (db/exec! conn [sql ids])
-                        (mapv #(assoc % :file-id id)))))))
+                 (let [used (cfh/collect-used-media data)
+                       used (db/create-array conn "uuid" used)]
+                   (db/exec! conn [sql:get-file-media used])))))
 
 (def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
@@ -311,48 +378,7 @@
   replace the old :component-file reference with the new
   ones, using the provided file-index."
   [data]
-  (letfn [(process-map-form [form]
-            (cond-> form
-              ;; Relink image shapes
-              (and (map? (:metadata form))
-                   (= :image (:type form)))
-              (update-in [:metadata :id] lookup-index)
-
-              ;; Relink paths with fill image
-              (map? (:fill-image form))
-              (update-in [:fill-image :id] lookup-index)
-
-              ;; This covers old shapes and the new :fills.
-              (uuid? (:fill-color-ref-file form))
-              (update :fill-color-ref-file lookup-index)
-
-              ;; This covers the old shapes and the new :strokes
-              (uuid? (:stroke-color-ref-file form))
-              (update :stroke-color-ref-file lookup-index)
-
-              ;; This covers all text shapes that have typography referenced
-              (uuid? (:typography-ref-file form))
-              (update :typography-ref-file lookup-index)
-
-              ;; This covers the component instance links
-              (uuid? (:component-file form))
-              (update :component-file lookup-index)
-
-              ;; This covers the shadows and grids (they have directly
-              ;; the :file-id prop)
-              (uuid? (:file-id form))
-              (update :file-id lookup-index)))
-
-          (process-form [form]
-            (if (map? form)
-              (try
-                (process-map-form form)
-                (catch Throwable cause
-                  (l/warn :hint "failed form" :form (pr-str form) ::l/sync? true)
-                  (throw cause)))
-              form))]
-
-    (walk/postwalk process-form data)))
+  (cfh/relink-refs data lookup-index))
 
 (defn- relink-media
   "A function responsible of process the :media attr of file data and
@@ -383,8 +409,9 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (let [library (get-file cfg library-id)]
-                (ctf/absorb-assets data (:data library))))
+              (if-let [library (get-file cfg library-id)]
+                (ctf/absorb-assets data (:data library))
+                data))
             data
             library-ids)))
 
@@ -394,130 +421,161 @@
     (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
-(defn- fix-version
-  [file]
-  (let [file (fmg/fix-version file)]
-    ;; FIXME: We're temporarily activating all migrations because a
-    ;; problem in the environments messed up with the version numbers
-    ;; When this problem is fixed delete the following line
-    (if (> (:version file) 22)
-      (assoc file :version 22)
-      file)))
-
 (defn process-file
-  [{:keys [id] :as file}]
-  (-> file
-      (fix-version)
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (assoc :id id)
-                          (dissoc :recent-colors))))
-      (fmg/migrate-file)
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (update :pages-index relink-shapes)
-                          (update :components relink-shapes)
-                          (update :media relink-media)
-                          (update :colors relink-colors)
-                          (d/without-nils))))))
+  [cfg {:keys [id] :as file}]
+  (let [libs (delay (get-resolved-file-libraries cfg file))]
+    (-> file
+        (update :data (fn [fdata]
+                        (-> fdata
+                            (assoc :id id)
+                            (dissoc :recent-colors))))
+        (update :data (fn [fdata]
+                        (-> fdata
+                            (update :pages-index relink-shapes)
+                            (update :components relink-shapes)
+                            (update :media relink-media)
+                            (update :colors relink-colors)
+                            (d/without-nils))))
+        (fmg/migrate-file libs)
 
-(defn- upsert-file!
-  [conn file]
-  (let [sql (str "INSERT INTO file (id, project_id, name, revn, version, is_shared, data, created_at, modified_at) "
-                 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
-                 "ON CONFLICT (id) DO UPDATE SET data=?, version=?")]
-    (db/exec-one! conn [sql
-                        (:id file)
-                        (:project-id file)
-                        (:name file)
-                        (:revn file)
-                        (:version file)
-                        (:is-shared file)
-                        (:data file)
-                        (:created-at file)
-                        (:modified-at file)
-                        (:data file)
-                        (:version file)])))
+        ;; NOTE: this is necessary because when we just creating a new
+        ;; file from imported artifact or cloned file there are no
+        ;; migrations registered on the database, so we need to persist
+        ;; all of them, not only the applied
+        (vary-meta dissoc ::fmg/migrated))))
 
-(defn persist-file!
-  "Applies all the final validations and perist the file."
-  [{:keys [::db/conn ::timestamp] :as cfg} {:keys [id] :as file}]
+(defn encode-file
+  [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
+  (let [file (if (contains? features "fdata/objects-map")
+               (feat.fdata/enable-objects-map file)
+               file)
 
-  (dm/assert!
-   "expected valid timestamp"
-   (dt/instant? timestamp))
+        file (if (contains? features "fdata/pointer-map")
+               (binding [pmap/*tracked* (pmap/create-tracked)]
+                 (let [file (feat.fdata/enable-pointer-map file)]
+                   (feat.fdata/persist-pointers! cfg id)
+                   file))
+               file)]
 
-  (let [file   (-> file
-                   (assoc :created-at timestamp)
-                   (assoc :modified-at timestamp)
-                   (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
-                   (update :features
-                           (fn [features]
-                             (let [features (cfeat/check-supported-features! features)]
-                               (-> (::features cfg #{})
-                                   (set/union features)
-                                   ;; We never want to store
-                                   ;; frontend-only features on file
-                                   (set/difference cfeat/frontend-only-features))))))
+    (-> file
+        (update :features db/encode-pgarray conn "text")
+        (update :data blob/encode))))
+
+(defn get-params-from-file
+  [file]
+  (let [params {:has-media-trimmed (:has-media-trimmed file)
+                :ignore-sync-until (:ignore-sync-until file)
+                :project-id (:project-id file)
+                :features (:features file)
+                :name (:name file)
+                :is-shared (:is-shared file)
+                :version (:version file)
+                :data (:data file)
+                :id (:id file)
+                :deleted-at (:deleted-at file)
+                :created-at (:created-at file)
+                :modified-at (:modified-at file)
+                :revn (:revn file)
+                :vern (:vern file)}]
+
+    (-> (d/without-nils params)
+        (assoc :data-backend nil)
+        (assoc :data-ref-id nil))))
+
+(defn insert-file!
+  "Insert a new file into the database table"
+  [{:keys [::db/conn] :as cfg} file & {:as opts}]
+  (feat.fmigr/upsert-migrations! conn file)
+  (let [params (-> (encode-file cfg file)
+                   (get-params-from-file))]
+    (db/insert! conn :file params opts)))
+
+(defn update-file!
+  "Update an existing file on the database."
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
+  (let [file   (encode-file cfg file)
+        params (-> (get-params-from-file file)
+                   (dissoc :id))]
+
+    ;; If file was already offloaded, we touch the underlying storage
+    ;; object for properly trigger storage-gc-touched task
+    (when (feat.fdata/offloaded? file)
+      (some->> (:data-ref-id file) (sto/touch-object! storage)))
+
+    (feat.fmigr/upsert-migrations! conn file)
+    (db/update! conn :file params {:id id} opts)))
+
+(defn save-file!
+  "Applies all the final validations and perist the file, binfile
+  specific, should not be used outside of binfile domain"
+  [{:keys [::timestamp] :as cfg} file & {:as opts}]
+
+  (assert (dt/instant? timestamp) "expected valid timestamp")
+
+  (let [file (-> file
+                 (assoc :created-at timestamp)
+                 (assoc :modified-at timestamp)
+                 (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
+                 (update :features
+                         (fn [features]
+                           (-> (::features cfg #{})
+                               (set/union features)
+                               ;; We never want to store
+                               ;; frontend-only features on file
+                               (set/difference cfeat/frontend-only-features)))))]
+
+    (when (contains? cf/flags :file-schema-validation)
+      (fval/validate-file-schema! file))
+
+    (when (contains? cf/flags :soft-file-schema-validation)
+      (let [result (ex/try! (fval/validate-file-schema! file))]
+        (when (ex/exception? result)
+          (l/error :hint "file schema validation error" :cause result))))
+
+    (insert-file! cfg file opts)))
 
 
-        _      (when (contains? cf/flags :file-schema-validation)
-                 (fval/validate-file-schema! file))
+(def ^:private sql:get-file-libraries
+  "WITH RECURSIVE libs AS (
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+      WHERE flr.file_id = ?::uuid
+    UNION
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+       JOIN libs AS l ON (flr.file_id = l.id)
+   )
+   SELECT l.id,
+          l.features,
+          l.project_id,
+          p.team_id,
+          l.created_at,
+          l.modified_at,
+          l.deleted_at,
+          l.name,
+          l.revn,
+          l.vern,
+          l.synced_at,
+          l.is_shared
+     FROM libs AS l
+    INNER JOIN project AS p ON (p.id = l.project_id)
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
 
-        _      (when (contains? cf/flags :soft-file-schema-validation)
-                 (let [result (ex/try! (fval/validate-file-schema! file))]
-                   (when (ex/exception? result)
-                     (l/error :hint "file schema validation error" :cause result))))
+(defn get-file-libraries
+  [conn file-id]
+  (into []
+        (comp
+         ;; FIXME: :is-indirect set to false to all rows looks
+         ;; completly useless
+         (map #(assoc % :is-indirect false))
+         (map decode-row))
+        (db/exec! conn [sql:get-file-libraries file-id])))
 
-        file   (if (contains? (:features file) "fdata/objects-map")
-                 (feat.fdata/enable-objects-map file)
-                 file)
-
-        file   (if (contains? (:features file) "fdata/pointer-map")
-                 (binding [pmap/*tracked* (pmap/create-tracked)]
-                   (let [file (feat.fdata/enable-pointer-map file)]
-                     (feat.fdata/persist-pointers! cfg id)
-                     file))
-                 file)
-
-        params (-> file
-                   (update :features db/encode-pgarray conn "text")
-                   (update :data blob/encode))]
-
-    (if (::overwrite cfg)
-      (upsert-file! conn params)
-      (db/insert! conn :file params ::db/return-keys false))
-
-    file))
-
-
-(defn register-pending-migrations
-  "All features that are enabled and requires explicit migration are
-  added to the state for a posterior migration step."
-  [cfg {:keys [id features] :as file}]
-  (doseq [feature (-> (::features cfg)
-                      (set/difference cfeat/no-migration-features)
-                      (set/difference cfeat/backend-only-features)
-                      (set/difference features))]
-    (vswap! *state* update :pending-to-migrate (fnil conj []) [feature id]))
-
-  file)
-
-
-(defn apply-pending-migrations!
-  "Apply alredy registered pending migrations to files"
-  [cfg]
-  (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
-    (case feature
-      "components/v2"
-      (feat.compv2/migrate-file! cfg file-id
-                                 :validate? (::validate cfg true)
-                                 :skip-on-graphic-error? true)
-
-      "fdata/shape-data-type"
-      nil
-
-      (ex/raise :type :internal
-                :code :no-migration-defined
-                :hint (str/ffmt "no migation for feature '%' on file importation" feature)
-                :feature feature))))
+(defn get-resolved-file-libraries
+  "A helper for preload file libraries"
+  [{:keys [::db/conn] :as cfg} file]
+  (->> (get-file-libraries conn (:id file))
+       (into [file] (map #(get-file cfg (:id %))))
+       (d/index-by :id)))

backend/src/app/binfile/migrations.clj

@@ -0,0 +1,47 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.binfile.migrations
+  "A binfile related migrations handling"
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [clojure.set :as set]
+   [cuerdas.core :as str]))
+
+(defn register-pending-migrations!
+  "All features that are enabled and requires explicit migration are
+  added to the state for a posterior migration step."
+  [cfg {:keys [id features] :as file}]
+  (doseq [feature (-> (::features cfg)
+                      (set/difference cfeat/no-migration-features)
+                      (set/difference cfeat/backend-only-features)
+                      (set/difference features))]
+    (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature id]))
+
+  file)
+
+(defn apply-pending-migrations!
+  "Apply alredy registered pending migrations to files"
+  [_cfg]
+  (doseq [[feature _file-id] (-> bfc/*state* deref :pending-to-migrate)]
+    (case feature
+      "components/v2"
+      nil
+
+      "fdata/shape-data-type"
+      nil
+
+      ;; There is no migration needed, but we don't want to allow
+      ;; copy paste nor import of variant files into no-variant teams
+      "variants/v1"
+      nil
+
+      (ex/raise :type :internal
+                :code :no-migration-defined
+                :hint (str/ffmt "no migation for feature '%' on file importation" feature)
+                :feature feature))))

backend/src/app/binfile/v1.clj

@@ -9,6 +9,7 @@
   (:refer-clojure :exclude [assert])
   (:require
    [app.binfile.common :as bfc]
+   [app.binfile.migrations :as bfm]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -298,7 +299,7 @@
 (defmulti write-section ::section)
 
 (defn write-export!
-  [{:keys [::include-libraries ::embed-assets] :as cfg}]
+  [{:keys [::bfc/include-libraries ::bfc/embed-assets] :as cfg}]
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
@@ -323,7 +324,7 @@
             [:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
 
 (defmethod write-section :v1/metadata
-  [{:keys [::output ::ids ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/ids ::bfc/include-libraries] :as cfg}]
   (if-let [fids (get-files cfg ids)]
     (let [lids (when include-libraries
                  (bfc/get-libraries cfg ids))
@@ -335,7 +336,7 @@
               :hint "unable to retrieve files for export")))
 
 (defmethod write-section :v1/files
-  [{:keys [::output ::embed-assets ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/embed-assets ::bfc/include-libraries] :as cfg}]
 
   ;; Initialize SIDS with empty vector
   (vswap! bfc/*state* assoc :sids [])
@@ -382,7 +383,7 @@
       (vswap! bfc/*state* update :sids into bfc/xf-map-media-id thumbnails))))
 
 (defmethod write-section :v1/rels
-  [{:keys [::output ::include-libraries] :as cfg}]
+  [{:keys [::output ::bfc/include-libraries] :as cfg}]
   (let [ids  (-> bfc/*state* deref :files set)
         rels (when include-libraries
                (bfc/get-files-rels cfg ids))]
@@ -421,25 +422,19 @@
 (defmulti read-import ::version)
 (defmulti read-section ::section)
 
-(s/def ::profile-id ::us/uuid)
-(s/def ::project-id ::us/uuid)
-(s/def ::input io/input-stream?)
-(s/def ::overwrite? (s/nilable ::us/boolean))
+(s/def ::bfc/profile-id ::us/uuid)
+(s/def ::bfc/project-id ::us/uuid)
+(s/def ::bfc/input io/input-stream?)
 (s/def ::ignore-index-errors? (s/nilable ::us/boolean))
 
-;; FIXME: replace with schema
 (s/def ::read-import-options
-  (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
-          :opt [::overwrite? ::ignore-index-errors?]))
+  (s/keys :req [::db/pool ::sto/storage ::bfc/project-id ::bfc/profile-id ::bfc/input]
+          :opt [::ignore-index-errors?]))
 
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
-  format. There are some options for customize the importation
-  behavior:
-
-  `::bfc/overwrite`: if true, instead of creating new files and remapping id references,
-  it reuses all ids and updates existing objects; defaults to `false`."
-  [{:keys [::input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
+  format."
+  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
 
   (dm/assert!
    "expected input stream"
@@ -453,7 +448,7 @@
     (read-import (assoc options ::version version ::bfc/timestamp timestamp))))
 
 (defn- read-import-v1
-  [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}]
+  [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}]
 
   (bfc/disable-database-timeouts! cfg)
 
@@ -473,13 +468,13 @@
                 (let [options (-> cfg
                                   (assoc ::bfc/features features)
                                   (assoc ::section section)
-                                  (assoc ::input input))]
+                                  (assoc ::bfc/input input))]
                   (binding [bfc/*options* options]
                     (events/tap :progress {:op :import :section section})
                     (read-section options))))
               [:v1/metadata :v1/files :v1/rels :v1/sobjects])
 
-        (bfc/apply-pending-migrations! cfg)
+        (bfm/apply-pending-migrations! cfg)
 
         ;; Knowing that the ids of the created files are in index,
         ;; just lookup them and return it as a set
@@ -491,7 +486,7 @@
   (db/tx-run! options read-import-v1))
 
 (defmethod read-section :v1/metadata
-  [{:keys [::input]}]
+  [{:keys [::bfc/input]}]
   (let [{:keys [version files]} (read-obj! input)]
     (l/dbg :hint "metadata readed"
            :version (:full version)
@@ -509,8 +504,7 @@
         thumbnails))
 
 (defmethod read-section :v1/files
-  [{:keys [::db/conn ::input ::project-id ::bfc/overwrite ::name] :as system}]
-
+  [{:keys [::bfc/input ::bfc/project-id ::bfc/name] :as system}]
   (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))]
     (let [file       (read-obj! input)
           media      (read-obj! input)
@@ -557,8 +551,8 @@
                      (cond-> (and (= idx 0) (some? name))
                        (assoc :name name))
                      (assoc :project-id project-id)
-                     (dissoc :thumbnails)
-                     (bfc/process-file))]
+                     (dissoc :thumbnails))
+            file  (bfc/process-file system file)]
 
         ;; All features that are enabled and requires explicit migration are
         ;; added to the state for a posterior migration step.
@@ -568,15 +562,12 @@
           (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id']))
 
         (l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
-        (bfc/persist-file! system file)
-
-        (when overwrite
-          (db/delete! conn :file-thumbnail {:file-id file-id'}))
+        (bfc/save-file! system file ::db/return-keys false)
 
         file-id'))))
 
 (defmethod read-section :v1/rels
-  [{:keys [::db/conn ::input ::bfc/timestamp]}]
+  [{:keys [::db/conn ::bfc/input ::bfc/timestamp]}]
   (let [rels (read-obj! input)
         ids  (into #{} (-> bfc/*state* deref :files))]
     ;; Insert all file relations
@@ -600,7 +591,7 @@
                   ::l/sync? true))))))
 
 (defmethod read-section :v1/sobjects
-  [{:keys [::db/conn ::input ::bfc/overwrite ::bfc/timestamp] :as cfg}]
+  [{:keys [::db/conn ::bfc/input ::bfc/timestamp] :as cfg}]
   (let [storage (sto/resolve cfg)
         ids     (read-obj! input)
         thumb?  (into #{} (map :media-id) (:thumbnails @bfc/*state*))]
@@ -653,8 +644,7 @@
                       (-> item
                           (assoc :file-id file-id)
                           (d/update-when :media-id bfc/lookup-index)
-                          (d/update-when :thumbnail-id bfc/lookup-index))
-                      {::db/on-conflict-do-nothing? overwrite}))))
+                          (d/update-when :thumbnail-id bfc/lookup-index))))))
 
     (doseq [item (:thumbnails @bfc/*state*)]
       (let [item (update item :media-id bfc/lookup-index)]
@@ -663,8 +653,7 @@
                :media-id (str (:media-id item))
                :object-id (:object-id item)
                ::l/sync? true)
-        (db/insert! conn :file-tagged-object-thumbnail item
-                    {::db/on-conflict-do-nothing? overwrite})))))
+        (db/insert! conn :file-tagged-object-thumbnail item)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HIGH LEVEL API
@@ -674,17 +663,17 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
   (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
+   "expected a set of uuid's for `::bfc/ids` parameter"
    (and (set? ids)
         (every? uuid? ids)))
 
@@ -719,12 +708,12 @@
                 :cause @cs)))))
 
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
   (dm/assert!
    "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg))))
 
   (dm/assert!
    "expected instance of jio/IOFactory for `input`"
@@ -738,7 +727,7 @@
     (try
       (binding [*position* (atom 0)]
         (pu/with-open [input (io/input-stream input)]
-          (read-import! (assoc cfg ::input input))))
+          (read-import! (assoc cfg ::bfc/input input))))
 
       (catch ZstdIOException cause
         (ex/raise :type :validation

backend/src/app/binfile/v2.clj

@@ -281,8 +281,8 @@
 
   (let [file (-> (read-obj cfg :file file-id)
                  (update :id bfc/lookup-index)
-                 (update :project-id bfc/lookup-index)
-                 (bfc/process-file))]
+                 (update :project-id bfc/lookup-index))
+        file (bfc/process-file cfg file)]
 
     (events/tap :progress
                 {:op :import
@@ -297,7 +297,7 @@
                         (set/difference (:features file)))]
       (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature (:id file)]))
 
-    (bfc/persist-file! cfg file))
+    (bfc/save-file! cfg file ::db/return-keys false))
 
   (doseq [thumbnail (read-seq cfg :file-object-thumbnail file-id)]
     (let [thumbnail (-> thumbnail

backend/src/app/binfile/v3.clj

@@ -8,13 +8,16 @@
   "A ZIP based binary file exportation"
   (:refer-clojure :exclude [read])
   (:require
+   [app.binfile.cleaner :as bfl]
    [app.binfile.common :as bfc]
+   [app.binfile.migrations :as bfm]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
+   [app.common.files.migrations :as-alias fmg]
    [app.common.json :as json]
    [app.common.logging :as l]
+   [app.common.media :as cmedia]
    [app.common.schema :as sm]
    [app.common.thumbnails :as cth]
    [app.common.types.color :as ctcl]
@@ -23,10 +26,12 @@
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
+   [app.common.types.tokens-lib :as cto]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.storage.impl :as sto.impl]
    [app.util.events :as events]
@@ -48,7 +53,7 @@
   [:map {:title "Manifest"}
    [:version ::sm/int]
    [:type :string]
-
+   [:referer {:optional true} :string]
    [:generated-by {:optional true} :string]
 
    [:files
@@ -56,7 +61,6 @@
      [:map
       [:id ::sm/uuid]
       [:name :string]
-      [:project-id ::sm/uuid]
       [:features ::cfeat/features]]]]
 
    [:relations {:optional true}
@@ -69,7 +73,7 @@
    [:size ::sm/int]
    [:content-type :string]
    [:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
-   [:hash :string]])
+   [:hash {:optional true} :string]])
 
 (def ^:private schema:file-thumbnail
   [:map {:title "FileThumbnail"}
@@ -79,13 +83,24 @@
    [:tag :string]
    [:media-id ::sm/uuid]])
 
+(def ^:private schema:file
+  [:merge
+   ctf/schema:file
+   [:map [:options {:optional true} ctf/schema:options]]])
+
+;; --- HELPERS
+
+(defn- default-now
+  [o]
+  (or o (dt/now)))
+
 ;; --- ENCODERS
 
 (def encode-file
-  (sm/encoder ::ctf/file sm/json-transformer))
+  (sm/encoder schema:file sm/json-transformer))
 
 (def encode-page
-  (sm/encoder ::ctp/page sm/json-transformer))
+  (sm/encoder ctp/schema:page sm/json-transformer))
 
 (def encode-shape
   (sm/encoder ::cts/shape sm/json-transformer))
@@ -97,11 +112,14 @@
   (sm/encoder ::ctc/component sm/json-transformer))
 
 (def encode-color
-  (sm/encoder ::ctcl/color sm/json-transformer))
+  (sm/encoder ctcl/schema:library-color sm/json-transformer))
 
 (def encode-typography
   (sm/encoder ::cty/typography sm/json-transformer))
 
+(def encode-tokens-lib
+  (sm/encoder ::cto/tokens-lib sm/json-transformer))
+
 (def encode-plugin-data
   (sm/encoder ::ctpg/plugin-data sm/json-transformer))
 
@@ -117,16 +135,16 @@
   (sm/decoder schema:manifest sm/json-transformer))
 
 (def decode-media
-  (sm/decoder ::ctf/media sm/json-transformer))
+  (sm/decoder ctf/schema:media sm/json-transformer))
 
 (def decode-component
   (sm/decoder ::ctc/component sm/json-transformer))
 
 (def decode-color
-  (sm/decoder ::ctcl/color sm/json-transformer))
+  (sm/decoder ctcl/schema:library-color sm/json-transformer))
 
 (def decode-file
-  (sm/decoder ::ctf/file sm/json-transformer))
+  (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
   (sm/decoder ::ctp/page sm/json-transformer))
@@ -137,6 +155,9 @@
 (def decode-typography
   (sm/decoder ::cty/typography sm/json-transformer))
 
+(def decode-tokens-lib
+  (sm/decoder cto/schema:tokens-lib sm/json-transformer))
+
 (def decode-plugin-data
   (sm/decoder ::ctpg/plugin-data sm/json-transformer))
 
@@ -164,7 +185,7 @@
   (sm/check-fn ::ctf/media))
 
 (def validate-color
-  (sm/check-fn ::ctcl/color))
+  (sm/check-fn ctcl/schema:library-color))
 
 (def validate-component
   (sm/check-fn ::ctc/component))
@@ -172,6 +193,9 @@
 (def validate-typography
   (sm/check-fn ::cty/typography))
 
+(def validate-tokens-lib
+  (sm/check-fn ::cto/tokens-lib))
+
 (def validate-plugin-data
   (sm/check-fn ::ctpg/plugin-data))
 
@@ -192,37 +216,24 @@
   (.closeEntry output))
 
 (defn- get-file
-  [{:keys [::embed-assets ::include-libraries] :as cfg} file-id]
+  [{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id]
 
   (when (and include-libraries embed-assets)
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
   (let [detach?  (and (not embed-assets) (not include-libraries))]
-    (cond-> (bfc/get-file cfg file-id)
-      detach?
-      (-> (ctf/detach-external-references file-id)
-          (dissoc :libraries))
+    (db/tx-run! cfg (fn [cfg]
+                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
+                        detach?
+                        (-> (ctf/detach-external-references file-id)
+                            (dissoc :libraries))
 
-      embed-assets
-      (update :data #(bfc/embed-assets cfg % file-id))
+                        embed-assets
+                        (update :data #(bfc/embed-assets cfg % file-id))
 
-      :always
-      (bfc/clean-file-features))))
-
-(defn- resolve-extension
-  [mtype]
-  (case mtype
-    "image/png"     ".png"
-    "image/jpeg"    ".jpg"
-    "image/gif"     ".gif"
-    "image/svg+xml" ".svg"
-    "image/webp"    ".webp"
-    "font/woff"     ".woff"
-    "font/woff2"    ".woff2"
-    "font/ttf"      ".ttf"
-    "font/otf"      ".otf"
-    "application/octet-stream" ".bin"))
+                        :always
+                        (bfc/clean-file-features))))))
 
 (defn- export-storage-objects
   [{:keys [::output] :as cfg}]
@@ -230,7 +241,7 @@
     (doseq [id (-> bfc/*state* deref :storage-objects not-empty)]
       (let [sobject (sto/get-object storage id)
             smeta   (meta sobject)
-            ext     (resolve-extension (:content-type smeta))
+            ext     (cmedia/mtype->extension (:content-type smeta))
             path    (str "objects/" id ".json")
             params  (-> (meta sobject)
                         (assoc :id (:id sobject))
@@ -247,6 +258,7 @@
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
   (let [file         (get-file cfg file-id)
+
         media        (->> (bfc/get-file-media cfg file)
                           (map (fn [media]
                                  (dissoc media :file-id))))
@@ -255,6 +267,7 @@
         typographies (:typographies data)
         components   (:components data)
         colors       (:colors data)
+        tokens-lib   (:tokens-lib data)
 
         pages        (:pages data)
         pages-index  (:pages-index data)
@@ -263,13 +276,16 @@
 
     (vswap! bfc/*state* update :files assoc file-id
             {:id file-id
-             :project-id (:project-id file)
              :name (:name file)
              :features (:features file)})
 
-    (let [file (cond-> (dissoc file :data)
+    (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
                  (assoc :options (:options data))
+
+                 :always
+                 (dissoc :data)
+
                  :always
                  (encode-file))
           path (str "files/" file-id ".json")]
@@ -325,12 +341,17 @@
         (write-entry! output path color)))
 
     (doseq [[id object] typographies]
-      (let [path  (str "files/" file-id "/typographies/" id ".json")
-            color (encode-typography object)]
-        (write-entry! output path color)))))
+      (let [path       (str "files/" file-id "/typographies/" id ".json")
+            typography (encode-typography object)]
+        (write-entry! output path typography)))
+
+    (when tokens-lib
+      (let [path           (str "files/" file-id "/tokens.json")
+            encoded-tokens (encode-tokens-lib tokens-lib)]
+        (write-entry! output path encoded-tokens)))))
 
 (defn- export-files
-  [{:keys [::ids ::include-libraries ::output] :as cfg}]
+  [{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}]
   (let [ids  (into ids (when include-libraries (bfc/get-libraries cfg ids)))
         rels (if include-libraries
                (->> (bfc/get-files-rels cfg ids)
@@ -351,6 +372,7 @@
           params {:type "penpot/export-files"
                   :version 1
                   :generated-by (str "penpot/" (:full cf/version))
+                  :refer "penpot"
                   :files (vec (vals files))
                   :relations rels}]
       (write-entry! output "manifest.json" params))))
@@ -463,6 +485,14 @@
         {:entry entry
          :id (parse-uuid id)}))))
 
+(defn- match-tokens-lib-entry-fn
+  [file-id]
+  (let [pattern (str "^files/" file-id "/tokens.json$")
+        pattern (re-pattern pattern)]
+    (fn [entry]
+      (when-let [[_] (re-matches pattern (zip-entry-name entry))]
+        {:entry entry}))))
+
 (defn- match-thumbnail-entry-fn
   [file-id]
   (let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+)/([^/]+).json$")
@@ -496,7 +526,7 @@
 
 (defn- match-storage-entry-fn
   []
-  (let [pattern (str "^objects/([^/]+).json$")
+  (let [pattern "^objects/([^/]+).json$"
         pattern (re-pattern pattern)]
     (fn [entry]
       (when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
@@ -508,8 +538,13 @@
   (with-open [reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
+(defn- read-plain-entry
+  [^ZipFile input entry]
+  (with-open [reader (zip-entry-reader input entry)]
+    (json/read reader)))
+
 (defn- read-file
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
@@ -517,7 +552,7 @@
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::input ::file-id]}]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -526,13 +561,19 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
                                    (decode-media)
                                    (validate-media))
-                       object (assoc object :file-id file-id)]
+                       object (-> object
+                                  (assoc :file-id file-id)
+                                  (update :created-at default-now)
+                                  ;; FIXME: this is set default to true for
+                                  ;; setting a value, this prop is no longer
+                                  ;; relevant;
+                                  (assoc :is-local true))]
                    (if (= id (:id object))
                      (conj result object)
                      result)))
@@ -540,7 +581,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -553,20 +594,38 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::input ::file-id ::entries]}]
-  (->> (keep (match-component-entry-fn file-id) entries)
-       (reduce (fn [result {:keys [id entry]}]
-                 (let [object (->> (read-entry input entry)
-                                   (decode-component)
-                                   (validate-component))]
-                   (if (= id (:id object))
-                     (assoc result id object)
-                     result)))
-               {})
-       (not-empty)))
+  [{:keys [::bfc/input ::file-id ::entries]}]
+  (let [clean-component-post-decode
+        (fn [component]
+          (d/update-when component :objects
+                         (fn [objects]
+                           (reduce-kv (fn [objects id shape]
+                                        (assoc objects id (bfl/clean-shape-post-decode shape)))
+                                      objects
+                                      objects))))
+        clean-component-pre-decode
+        (fn [component]
+          (d/update-when component :objects
+                         (fn [objects]
+                           (reduce-kv (fn [objects id shape]
+                                        (assoc objects id (bfl/clean-shape-pre-decode shape)))
+                                      objects
+                                      objects))))]
+
+    (->> (keep (match-component-entry-fn file-id) entries)
+         (reduce (fn [result {:keys [id entry]}]
+                   (let [object (->> (read-entry input entry)
+                                     (clean-component-pre-decode)
+                                     (decode-component)
+                                     (clean-component-post-decode))]
+                     (if (= id (:id object))
+                       (assoc result id object)
+                       result)))
+                 {})
+         (not-empty))))
 
 (defn- read-file-typographies
-  [{:keys [::input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -578,13 +637,21 @@
                {})
        (not-empty)))
 
+(defn- read-file-tokens-lib
+  [{:keys [::bfc/input ::file-id ::entries]}]
+  (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
+    (->> (read-plain-entry input entry)
+         (decode-tokens-lib)
+         (validate-tokens-lib))))
+
 (defn- read-file-shapes
-  [{:keys [::input ::file-id ::page-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
+                                   (bfl/clean-shape-pre-decode)
                                    (decode-shape)
-                                   (validate-shape))]
+                                   (bfl/clean-shape-post-decode))]
                    (if (= id (:id object))
                      (assoc result id object)
                      result)))
@@ -592,7 +659,7 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
@@ -608,7 +675,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -623,22 +690,23 @@
        (not-empty)))
 
 (defn- read-file-data
-  [{:keys [] :as cfg}]
+  [cfg]
   (let [colors       (read-file-colors cfg)
         typographies (read-file-typographies cfg)
+        tokens-lib   (read-file-tokens-lib cfg)
         components   (read-file-components cfg)
         plugin-data  (read-file-plugin-data cfg)
         pages        (read-file-pages cfg)]
-
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
      :typographies typographies
+     :tokens-lib tokens-lib
      :components components
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::db/conn ::project-id ::file-id ::file-name] :as cfg}]
+  [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
   (let [file-id'   (bfc/lookup-index file-id)
         file       (read-file cfg)
         media      (read-file-media cfg)
@@ -678,20 +746,18 @@
                    (cond-> (:options file)
                      (assoc :options (:options file))))
 
-          file (-> file
+          file (-> (select-keys file bfc/file-attrs)
                    (assoc :id file-id')
                    (assoc :data data)
                    (assoc :name file-name)
                    (assoc :project-id project-id)
-                   (dissoc :options)
-                   (bfc/process-file))]
+                   (dissoc :options))
 
-      (->> file
-           (bfc/register-pending-migrations cfg)
-           (bfc/persist-file! cfg))
+          file  (bfc/process-file cfg file)
+          file  (ctf/check-file file)]
 
-      (when (::bfc/overwrite cfg)
-        (db/delete! conn :file-thumbnail {:file-id file-id'}))
+      (bfm/register-pending-migrations! cfg file)
+      (bfc/save-file! cfg file ::db/return-keys false)
 
       file-id')))
 
@@ -714,7 +780,7 @@
                      :library-file-id libr-id})))))
 
 (defn- import-storage-objects
-  [{:keys [::input ::entries ::bfc/timestamp] :as cfg}]
+  [{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}]
   (events/tap :progress {:section :storage-objects})
 
   (let [storage (sto/resolve cfg)
@@ -732,7 +798,7 @@
                     :expected-id (str id)
                     :found-id (str (:id object))))
 
-        (let [ext     (resolve-extension (:content-type object))
+        (let [ext     (cmedia/mtype->extension (:content-type object))
               path    (str "objects/" id ext)
               content (->> path
                            (get-zip-entry input)
@@ -746,13 +812,14 @@
                       :expected-size (:size object)
                       :found-size (sto/get-size content)))
 
-          (when (not= (:hash object) (sto/get-hash content))
-            (ex/raise :type :validation
-                      :code :inconsistent-penpot-file
-                      :hint "found corrupted storage object: hash does not match"
-                      :path path
-                      :expected-hash (:hash object)
-                      :found-hash (sto/get-hash content)))
+          (when-let [hash (get object :hash)]
+            (when (not= hash (sto/get-hash content))
+              (ex/raise :type :validation
+                        :code :inconsistent-penpot-file
+                        :hint "found corrupted storage object: hash does not match"
+                        :path path
+                        :expected-hash (:hash object)
+                        :found-hash (sto/get-hash content))))
 
           (let [params  (-> object
                             (dissoc :id :size)
@@ -781,12 +848,12 @@
                      (d/update-when :thumbnail-id bfc/lookup-index))]
 
       (l/dbg :hint "inserting file media object"
+             :old-id (str (:id item))
              :id (str (:id params))
              :file-id (str (:file-id params))
              ::l/sync? true)
 
-      (db/insert! conn :file-media-object params
-                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+      (db/insert! conn :file-media-object params))))
 
 (defn- import-file-thumbnails
   [{:keys [::db/conn] :as cfg}]
@@ -806,19 +873,13 @@
              :media-id (str media-id)
              ::l/sync? true)
 
-      (db/insert! conn :file-tagged-object-thumbnail params
-                  {::db/on-conflict-do-nothing? (::bfc/overwrite cfg)}))))
+      (db/insert! conn :file-tagged-object-thumbnail params))))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::input ::name] :or {timestamp (dt/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
 
-  (dm/assert!
-   "expected zip file"
-   (instance? ZipFile input))
-
-  (dm/assert!
-   "expected valid instant"
-   (dt/instant? timestamp))
+  (assert (instance? ZipFile input) "expected zip file")
+  (assert (dt/instant? timestamp) "expected valid instant")
 
   (let [manifest (-> (read-manifest input)
                      (validate-manifest))
@@ -830,15 +891,19 @@
                 :hint "unexpected type on manifest"
                 :manifest manifest))
 
+
     ;; Check if all files referenced on manifest are present
-    (doseq [{file-id :id} (:files manifest)]
+    (doseq [{file-id :id features :features} (:files manifest)]
       (let [path (str "files/" file-id ".json")]
+
         (when-not (get-zip-entry input path)
           (ex/raise :type :validation
                     :code :invalid-binfile-v3
                     :hint "some files referenced on manifest not found"
                     :path path
-                    :file-id file-id))))
+                    :file-id file-id))
+
+        (cfeat/check-supported-features! features)))
 
     (events/tap :progress {:section :manifest})
 
@@ -868,7 +933,7 @@
                             (import-file-media cfg)
                             (import-file-thumbnails cfg)
 
-                            (bfc/apply-pending-migrations! cfg)
+                            (bfm/apply-pending-migrations! cfg)
 
                             ids)))))))
 
@@ -878,23 +943,22 @@
   "Do the exportation of a specified file in custom penpot binary
   format. There are some options available for customize the output:
 
-  `::include-libraries`: additionally to the specified file, all the
+  `::bfc/include-libraries`: additionally to the specified file, all the
   linked libraries also will be included (including transitive
   dependencies).
 
-  `::embed-assets`: instead of including the libraries, embed in the
+  `::bfc/embed-assets`: instead of including the libraries, embed in the
   same file library all assets used from external libraries."
 
-  [{:keys [::ids] :as cfg} output]
+  [{:keys [::bfc/ids] :as cfg} output]
 
-  (dm/assert!
-   "expected a set of uuid's for `::ids` parameter"
-   (and (set? ids)
-        (every? uuid? ids)))
+  (assert
+   (and (set? ids) (every? uuid? ids))
+   "expected a set of uuid's for `::bfc/ids` parameter")
 
-  (dm/assert!
-   "expected instance of jio/IOFactory for `input`"
-   (satisfies? jio/IOFactory output))
+  (assert
+   (satisfies? jio/IOFactory output)
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
         tp (dt/tpoint)
@@ -930,18 +994,17 @@
                 :aborted @ab
                 :cause @cs)))))
 
-
 (defn import-files!
-  [{:keys [::input] :as cfg}]
+  [{:keys [::bfc/input] :as cfg}]
 
-  (dm/assert!
-   "expected valid profile-id and project-id on `cfg`"
-   (and (uuid? (::profile-id cfg))
-        (uuid? (::project-id cfg))))
+  (assert
+   (and (uuid? (::bfc/profile-id cfg))
+        (uuid? (::bfc/project-id cfg)))
+   "expected valid profile-id and project-id on `cfg`")
 
-  (dm/assert!
-   "expected instance of jio/IOFactory for `input`"
-   (io/coercible? input))
+  (assert
+   (io/coercible? input)
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
         tp (dt/tpoint)
@@ -950,7 +1013,7 @@
     (l/info :hint "import: started" :id (str id))
     (try
       (with-open [input (ZipFile. (fs/file input))]
-        (import-files (assoc cfg ::input input)))
+        (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
         (vreset! cs cause)
@@ -961,3 +1024,9 @@
                 :id (str id)
                 :elapsed (dt/format-duration (tp))
                 :error? (some? @cs))))))
+
+(defn get-manifest
+  [path]
+  (with-open [input (ZipFile. (fs/file path))]
+    (-> (read-manifest input)
+        (validate-manifest))))

backend/src/app/config.clj

@@ -12,6 +12,7 @@
    [app.common.exceptions :as ex]
    [app.common.flags :as flags]
    [app.common.schema :as sm]
+   [app.common.uri :as u]
    [app.common.version :as v]
    [app.util.overrides]
    [app.util.time :as dt]
@@ -228,19 +229,16 @@
     [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
     [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
-(def default-flags
-  [:enable-backend-api-doc
-   :enable-backend-openapi-doc
-   :enable-backend-worker
-   :enable-secure-session-cookies
-   :enable-email-verification
-   :enable-v2-migration])
-
 (defn- parse-flags
   [config]
-  (flags/parse flags/default
-               default-flags
-               (:flags config)))
+  (let [public-uri  (c/get config :public-uri)
+        public-uri  (some-> public-uri (u/uri))
+        extra-flags (if (and public-uri
+                             (= (:scheme public-uri) "http")
+                             (not= (:host public-uri) "localhost"))
+                      #{:disable-secure-session-cookies}
+                      #{})]
+    (flags/parse flags/default extra-flags (:flags config))))
 
 (defn read-env
   [prefix]

backend/src/app/db.clj

@@ -22,7 +22,8 @@
    [clojure.set :as set]
    [integrant.core :as ig]
    [next.jdbc :as jdbc]
-   [next.jdbc.date-time :as jdbc-dt])
+   [next.jdbc.date-time :as jdbc-dt]
+   [next.jdbc.transaction])
   (:import
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
@@ -41,6 +42,8 @@
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
+(def ^:dynamic *conn* nil)
+
 (declare open)
 (declare create-pool)
 
@@ -223,16 +226,6 @@
     (let [^OutputStream os (.getOutputStream ^LargeObject lobj)]
       (io/make-output-stream os opts))))
 
-(defmacro with-atomic
-  [& args]
-  (if (symbol? (first args))
-    (let [cfgs (first args)
-          body (rest args)]
-      `(jdbc/with-transaction [conn# (::pool ~cfgs)]
-         (let [~cfgs (assoc ~cfgs ::conn conn#)]
-           ~@body)))
-    `(jdbc/with-transaction ~@args)))
-
 (defn open
   [system-or-pool]
   (if (pool? system-or-pool)
@@ -413,7 +406,7 @@
 
 (def ^:private default-plan-opts
   (-> default-opts
-      (assoc :fetch-size 1)
+      (assoc :fetch-size 1000)
       (assoc :concurrency :read-only)
       (assoc :cursors :close)
       (assoc :result-type :forward-only)))
@@ -535,43 +528,31 @@
      (l/trc :hint "explicit rollback requested (savepoint)")
      (.rollback conn sp))))
 
+(defn transact!
+  "A lower-level function for executing function in a transaction"
+  ([transactable f] (transact! transactable f {}))
+  ([transactable f opts]
+   (binding [next.jdbc.transaction/*nested-tx* :ignore]
+     (jdbc/transact transactable f opts))))
+
 (defn tx-run!
+  "Run a function in a transaction."
   [system f & params]
-  (cond
-    (connection? system)
+  (if (connection? system)
     (tx-run! {::conn system} f)
-
-    (pool? system)
-    (tx-run! {::pool system} f)
-
-    (::conn system)
-    (let [conn (::conn system)
-          sp   (savepoint conn)]
-      (try
-        (let [system' (-> system
-                          (assoc ::savepoint sp)
-                          (dissoc ::rollback))
-              result  (apply f system' params)]
-          (if (::rollback system)
-            (rollback! conn sp)
-            (release! conn sp))
-          result)
-        (catch Throwable cause
-          (.rollback ^Connection conn ^Savepoint sp)
-          (throw cause))))
-
-    (::pool system)
-    (with-atomic [conn (::pool system)]
-      (let [system' (-> system
-                        (assoc ::conn conn)
-                        (dissoc ::rollback))
-            result  (apply f system' params)]
-        (when (::rollback system)
-          (rollback! conn))
-        result))
-
-    :else
-    (throw (IllegalArgumentException. "invalid system/cfg provided"))))
+    (if (pool? system)
+      (tx-run! {::pool system} f)
+      (if-let [conn (or (::conn system)
+                        (::pool system))]
+        (transact! conn
+                   (fn [conn]
+                     (let [system' (-> system
+                                       (dissoc ::rollback)
+                                       (assoc ::conn conn))]
+                       (apply f system' params)))
+                   {:rollback-only (::rollback system)
+                    :read-only (::read-only system)})
+        (throw (IllegalArgumentException. "invalid system/cfg provided"))))))
 
 (defn run!
   [system f & params]

backend/src/app/db/sql.clj

@@ -39,7 +39,10 @@
 
 (defn insert-many
   [table cols rows opts]
-  (let [opts (merge default-opts opts)]
+  (let [opts (merge default-opts opts)
+        opts (cond-> opts
+               (::on-conflict-do-nothing opts)
+               (assoc :suffix "ON CONFLICT DO NOTHING"))]
     (sql/for-insert-multi table cols rows opts)))
 
 (defn select

backend/src/app/email.clj

@@ -449,6 +449,45 @@
    :id ::request-team-access
    :schema schema:request-team-access))
 
+(def ^:private schema:comment-mention
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-mention
+  (template-factory
+   :id ::comment-mention
+   :schema schema:comment-mention))
+
+(def ^:private schema:comment-thread
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-thread
+  (template-factory
+   :id ::comment-thread
+   :schema schema:comment-thread))
+
+(def ^:private schema:comment-notification
+  [:map
+   [:name ::sm/text]
+   [:source-user ::sm/text]
+   [:comment-reference ::sm/text]
+   [:comment-content ::sm/text]
+   [:comment-url ::sm/text]])
+
+(def comment-notification
+  (template-factory
+   :id ::comment-notification
+   :schema schema:comment-notification))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; BOUNCE/COMPLAINS HELPERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/features/fdata.clj

@@ -9,7 +9,10 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.files.helpers :as cfh]
+   [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
+   [app.common.types.path :as path]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
@@ -30,7 +33,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn enable-objects-map
-  [file]
+  [file & _opts]
   (let [update-page
         (fn [page]
           (if (and (pmap/pointer-map? page)
@@ -136,10 +139,56 @@
 
 (defn enable-pointer-map
   "Enable the fdata/pointer-map feature on the file."
-  [file]
+  [file & _opts]
   (-> file
       (update :data (fn [fdata]
                       (-> fdata
                           (update :pages-index d/update-vals pmap/wrap)
                           (d/update-when :components pmap/wrap))))
       (update :features conj "fdata/pointer-map")))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PATH-DATA
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn enable-path-data
+  "Enable the fdata/path-data feature on the file."
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content path/content)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features conj "fdata/path-data"))))
+
+(defn disable-path-data
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content vec)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (when-let [conn db/*conn*]
+      (db/delete! conn :file-migration {:file-id (:id file)
+                                        :name "0003-convert-path-content"}))
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features disj "fdata/path-data")
+        (update :migrations disj "0003-convert-path-content")
+        (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))

backend/src/app/features/file_migrations.clj

@@ -0,0 +1,39 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.file-migrations
+  "Backend specific code for file migrations. Implemented as permanent feature of files."
+  (:require
+   [app.common.data :as d]
+   [app.common.files.migrations :as fmg :refer [xf:map-name]]
+   [app.db :as db]
+   [app.db.sql :as-alias sql]))
+
+(def ^:private sql:get-file-migrations
+  "SELECT name FROM file_migration WHERE file_id = ? ORDER BY created_at ASC")
+
+(defn resolve-applied-migrations
+  [cfg {:keys [id] :as file}]
+  (let [conn (db/get-connection cfg)]
+    (assoc file :migrations
+           (->> (db/plan conn [sql:get-file-migrations id])
+                (transduce xf:map-name conj (d/ordered-set))
+                (not-empty)))))
+
+(defn upsert-migrations!
+  "Persist or update file migrations. Return the updated/inserted number
+  of rows"
+  [conn {:keys [id] :as file}]
+  (let [migrations (or (-> file meta ::fmg/migrated)
+                       (-> file :migrations not-empty)
+                       fmg/available-migrations)
+        columns    [:file-id :name]
+        rows       (mapv (fn [name] [id name]) migrations)]
+
+    (-> (db/insert-many! conn :file-migration columns rows
+                         {::db/return-keys false
+                          ::sql/on-conflict-do-nothing true})
+        (db/get-update-count))))

backend/src/app/features/logical_deletion.clj

@@ -0,0 +1,31 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.logical-deletion
+  "A code related to handle logical deletion mechanism"
+  (:require
+   [app.config :as cf]
+   [app.util.time :as dt]))
+
+(defn get-deletion-delay
+  "Calculate the next deleted-at for a resource (file, team, etc) in function
+  of team settings"
+  [team]
+  (if-let [subscription (get team :subscription)]
+    (cond
+      (and (= (:type subscription) "unlimited")
+           (= (:status subscription) "active"))
+      (dt/duration {:days 30})
+
+      (and (= (:type subscription) "enterprise")
+           (= (:status subscription) "active"))
+      (dt/duration {:days 90})
+
+      :else
+      (cf/get-deletion-delay))
+
+    (cf/get-deletion-delay)))
+

backend/src/app/http.clj

@@ -42,8 +42,8 @@
 (def default-params
   {::port 6060
    ::host "0.0.0.0"
-   ::max-body-size (* 1024 1024 30)             ; default 30 MiB
-   ::max-multipart-body-size (* 1024 1024 120)}) ; default 120 MiB
+   ::max-body-size 31457280              ; default  30 MiB
+   ::max-multipart-body-size 367001600}) ; default 350 MiB
 
 (defmethod ig/expand-key ::server
   [k v]
@@ -155,10 +155,10 @@
    [["" {:middleware [[mw/server-timing]
                       [mw/params]
                       [mw/format-response]
-                      [mw/parse-request]
-                      [mw/errors errors/handle]
                       [session/soft-auth cfg]
                       [actoken/soft-auth cfg]
+                      [mw/parse-request]
+                      [mw/errors errors/handle]
                       [mw/restrict-methods]]}
 
      (::mtx/routes cfg)

backend/src/app/http/debug.clj

@@ -7,9 +7,12 @@
 (ns app.http.debug
   (:refer-clojure :exclude [error-handler])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
    [app.common.uuid :as uuid]
@@ -19,8 +22,9 @@
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
+   [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
-   [app.srepl.helpers :as srepl]
+   [app.srepl.main :as srepl]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [app.util.blob :as blob]
@@ -280,23 +284,23 @@
       (ex/raise :type :validation
                 :code :missing-arguments))
 
-    (let [path (tmp/tempfile :prefix "penpot.export.")]
+    (let [path (tmp/tempfile :prefix "penpot.export." :min-age "30m")]
       (with-open [output (io/output-stream path)]
         (-> cfg
-            (assoc ::bf.v1/ids file-ids)
-            (assoc ::bf.v1/embed-assets embed?)
-            (assoc ::bf.v1/include-libraries libs?)
-            (bf.v1/export-files! output)))
+            (assoc ::bfc/ids file-ids)
+            (assoc ::bfc/embed-assets embed?)
+            (assoc ::bfc/include-libraries libs?)
+            (bf.v3/export-files! output)))
 
       (if clone?
         (let [profile    (profile/get-profile pool profile-id)
               project-id (:default-project-id profile)
               cfg        (assoc cfg
-                                ::bf.v1/overwrite false
-                                ::bf.v1/profile-id profile-id
-                                ::bf.v1/project-id project-id
-                                ::bf.v1/input path)]
-          (bf.v1/import-files! cfg)
+                                ::bfc/overwrite false
+                                ::bfc/profile-id profile-id
+                                ::bfc/project-id project-id
+                                ::bfc/input path)]
+          (bf.v3/import-files! cfg)
           {::yres/status  200
            ::yres/headers {"content-type" "text/plain"}
            ::yres/body    "OK CLONED"})
@@ -316,22 +320,27 @@
 
   (let [profile    (profile/get-profile pool profile-id)
         project-id (:default-project-id profile)
-        overwrite? (contains? params :overwrite)
-        migrate?   (contains? params :migrate)]
+        team       (teams/get-team pool
+                                   :profile-id profile-id
+                                   :project-id project-id)]
 
     (when-not project-id
       (ex/raise :type :validation
                 :code :missing-project
                 :hint "project not found"))
 
-    (let [path (-> params :file :path)
-          cfg  (assoc cfg
-                      ::bf.v1/overwrite overwrite?
-                      ::bf.v1/migrate migrate?
-                      ::bf.v1/profile-id profile-id
-                      ::bf.v1/project-id project-id
-                      ::bf.v1/input path)]
-      (bf.v1/import-files! cfg)
+    (let [path   (-> params :file :path)
+          format (bfc/parse-file-format path)
+          cfg    (assoc cfg
+                        ::bfc/profile-id profile-id
+                        ::bfc/project-id project-id
+                        ::bfc/input path
+                        ::bfc/features (cfeat/get-team-enabled-features cf/flags team))]
+
+      (if (= format :binfile-v3)
+        (bf.v3/import-files! cfg)
+        (bf.v1/import-files! cfg))
+
       {::yres/status  200
        ::yres/headers {"content-type" "text/plain"}
        ::yres/body    "OK"})))
@@ -421,6 +430,50 @@
      ::yres/body    "OK"}))
 
 
+(defn- add-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id (some-> params :team-id d/parse-uuid)
+        feature (some-> params :feature str)
+        skip-check (contains? params :skip-check)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? team-id)
+      (ex/raise :type :validation
+                :code :invalid-team-id
+                :hint "provided invalid team id"))
+
+    (srepl/enable-team-feature! team-id feature :skip-check skip-check)
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
+(defn- remove-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id   (some-> params :team-id d/parse-uuid)
+        feature   (some-> params :feature str)
+        skip-check (contains? params :skip-check)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? team-id)
+      (ex/raise :type :validation
+                :code :invalid-team-id
+                :hint "provided invalid team id"))
+
+    (srepl/disable-team-feature! team-id feature :skip-check skip-check)
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -491,6 +544,10 @@
      {:handler (partial resend-email-notification cfg)}]
     ["/actions/reset-file-version"
      {:handler (partial reset-file-version cfg)}]
+    ["/actions/add-team-feature"
+     {:handler (partial add-team-feature)}]
+    ["/actions/remove-team-feature"
+     {:handler (partial remove-team-feature)}]
     ["/file/export" {:handler (partial export-handler cfg)}]
     ["/file/import" {:handler (partial import-handler cfg)}]
     ["/file/data" {:handler (partial file-data-handler cfg)}]

backend/src/app/http/errors.clj

@@ -25,7 +25,6 @@
   (let [claims (-> {}
                    (into (::session/token-claims request))
                    (into (::actoken/token-claims request)))]
-
     {:request/path       (:path request)
      :request/method     (:method request)
      :request/params     (:params request)
@@ -55,13 +54,17 @@
    ::yres/body (ex-data err)})
 
 (defmethod handle-error :restriction
-  [err _ _]
+  [err request _]
   (let [{:keys [code] :as data} (ex-data err)]
     (if (= code :method-not-allowed)
       {::yres/status 405
        ::yres/body data}
-      {::yres/status 400
-       ::yres/body data})))
+
+      (binding [l/*context* (request->context request)]
+        (l/err :hint "restriction error"
+               :cause err)
+        {::yres/status 400
+         ::yres/body data}))))
 
 (defmethod handle-error :rate-limit
   [err _ _]
@@ -99,7 +102,7 @@
       (= code :invalid-image)
       (binding [l/*context* (request->context request)]
         (let [cause (or parent-cause err)]
-          (l/warn :hint "unexpected error on processing image" :cause cause)
+          (l/warn :hint "image process error" :cause cause)
           {::yres/status 400 ::yres/body data}))
 
       :else
@@ -174,7 +177,7 @@
   (let [state (.getSQLState ^java.sql.SQLException error)
         cause (or parent-cause error)]
     (binding [l/*context* (request->context request)]
-      (l/error :hint "PSQL error"
+      (l/error :hint "postgresql error"
                :cause cause)
       (cond
         (= state "57014")

backend/src/app/http/session.clj

@@ -337,16 +337,17 @@
        or (updated_at is null and
            created_at < now() - ?::interval)")
 
-(defmethod ig/init-key ::tasks/gc
-  [_ {:keys [::db/pool ::tasks/max-age] :as cfg}]
-  (l/debug :hint "initializing session gc task" :max-age max-age)
-  (fn [_]
-    (db/with-atomic [conn pool]
-      (let [interval (db/interval max-age)
-            result   (db/exec-one! conn [sql:delete-expired interval interval])
-            result   (:next.jdbc/update-count result)]
-        (l/debug :task "gc"
-                 :hint "clean http sessions"
-                 :deleted result)
-        result))))
+(defn- collect-expired-tasks
+  [{:keys [::db/conn ::tasks/max-age]}]
+  (let [interval (db/interval max-age)
+        result   (db/exec-one! conn [sql:delete-expired interval interval])
+        result   (:next.jdbc/update-count result)]
+    (l/debug :task "gc"
+             :hint "clean http sessions"
+             :deleted result)
+    result))
 
+(defmethod ig/init-key ::tasks/gc
+  [_ {:keys [::tasks/max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
+  (fn [_] (db/tx-run! cfg collect-expired-tasks)))

backend/src/app/http/sse.clj

@@ -9,7 +9,6 @@
   (:refer-clojure :exclude [tap])
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.transit :as t]
    [app.http.errors :as errors]
@@ -54,17 +53,20 @@
      ::yres/status 200
      ::yres/body (yres/stream-body
                   (fn [_ output]
-                    (binding [events/*channel* (sp/chan :buf buf :xf (keep encode))]
-                      (let [listener (events/start-listener
-                                      (partial write! output)
-                                      (partial pu/close! output))]
-                        (try
+                    (let [channel  (sp/chan :buf buf :xf (keep encode))
+                          listener (events/start-listener
+                                    channel
+                                    (partial write! output)
+                                    (partial pu/close! output))]
+                      (try
+                        (binding [events/*channel* channel]
                           (let [result (handler)]
-                            (events/tap :end result))
-                          (catch Throwable cause
-                            (events/tap :error (errors/handle' cause request))
-                            (when-not (ex/instance? java.io.EOFException cause)
-                              (l/err :hint "unexpected error on processing sse response" :cause cause)))
-                          (finally
-                            (sp/close! events/*channel*)
-                            (px/await! listener)))))))}))
+                            (events/tap :end result)))
+
+                        (catch Throwable cause
+                          (let [result (errors/handle' cause request)]
+                            (events/tap channel :error result)))
+
+                        (finally
+                          (sp/close! channel)
+                          (px/await! listener))))))}))

backend/src/app/http/websocket.clj

@@ -273,7 +273,7 @@
 
 (defn- http-handler
   [cfg {:keys [params ::session/profile-id] :as request}]
-  (let [session-id (some-> params :session-id sm/parse-uuid)]
+  (let [session-id (some-> params :session-id uuid/parse*)]
     (when-not (uuid? session-id)
       (ex/raise :type :validation
                 :code :missing-session-id

backend/src/app/loggers/audit.clj

@@ -108,6 +108,7 @@
    [::ip-addr {:optional true} ::sm/text]
    [::props {:optional true} [:map-of :keyword :any]]
    [::context {:optional true} [:map-of :keyword :any]]
+   [::tracked-at {:optional true} ::sm/inst]
    [::webhooks/event? {:optional true} ::sm/boolean]
    [::webhooks/batch-timeout {:optional true} ::dt/duration]
    [::webhooks/batch-key {:optional true}
@@ -118,12 +119,12 @@
 
 (defn prepare-event
   [cfg mdata params result]
-  (let [resultm    (meta result)
-        request    (-> params meta ::http/request)
-        profile-id (or (::profile-id resultm)
-                       (:profile-id result)
-                       (::rpc/profile-id params)
-                       uuid/zero)
+  (let [resultm      (meta result)
+        request      (-> params meta ::http/request)
+        profile-id   (or (::profile-id resultm)
+                         (:profile-id result)
+                         (::rpc/profile-id params)
+                         uuid/zero)
 
         session-id   (get params ::rpc/external-session-id)
         event-origin (get params ::rpc/external-event-origin)
@@ -135,14 +136,14 @@
 
                          (clean-props))
 
-        token-id  (::actoken/id request)
-        context   (-> (::context resultm)
-                      (assoc :external-session-id session-id)
-                      (assoc :external-event-origin event-origin)
-                      (assoc :access-token-id (some-> token-id str))
-                      (d/without-nils))
+        token-id     (::actoken/id request)
+        context      (-> (::context resultm)
+                         (assoc :external-session-id session-id)
+                         (assoc :external-event-origin event-origin)
+                         (assoc :access-token-id (some-> token-id str))
+                         (d/without-nils))
 
-        ip-addr   (inet/parse-request request)]
+        ip-addr      (inet/parse-request request)]
 
     {::type (or (::type resultm)
                 (::rpc/type cfg))

backend/src/app/loggers/database.clj

@@ -41,7 +41,7 @@
   (if (or (instance? java.util.concurrent.CompletionException cause)
           (instance? java.util.concurrent.ExecutionException cause))
     (-> record
-        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
+        (assoc ::trace (ex/format-throwable cause :data? true :explain? false :header? false :summary? false))
         (assoc ::l/cause (ex-cause cause))
         (record->report))
 
@@ -53,30 +53,34 @@
                    (assoc :logger/name logger)
                    (assoc :logger/level level)
                    (dissoc :request/params :value :params :data))]
+
       (merge
        {:context (-> (into (sorted-map) ctx)
                      (pp/pprint-str :length 50))
         :props   (pp/pprint-str props :length 50)
-        :hint    (or (ex-message cause) @message)
+        :hint    (or (when-let [message (ex-message cause)]
+                       (if-let [props-hint (:hint props)]
+                         (str props-hint ": " message)
+                         message))
+                     @message)
         :trace   (or (::trace record)
-                     (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))}
+                     (some-> cause (ex/format-throwable :data? true :explain? false :header? false :summary? false)))}
 
        (when-let [params (or (:request/params context) (:params context))]
-         {:params (pp/pprint-str params :length 30 :level 13)})
+         {:params (pp/pprint-str params :length 20 :level 20)})
 
        (when-let [value (:value context)]
-         {:value (pp/pprint-str value :length 30 :level 12)})
+         {:value (pp/pprint-str value :length 30 :level 13)})
 
        (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-         {:data (pp/pprint-str data :length 30 :level 12)})
+         {:data (pp/pprint-str data :length 30 :level 13)})
 
-       (when-let [explain (ex/explain data :length 30 :level 12)]
+       (when-let [explain (ex/explain data :length 30 :level 13)]
          {:explain explain})))))
 
 (defn error-record?
-  [{:keys [::l/level ::l/cause]}]
-  (and (= :error level)
-       (ex/exception? cause)))
+  [{:keys [::l/level]}]
+  (= :error level))
 
 (defn- handle-event
   [{:keys [::db/pool]} {:keys [::l/id] :as record}]

backend/src/app/loggers/webhooks.clj

@@ -15,6 +15,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
+   [app.loggers.audit :as audit]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.data.json :as json]
@@ -67,18 +68,27 @@
 (defmethod ig/init-key ::process-event-handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (l/dbg :hint "process webhook event" :name (:name props))
 
-    (when-let [items (lookup-webhooks cfg props)]
-      (l/trc :hint "webhooks found for event" :total (count items))
-      (db/tx-run! cfg (fn [cfg]
-                        (doseq [item items]
-                          (wrk/submit! (-> cfg
-                                           (assoc ::wrk/task :run-webhook)
-                                           (assoc ::wrk/queue :webhooks)
-                                           (assoc ::wrk/max-retries 3)
-                                           (assoc ::wrk/params {:event props
-                                                                :config item})))))))))
+    (let [items (lookup-webhooks cfg props)
+          event {::audit/profile-id (:profile-id props)
+                 ::audit/name "webhook"
+                 ::audit/type "trigger"
+                 ::audit/props {:name (get props :name)
+                                :event-id (get props :id)
+                                :total-affected (count items)}}]
+
+      (audit/insert! cfg event)
+
+      (when items
+        (l/trc :hint "webhooks found for event" :total (count items))
+        (db/tx-run! cfg (fn [cfg]
+                          (doseq [item items]
+                            (wrk/submit! (-> cfg
+                                             (assoc ::wrk/task :run-webhook)
+                                             (assoc ::wrk/queue :webhooks)
+                                             (assoc ::wrk/max-retries 3)
+                                             (assoc ::wrk/params {:event props
+                                                                  :config item}))))))))))
 ;; --- RUN
 
 (declare interpret-exception)

backend/src/app/main.clj

@@ -25,7 +25,6 @@
    [app.loggers.webhooks :as-alias webhooks]
    [app.metrics :as-alias mtx]
    [app.metrics.definition :as-alias mdef]
-   [app.migrations.v2 :as migrations.v2]
    [app.msgbus :as-alias mbus]
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
@@ -41,7 +40,6 @@
    [app.svgo :as-alias svgo]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [cider.nrepl :refer [cider-nrepl-handler]]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -606,14 +604,9 @@
     (let [p (promise)]
       (when (contains? cf/flags :nrepl-server)
         (l/inf :hint "start nrepl server" :port 6064)
-        (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
+        (nrepl/start-server :bind "0.0.0.0" :port 6064))
 
       (start)
-
-      (when (contains? cf/flags :v2-migration)
-        (px/sleep 5000)
-        (migrations.v2/migrate app.main/system))
-
       (deref p))
     (catch Throwable cause
       (ex/print-throwable cause)

backend/src/app/media.clj

@@ -8,12 +8,11 @@
   "Media & Font postprocessing."
   (:require
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.schema.openapi :as-alias oapi]
-   [app.common.spec :as us]
-   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
@@ -22,42 +21,41 @@
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.shell :as sh]
-   [clojure.spec.alpha :as s]
+   [clojure.xml :as xml]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
+   clojure.lang.XMLHandler
+   java.io.InputStream
+   javax.xml.XMLConstants
+   javax.xml.parsers.SAXParserFactory
+   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(s/def ::path fs/path?)
-(s/def ::filename string?)
-(s/def ::size integer?)
-(s/def ::headers (s/map-of string? string?))
-(s/def ::mtype string?)
+(def schema:upload
+  (sm/register!
+   ^{::sm/type ::upload}
+   [:map {:title "Upload"}
+    [:filename :string]
+    [:size ::sm/int]
+    [:path ::fs/path]
+    [:mtype {:optional true} :string]
+    [:headers {:optional true}
+     [:map-of :string :string]]]))
 
-(s/def ::upload
-  (s/keys :req-un [::filename ::size ::path]
-          :opt-un [::mtype ::headers]))
+(def ^:private schema:input
+  [:map {:title "Input"}
+   [:path ::fs/path]
+   [:mtype {:optional true} ::sm/text]])
 
-;; A subset of fields from the ::upload spec
-(s/def ::input
-  (s/keys :req-un [::path]
-          :opt-un [::mtype]))
-
-(sm/register!
- ^{::sm/type ::upload}
- [:map {:title "Upload"}
-  [:filename :string]
-  [:size ::sm/int]
-  [:path ::fs/path]
-  [:mtype {:optional true} :string]
-  [:headers {:optional true}
-   [:map-of :string :string]]])
+(def ^:private check-input
+  (sm/check-fn schema:input))
 
 (defn validate-media-type!
-  ([upload] (validate-media-type! upload cm/valid-image-types))
+  ([upload] (validate-media-type! upload cm/image-types))
   ([upload allowed]
    (when-not (contains? allowed (:mtype upload))
      (ex/raise :type :validation
@@ -97,17 +95,44 @@
     (catch Throwable e
       (process-error e))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SVG PARSING
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- secure-parser-factory
+  [^InputStream input ^XMLHandler handler]
+  (.. (doto (SAXParserFactory/newInstance)
+        (.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
+        (.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
+      (newSAXParser)
+      (parse input handler)))
+
+(defn- strip-doctype
+  [data]
+  (cond-> data
+    (str/includes? data "<!DOCTYPE")
+    (str/replace #"<\!DOCTYPE[^>]*>" "")))
+
+(defn- parse-svg
+  [text]
+  (let [text (strip-doctype text)]
+    (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
+      (xml/parse istream secure-parser-factory))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; IMAGE THUMBNAILS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(s/def ::width integer?)
-(s/def ::height integer?)
-(s/def ::format #{:jpeg :webp :png})
-(s/def ::quality #(< 0 % 101))
+(def ^:private schema:thumbnail-params
+  [:map {:title "ThumbnailParams"}
+   [:input schema:input]
+   [:format [:enum :jpeg :webp :png]]
+   [:quality [:int {:min 1 :max 100}]]
+   [:width :int]
+   [:height :int]])
 
-(s/def ::thumbnail-params
-  (s/keys :req-un [::input ::format ::width ::height]))
+(def ^:private check-thumbnail-params
+  (sm/check-fn schema:thumbnail-params))
 
 ;; Related info on how thumbnails generation
 ;;  http://www.imagemagick.org/Usage/thumbnails/
@@ -129,30 +154,38 @@
            :data   tmp)))
 
 (defmethod process :generic-thumbnail
-  [{:keys [quality width height] :as params}]
-  (us/assert ::thumbnail-params params)
-  (let [op (doto (IMOperation.)
-             (.addImage)
-             (.autoOrient)
-             (.strip)
-             (.thumbnail ^Integer (int width) ^Integer (int height) ">")
-             (.quality (double quality))
-             (.addImage))]
-    (generic-process (assoc params :operation op))))
+  [params]
+  (let [{:keys [quality width height] :as params}
+        (check-thumbnail-params params)
+
+        operation
+        (doto (IMOperation.)
+          (.addImage)
+          (.autoOrient)
+          (.strip)
+          (.thumbnail ^Integer (int width) ^Integer (int height) ">")
+          (.quality (double quality))
+          (.addImage))]
+
+    (generic-process (assoc params :operation operation))))
 
 (defmethod process :profile-thumbnail
-  [{:keys [quality width height] :as params}]
-  (us/assert ::thumbnail-params params)
-  (let [op (doto (IMOperation.)
-             (.addImage)
-             (.autoOrient)
-             (.strip)
-             (.thumbnail ^Integer (int width) ^Integer (int height) "^")
-             (.gravity "center")
-             (.extent (int width) (int height))
-             (.quality (double quality))
-             (.addImage))]
-    (generic-process (assoc params :operation op))))
+  [params]
+  (let [{:keys [quality width height] :as params}
+        (check-thumbnail-params params)
+
+        operation
+        (doto (IMOperation.)
+          (.addImage)
+          (.autoOrient)
+          (.strip)
+          (.thumbnail ^Integer (int width) ^Integer (int height) "^")
+          (.gravity "center")
+          (.extent (int width) (int height))
+          (.quality (double quality))
+          (.addImage))]
+
+    (generic-process (assoc params :operation operation))))
 
 (defn get-basic-info-from-svg
   [{:keys [tag attrs] :as data}]
@@ -184,10 +217,9 @@
 
 (defmethod process :info
   [{:keys [input] :as params}]
-  (us/assert ::input input)
-  (let [{:keys [path mtype]} input]
+  (let [{:keys [path mtype] :as input} (check-input input)]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp parse-svg get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file

backend/src/app/migrations.clj

@@ -426,7 +426,19 @@
     :fn (mg/resource "app/migrations/sql/0134-mod-file-change-table.sql")}
 
    {:name "0135-mod-team-invitation-table.sql"
-    :fn (mg/resource "app/migrations/sql/0135-mod-team-invitation-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0135-mod-team-invitation-table.sql")}
+
+   {:name "0136-mod-comments-mentions.sql"
+    :fn (mg/resource "app/migrations/sql/0136-mod-comments-mentions.sql")}
+
+   {:name "0137-add-file-migration-table.sql"
+    :fn (mg/resource "app/migrations/sql/0137-add-file-migration-table.sql")}
+
+   {:name "0138-mod-file-data-fragment-table.sql"
+    :fn (mg/resource "app/migrations/sql/0138-mod-file-data-fragment-table.sql")}
+
+   {:name "0139-mod-file-change-table.sql"
+    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/media_refs.clj

@@ -0,0 +1,49 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.migrations.media-refs
+  "A media refs migration fixer script"
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.pprint]
+   [app.srepl.fixes.media-refs :refer [process-file]]
+   [app.srepl.main :as srepl]
+   [clojure.edn :as edn]))
+
+(def ^:private required-services
+  [:app.storage.s3/backend
+   :app.storage.fs/backend
+   :app.storage/storage
+   :app.metrics/metrics
+   :app.db/pool
+   :app.worker/executor])
+
+(defn -main
+  [& [options]]
+  (try
+    (let [config-var (requiring-resolve 'app.main/system-config)
+          start-var  (requiring-resolve 'app.main/start-custom)
+          stop-var   (requiring-resolve 'app.main/stop)
+          config     (select-keys @config-var required-services)]
+
+      (start-var config)
+
+      (let [options (if (string? options)
+                      (ex/ignoring (edn/read-string options))
+                      {})]
+
+        (l/inf :hint "executing media-refs migration" :options options)
+        (srepl/process-files! process-file options))
+
+      (stop-var)
+      (System/exit 0))
+    (catch Throwable cause
+      (ex/print-throwable cause)
+      (flush)
+      (System/exit -1))))
+
+

backend/src/app/migrations/sql/0136-mod-comments-mentions.sql

@@ -0,0 +1,3 @@
+ALTER TABLE comment ADD COLUMN mentions uuid[] NULL DEFAULT '{}';
+
+ALTER TABLE comment_thread ADD COLUMN mentions uuid[] NULL DEFAULT '{}';

backend/src/app/migrations/sql/0137-add-file-migration-table.sql

@@ -0,0 +1,7 @@
+CREATE TABLE file_migration (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  name text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  PRIMARY KEY(file_id, name)
+);

backend/src/app/migrations/sql/0138-mod-file-data-fragment-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_data_fragment
+ALTER CONSTRAINT file_data_fragment_file_id_fkey DEFERRABLE INITIALLY DEFERRED;

backend/src/app/migrations/sql/0139-mod-file-change-table.sql

@@ -0,0 +1,5 @@
+ALTER TABLE file_change
+ DROP CONSTRAINT file_change_file_id_fkey,
+ DROP CONSTRAINT file_change_profile_id_fkey,
+  ADD FOREIGN KEY (file_id) REFERENCES file(id) DEFERRABLE,
+  ADD FOREIGN KEY (profile_id) REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;

backend/src/app/migrations/v2.clj

@@ -1,103 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.migrations.v2
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.logging :as l]
-   [app.db :as db]
-   [app.features.components-v2 :as feat]
-   [app.setup :as setup]
-   [app.util.time :as dt]))
-
-(def ^:private sql:get-teams
-  "SELECT id, features,
-          row_number() OVER (ORDER BY created_at DESC) AS rown
-    FROM team
-   WHERE deleted_at IS NULL
-     AND (not (features @> '{components/v2}') OR features IS NULL)
-   ORDER BY created_at DESC")
-
-(defn- get-teams
-  [conn]
-  (->> (db/cursor conn [sql:get-teams] {:chunk-size 1})
-       (map feat/decode-row)))
-
-(defn- migrate-teams
-  [{:keys [::db/conn] :as system}]
-  ;; Allow long running transaction for this connection
-  (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-
-  ;; Do not allow other migration running in the same time
-  (db/xact-lock! conn 0)
-
-  ;; Run teams migration
-  (run! (fn [{:keys [id rown]}]
-          (try
-            (-> (assoc system ::db/rollback false)
-                (feat/migrate-team! id
-                                    :rown rown
-                                    :label "v2-migration"
-                                    :validate? false
-                                    :skip-on-graphics-error? true))
-            (catch Throwable _
-              (swap! feat/*stats* update :errors (fnil inc 0))
-              (l/wrn :hint "error on migrating team (skiping)"))))
-        (get-teams conn))
-
-  (setup/set-prop! system :v2-migrated true))
-
-(defn migrate
-  [system]
-  (let [tpoint    (dt/tpoint)
-        stats     (atom {})
-        migrated? (setup/get-prop system :v2-migrated false)]
-
-    (when-not migrated?
-      (l/inf :hint "v2 migration started")
-      (try
-        (binding [feat/*stats* stats]
-          (db/tx-run! system migrate-teams))
-
-        (let [stats   (deref stats)
-              elapsed (dt/format-duration (tpoint))]
-          (l/inf :hint "v2 migration finished"
-                 :files (:processed-files stats)
-                 :teams (:processed-teams stats)
-                 :errors (:errors stats)
-                 :elapsed elapsed))
-
-        (catch Throwable cause
-          (l/err :hint "error on aplying v2 migration" :cause cause))))))
-
-(def ^:private required-services
-  [[:app.main/assets :app.storage.s3/backend]
-   [:app.main/assets :app.storage.fs/backend]
-   :app.storage/storage
-   :app.db/pool
-   :app.setup/props
-   :app.svgo/optimizer
-   :app.metrics/metrics
-   :app.migrations/migrations
-   :app.http.client/client])
-
-(defn -main
-  [& _args]
-  (try
-    (let [config-var (requiring-resolve 'app.main/system-config)
-          start-var  (requiring-resolve 'app.main/start-custom)
-          stop-var   (requiring-resolve 'app.main/stop)
-          system-var (requiring-resolve 'app.main/system)
-          config     (select-keys @config-var required-services)]
-
-      (start-var config)
-      (migrate @system-var)
-      (stop-var)
-      (System/exit 0))
-    (catch Throwable cause
-      (ex/print-throwable cause)
-      (flush)
-      (System/exit -1))))

backend/src/app/rpc/commands/access_token.clj

@@ -43,13 +43,8 @@
     (decode-row token)))
 
 (defn repl:create-access-token
-  [{:keys [::db/pool] :as system} profile-id name expiration]
-  (db/with-atomic [conn pool]
-    (let [props (:app.setup/props system)]
-      (create-access-token {::db/conn conn ::setup/props props}
-                           profile-id
-                           name
-                           expiration))))
+  [cfg profile-id name expiration]
+  (db/tx-run! cfg create-access-token profile-id name expiration))
 
 (def ^:private schema:create-access-token
   [:map {:title "create-access-token"}

backend/src/app/rpc/commands/audit.clj

@@ -92,9 +92,9 @@
      [:string {:max 250}]
      [::sm/one-of {:format "string"} valid-event-types]]]
    [:props
-    [:map-of :keyword :any]]
+    [:map-of :keyword ::sm/any]]
    [:context {:optional true}
-    [:map-of :keyword :any]]])
+    [:map-of :keyword ::sm/any]]])
 
 (def schema:push-audit-events
   [:map {:title "push-audit-events"}

backend/src/app/rpc/commands/auth.clj

@@ -55,7 +55,7 @@
                 (contains? cf/flags :login-with-password))
     (ex/raise :type :restriction
               :code :login-disabled
-              :hint "login is disabled in this instance"))
+              :hint "login is disabled"))
 
   (letfn [(check-password [cfg profile password]
             (if (= (:password profile) "!")
@@ -79,7 +79,8 @@
                         :code :wrong-credentials))
             (when (:is-blocked profile)
               (ex/raise :type :restriction
-                        :code :profile-blocked))
+                        :code :profile-blocked
+                        :hint "profile is marked as blocked"))
             (when-not (check-password cfg profile password)
               (ex/raise :type :validation
                         :code :wrong-credentials))
@@ -149,7 +150,7 @@
 ;; ---- COMMAND: Recover Profile
 
 (defn recover-profile
-  [{:keys [::db/pool] :as cfg} {:keys [token password]}]
+  [{:keys [::db/conn] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
             (let [tdata (tokens/verify (::setup/props cfg) {:token token :iss :password-recovery})]
               (:profile-id tdata)))
@@ -159,10 +160,10 @@
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
-    (db/with-atomic [conn pool]
-      (->> (validate-token token)
-           (update-password conn))
-      nil)))
+    (->> (validate-token token)
+         (update-password conn))
+
+    nil))
 
 (def schema:recover-profile
   [:map {:title "recover-profile"}
@@ -173,7 +174,8 @@
   {::rpc/auth false
    ::doc/added "1.15"
    ::sm/params schema:recover-profile
-   ::climit/id :auth/global}
+   ::climit/id :auth/global
+   ::db/transaction true}
   [cfg params]
   (recover-profile cfg params))
 
@@ -182,11 +184,11 @@
 (defn- validate-register-attempt!
   [cfg params]
 
-  (when (or
-         (not (contains? cf/flags :registration))
-         (not (contains? cf/flags :login-with-password)))
+  (when (or (not (contains? cf/flags :registration))
+            (not (contains? cf/flags :login-with-password)))
     (ex/raise :type :restriction
-              :code :registration-disabled))
+              :code :registration-disabled
+              :hint "registration disabled"))
 
   (when (contains? params :invitation-token)
     (let [invitation (tokens/verify (::setup/props cfg)
@@ -200,12 +202,14 @@
   (when (and (email.blacklist/enabled? cfg)
              (email.blacklist/contains? cfg (:email params)))
     (ex/raise :type :restriction
-              :code :email-domain-is-not-allowed))
+              :code :email-domain-is-not-allowed
+              :hint "email domain in blacklist"))
 
   (when (and (email.whitelist/enabled? cfg)
              (not (email.whitelist/contains? cfg (:email params))))
     (ex/raise :type :restriction
-              :code :email-domain-is-not-allowed))
+              :code :email-domain-is-not-allowed
+              :hint "email domain not in whitelist"))
 
   ;; Perform a basic validation of email & password
   (when (= (str/lower (:email params))
@@ -218,28 +222,30 @@
     (ex/raise :type :restriction
               :code :email-has-permanent-bounces
               :email (:email params)
-              :hint "looks like the email has bounce reports"))
+              :hint "email has bounce reports"))
 
   (when (eml/has-complaint-reports? cfg (:email params))
     (ex/raise :type :restriction
               :code :email-has-complaints
               :email (:email params)
-              :hint "looks like the email has complaint reports")))
+              :hint "email has complaint reports")))
 
 (defn prepare-register
-  [{:keys [::db/pool] :as cfg} {:keys [email] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [fullname email accept-newsletter-updates] :as params}]
 
   (validate-register-attempt! cfg params)
 
   (let [email   (profile/clean-email email)
         profile (profile/get-profile-by-email pool email)
         params  {:email email
+                 :fullname fullname
                  :password (:password params)
                  :invitation-token (:invitation-token params)
                  :backend "penpot"
                  :iss :prepared-register
                  :profile-id (:id profile)
-                 :exp (dt/in-future {:days 7})}
+                 :exp (dt/in-future {:days 7})
+                 :props {:newsletter-updates (or accept-newsletter-updates false)}}
 
         params (d/without-nils params)
         token  (tokens/generate (::setup/props cfg) params)]
@@ -249,8 +255,10 @@
 
 (def schema:prepare-register-profile
   [:map {:title "prepare-register-profile"}
+   [:fullname ::sm/text]
    [:email ::sm/email]
    [:password schema:password]
+   [:create-welcome-file {:optional true} :boolean]
    [:invitation-token {:optional true} schema:token]])
 
 (sv/defmethod ::prepare-register-profile
@@ -354,13 +362,9 @@
                 :extra-data ptoken})))
 
 (defn register-profile
-  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token fullname theme] :as params}]
-  (let [theme      (when (= theme "light") theme)
-        claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
-        params     (-> claims
-                       (into params)
-                       (assoc :fullname fullname)
-                       (assoc :theme theme))
+  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token] :as params}]
+  (let [claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
+        params     (into claims params)
 
         profile    (if-let [profile-id (:profile-id claims)]
                      (profile/get-profile conn profile-id)
@@ -474,10 +478,7 @@
 
 (def schema:register-profile
   [:map {:title "register-profile"}
-   [:token schema:token]
-   [:fullname [::sm/word-string {:max 100}]]
-   [:theme {:optional true} [:string {:max 10}]]
-   [:create-welcome-file {:optional true} :boolean]])
+   [:token schema:token]])
 
 (sv/defmethod ::register-profile
   {::rpc/auth false

backend/src/app/rpc/commands/binfile.clj

@@ -7,10 +7,13 @@
 (ns app.rpc.commands.binfile
   (:refer-clojure :exclude [assert])
   (:require
+   [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
+   [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
@@ -19,6 +22,7 @@
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.tasks.file-gc]
    [app.util.services :as sv]
@@ -34,7 +38,6 @@
 (def ^:private
   schema:export-binfile
   [:map {:title "export-binfile"}
-   [:name [:string {:max 250}]]
    [:file-id ::sm/uuid]
    [:version {:optional true} ::sm/int]
    [:include-libraries ::sm/boolean]
@@ -46,9 +49,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v1/ids #{file-id})
-           (assoc ::bf.v1/embed-assets embed-assets)
-           (assoc ::bf.v1/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v1/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -61,9 +64,9 @@
    (fn [_ output-stream]
      (try
        (-> cfg
-           (assoc ::bf.v3/ids #{file-id})
-           (assoc ::bf.v3/embed-assets embed-assets)
-           (assoc ::bf.v3/include-libraries include-libraries)
+           (assoc ::bfc/ids #{file-id})
+           (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/include-libraries include-libraries)
            (bf.v3/export-files! output-stream))
        (catch Throwable cause
          (l/err :hint "exception on exporting file"
@@ -74,7 +77,7 @@
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
    ::webhooks/event? true
-   ::sm/result schema:export-binfile}
+   ::sm/params schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
   (fn [_]
@@ -90,41 +93,31 @@
 
 ;; --- Command: import-binfile
 
-(defn- import-binfile-v1
-  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
-  (let [cfg (-> cfg
-                (assoc ::bf.v1/project-id project-id)
-                (assoc ::bf.v1/profile-id profile-id)
-                (assoc ::bf.v1/name name)
-                (assoc ::bf.v1/input (:path file)))]
-
-    ;; NOTE: the importation process performs some operations that are
-    ;; not very friendly with virtual threads, and for avoid
-    ;; unexpected blocking of other concurrent operations we dispatch
-    ;; that operation to a dedicated executor.
-    (px/invoke! executor (partial bf.v1/import-files! cfg))))
-
-(defn- import-binfile-v3
-  [{:keys [::wrk/executor] :as cfg} {:keys [project-id profile-id name file]}]
-  (let [cfg (-> cfg
-                (assoc ::bf.v3/project-id project-id)
-                (assoc ::bf.v3/profile-id profile-id)
-                (assoc ::bf.v3/name name)
-                (assoc ::bf.v3/input (:path file)))]
-    ;; NOTE: the importation process performs some operations that are
-    ;; not very friendly with virtual threads, and for avoid
-    ;; unexpected blocking of other concurrent operations we dispatch
-    ;; that operation to a dedicated executor.
-    (px/invoke! executor (partial bf.v3/import-files! cfg))))
-
 (defn- import-binfile
-  [{:keys [::db/pool] :as cfg} {:keys [project-id version] :as params}]
-  (let [result (case (int version)
-                 1 (import-binfile-v1 cfg params)
-                 3 (import-binfile-v3 cfg params))]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
+  (let [team   (teams/get-team pool
+                               :profile-id profile-id
+                               :project-id project-id)
+        cfg    (-> cfg
+                   (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team))
+                   (assoc ::bfc/project-id project-id)
+                   (assoc ::bfc/profile-id profile-id)
+                   (assoc ::bfc/name name)
+                   (assoc ::bfc/input (:path file)))
+
+        ;; NOTE: the importation process performs some operations that are
+        ;; not very friendly with virtual threads, and for avoid
+        ;; unexpected blocking of other concurrent operations we dispatch
+        ;; that operation to a dedicated executor.
+        result (case (int version)
+                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
+                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
+
     (db/update! pool :project
                 {:modified-at (dt/now)}
-                {:id project-id})
+                {:id project-id}
+                {::db/return-keys false})
+
     result))
 
 (def ^:private schema:import-binfile
@@ -141,11 +134,18 @@
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [params (-> params
-                   (assoc :profile-id profile-id)
-                   (assoc :version (or version 1)))]
+  (let [version  (or version 1)
+        params   (-> params
+                     (assoc :profile-id profile-id)
+                     (assoc :version version))
+        manifest (case (int version)
+                   1 nil
+                   3 (bf.v3/get-manifest (:path file)))]
+
     (with-meta
       (sse/response (partial import-binfile cfg params))
-      {::audit/props {:file nil}})))
+      {::audit/props {:file nil
+                      :generated-by (:generated-by manifest)
+                      :referer (:referer manifest)}})))

backend/src/app/rpc/commands/comments.clj

@@ -6,28 +6,153 @@
 
 (ns app.rpc.commands.comments
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
    [app.common.schema :as sm]
+   [app.common.uri :as uri]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.email :as eml]
    [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
+   [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.quotes :as quotes]
    [app.rpc.retry :as rtry]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [app.util.time :as dt]))
+   [app.util.time :as dt]
+   [clojure.set :as set]
+   [cuerdas.core :as str]))
 
 ;; --- GENERAL PURPOSE INTERNAL HELPERS
 
+(def r-mentions-split #"@\[[^\]]*\]\([^\)]*\)")
+(def r-mentions #"@\[([^\]]*)\]\(([^\)]*)\)")
+
+(def comment-max-length 750)
+
+(defn- format-comment
+  [{:keys [content]}]
+  (->> (d/interleave-all
+        (str/split content r-mentions-split)
+        (->> (re-seq r-mentions content)
+             (map (fn [[_ user _]] user))))
+       (str/join "")))
+
+(defn- format-comment-url
+  [thread file]
+  (str/ffmt "%/#/workspace?%"
+            (cf/get :public-uri)
+            (uri/map->query-string
+             {:file-id (:id file)
+              :page-id (:page-id file)
+              :team-id (:team-id file)
+              :comment-id (:id thread)})))
+
+(defn- format-comment-ref
+  [thread file]
+  (str/ffmt "#%, %, %"
+            (:seqn thread)
+            (:name file)
+            (:page-name file)))
+
+(defn- get-team-users
+  [conn team-id]
+  (->> (teams/get-users+props conn team-id)
+       (map profile/decode-row)
+       (d/index-by :id)))
+
+(defn- notification-email?
+  [profile-id owner-id props]
+  (if (= profile-id owner-id)
+    (not= :none (-> props :notifications :email-comments))
+    (= :all (-> props :notifications :email-comments))))
+
+(defn- mention-email?
+  [props]
+  (not= :none (-> props :notifications :email-comments)))
+
+(defn send-comment-emails!
+  [conn profile comment thread file]
+  (let [team-users        (get-team-users conn (:team-id file))
+        comment-reference (format-comment-ref thread file)
+        comment-content   (format-comment comment)
+        comment-url       (format-comment-url thread file)
+        profile-id        (get profile :id)
+
+        ;; Users mentioned in this comment
+        comment-mentions
+        (-> (:mentions comment)
+            (disj profile-id))
+
+        ;; Users mentioned in this thread
+        thread-mentions
+        (-> (:mentions thread)
+            ;; Remove the mentions in the comment because we're already sending a
+            ;; notification
+            (set/difference comment-mentions)
+            (disj profile-id))
+
+        ;; All users
+        notificate-users-ids
+        (-> (set (keys team-users))
+            (set/difference comment-mentions)
+            (set/difference thread-mentions)
+            (disj profile-id))]
+
+    (doseq [mention comment-mentions]
+      (let [{:keys [fullname email props]} (get team-users mention)]
+        (when (mention-email? props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-mention
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))
+
+    ;; Send to the thread users
+    (doseq [mention thread-mentions]
+      (let [{:keys [fullname email props]} (get team-users mention)]
+        (when (mention-email? props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-thread
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))
+
+    ;; Send to users with the "all" flag activated
+    (doseq [user-id notificate-users-ids]
+      (let [{:keys [id fullname email props]} (get team-users user-id)]
+        (when (notification-email? id (:owner-id thread) props)
+          (eml/send!
+           {::eml/conn conn
+            ::eml/factory eml/comment-notification
+            :public-uri (cf/get :public-uri)
+            :to email
+            :name fullname
+            :source-user (:fullname profile)
+            :comment-reference comment-reference
+            :comment-content comment-content
+            :comment-url comment-url}))))))
+
 (defn- decode-row
   [{:keys [participants position mentions] :as row}]
   (cond-> row
@@ -38,14 +163,15 @@
 (def xf-decode-row
   (map decode-row))
 
-(def ^:privateqpage-name
+(def ^:private
   sql:get-file
-  "select f.id, f.modified_at, f.revn, f.features,
-          f.project_id, p.team_id, f.data
-     from file as f
-     join project as p on (p.id = f.project_id)
-    where f.id = ?
-      and f.deleted_at is null")
+  "SELECT f.id, f.modified_at, f.revn, f.features, f.name,
+          f.project_id, p.team_id, f.data,
+          f.data_ref_id, f.data_backend
+     FROM file as f
+    INNER JOIN project as p on (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
 
 (defn- get-file
   "A specialized version of get-file for comments module."
@@ -57,12 +183,16 @@
                 :hint "file not found"))
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-      (let [{:keys [data] :as file} (files/decode-row file)]
+      (let [file (->> file
+                      (files/decode-row)
+                      (feat.fdata/resolve-file-data cfg))
+            data (get file :data)]
         (-> file
             (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
             (assoc :page-id page-id)
             (dissoc :data))))))
 
+;; FIXME: rename
 (defn- get-comment-thread
   [conn thread-id & {:as opts}]
   (-> (db/get-by-id conn :comment-thread thread-id opts)
@@ -92,7 +222,7 @@
 
 (defn upsert-comment-thread-status!
   ([conn profile-id thread-id]
-   (upsert-comment-thread-status! conn profile-id thread-id (dt/now)))
+   (upsert-comment-thread-status! conn profile-id thread-id (dt/in-future "1s")))
   ([conn profile-id thread-id mod-at]
    (db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id mod-at mod-at])))
 
@@ -117,36 +247,42 @@
   {::doc/added "1.15"
    ::sm/params schema:get-comment-threads}
   [cfg {:keys [::rpc/profile-id file-id share-id] :as params}]
-
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
 (def ^:private sql:comment-threads
-  "select distinct on (ct.id)
+  "SELECT DISTINCT ON (ct.id)
           ct.*,
-          f.name as file_name,
-          f.project_id as project_id,
-          first_value(c.content) over w as content,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id) as count_comments,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id
-              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
-     from comment_thread as ct
-    inner join comment as c on (c.thread_id = ct.id)
-    inner join file as f on (f.id = ct.file_id)
-     left join comment_thread_status as cts
-            on (cts.thread_id = ct.id and
-                cts.profile_id = ?)
-    where ct.file_id = ?
-   window w as (partition by c.thread_id order by c.created_at asc)")
+          pf.fullname AS owner_fullname,
+          pf.email AS owner_email,
+          pf.photo_id AS owner_photo_id,
+          p.team_id AS team_id,
+          f.name AS file_name,
+          f.project_id AS project_id,
+          first_value(c.content) OVER w AS content,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id) AS count_comments,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id
+              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+     FROM comment_thread AS ct
+    INNER JOIN comment AS c ON (c.thread_id = ct.id)
+    INNER JOIN file AS f ON (f.id = ct.file_id)
+    INNER JOIN project AS p ON (p.id = f.project_id)
+     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
+
+(def ^:private sql:comment-threads-by-file-id
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
-  (->> (db/exec! conn [sql:comment-threads profile-id file-id])
+  (->> (db/exec! conn [sql:comment-threads-by-file-id profile-id file-id])
        (into [] xf-decode-row)))
 
 ;; --- COMMAND: Get Unread Comment Threads
@@ -162,41 +298,41 @@
   {::doc/added "1.15"
    ::sm/params schema:get-unread-comment-threads}
   [cfg {:keys [::rpc/profile-id team-id] :as params}]
-  (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (teams/check-read-permissions! conn profile-id team-id)
-                 (get-unread-comment-threads conn profile-id team-id))))
+  (db/run!
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (teams/check-read-permissions! conn profile-id team-id)
+     (get-unread-comment-threads conn profile-id team-id))))
 
-(def sql:comment-threads-by-team
-  "select distinct on (ct.id)
-          ct.*,
-          f.name as file_name,
-          f.project_id as project_id,
-          first_value(c.content) over w as content,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id) as count_comments,
-          (select count(1)
-             from comment as c
-            where c.thread_id = ct.id
-              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
-     from comment_thread as ct
-    inner join comment as c on (c.thread_id = ct.id)
-    inner join file as f on (f.id = ct.file_id)
-    inner join project as p on (p.id = f.project_id)
-     left join comment_thread_status as cts
-            on (cts.thread_id = ct.id and
-                cts.profile_id = ?)
-    where p.team_id = ?
-   window w as (partition by c.thread_id order by c.created_at asc)")
+(def sql:unread-all-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
 
-(def sql:unread-comment-threads-by-team
-  (str "with threads as (" sql:comment-threads-by-team ")"
-       "select * from threads where count_unread_comments > 0"))
+;; The partial configuration will retrieve only comments created by the user and
+;; threads that have a mention to the user.
+(def sql:unread-partial-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads
+         WHERE count_unread_comments > 0
+           AND team_id = ?
+           AND (owner_id = ? OR ? = ANY(mentions))"))
 
 (defn- get-unread-comment-threads
   [conn profile-id team-id]
-  (->> (db/exec! conn [sql:unread-comment-threads-by-team profile-id team-id])
-       (into [] xf-decode-row)))
+  (let [profile (-> (db/get conn :profile {:id profile-id})
+                    (profile/decode-row))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
+
+    (case notify
+      :all
+      (->> (db/exec! conn [sql:unread-all-comment-threads-by-team profile-id team-id])
+           (into [] xf-decode-row))
+
+      :partial
+      (->> (db/exec! conn [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+           (into [] xf-decode-row))
+
+      [])))
 
 ;; --- COMMAND: Get Single Comment Thread
 
@@ -213,9 +349,9 @@
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (let [sql (str "with threads as (" sql:comment-threads ")"
-                                "select * from threads where id = ?")]
-                   (-> (db/exec-one! conn [sql profile-id file-id id])
+                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
+                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                   (-> (db/exec-one! conn [sql profile-id id file-id])
                        (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
@@ -233,15 +369,25 @@
    ::sm/params schema:get-comments}
   [cfg {:keys [::rpc/profile-id thread-id share-id]}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (let [{:keys [file-id] :as thread} (get-comment-thread conn thread-id)]
+                 (let [{:keys [file-id]} (get-comment-thread conn thread-id)]
                    (files/check-comment-permissions! conn profile-id file-id share-id)
                    (get-comments conn thread-id)))))
 
+(def sql:get-comments
+  "SELECT c.*,
+          ct.file_id AS file_id,
+          pf.fullname AS owner_fullname,
+          pf.email AS owner_email,
+          pf.photo_id AS owner_photo_id
+     FROM comment AS c
+    INNER JOIN comment_thread AS ct ON (ct.id = c.thread_id)
+     LEFT JOIN profile AS pf ON (c.owner_id = pf.id)
+    WHERE c.thread_id = ?
+    ORDER BY c.created_at ASC")
+
 (defn- get-comments
   [conn thread-id]
-  (->> (db/query conn :comment
-                 {:thread-id thread-id}
-                 {:order-by [[:created-at :asc]]})
+  (->> (db/exec! conn [sql:get-comments thread-id])
        (into [] xf-decode-row)))
 
 ;; --- COMMAND: Get file comments users
@@ -298,10 +444,25 @@
   [:map {:title "create-comment-thread"}
    [:file-id ::sm/uuid]
    [:position ::gpt/point]
-   [:content [:string {:max 750}]]
+   [:content [:string {:max comment-max-length}]]
    [:page-id ::sm/uuid]
    [:frame-id ::sm/uuid]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
+
+(defn- update-thread-seqn
+  [conn file-id seqn]
+  (db/update! conn :file
+              {:comment-thread-seqn seqn}
+              {:id file-id}
+              {::db/return-keys false}))
+
+(defn add-owner
+  [thread-or-comment profile]
+  (-> thread-or-comment
+      (assoc :owner-fullname (:fullname profile))
+      (assoc :owner-email (:email profile))
+      (assoc :owner-photo-id (:photo-id profile))))
 
 (sv/defmethod ::create-comment-thread
   {::doc/added "1.15"
@@ -309,11 +470,10 @@
    ::rtry/enabled true
    ::rtry/when rtry/conflict-exception?
    ::sm/params schema:create-comment-thread}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
+  [cfg {:keys [::rpc/profile-id file-id page-id share-id] :as params}]
   (files/check-comment-permissions! cfg profile-id file-id share-id)
 
-  (let [{:keys [team-id project-id page-name]} (get-file cfg file-id page-id)]
-
+  (let [{:keys [team-id project-id] :as file} (get-file cfg file-id page-id)]
     (-> cfg
         (assoc ::quotes/profile-id profile-id)
         (assoc ::quotes/team-id team-id)
@@ -322,21 +482,14 @@
         (quotes/check! {::quotes/id ::quotes/comment-threads-per-file}
                        {::quotes/id ::quotes/comments-per-file}))
 
-    (let [params {:created-at request-at
-                  :profile-id profile-id
-                  :file-id file-id
-                  :page-id page-id
-                  :page-name page-name
-                  :position position
-                  :content content
-                  :frame-id frame-id}
+    (let [params (assoc params ::file file)
           thread (db/tx-run! cfg create-comment-thread params)]
 
       (vary-meta thread assoc ::audit/props thread))))
 
 (defn- create-comment-thread
   [{:keys [::db/conn] :as cfg}
-   {:keys [profile-id file-id page-id page-name created-at position content frame-id]}]
+   {:keys [::rpc/profile-id ::rpc/request-at ::file position content mentions frame-id] :as params}]
 
   (let [;; NOTE: we take the next seq number from a separate query
         ;; because we need to lock the file for avoid race conditions
@@ -347,39 +500,47 @@
         ;; different storage (example: redis) for alivate the update
         ;; pression on the file table
 
-        seqn      (get-next-seqn conn file-id)
+        profile   (profile/get-profile conn profile-id)
+        seqn      (get-next-seqn conn (:id file))
+
+        file-id   (get file :id)
         thread-id (uuid/next)
-        thread    (db/insert! conn :comment-thread
-                              {:id thread-id
-                               :file-id file-id
-                               :owner-id profile-id
-                               :participants (db/tjson #{profile-id})
-                               :page-name page-name
-                               :page-id page-id
-                               :created-at created-at
-                               :modified-at created-at
-                               :seqn seqn
-                               :position (db/pgpoint position)
-                               :frame-id frame-id})
-        comment   (db/insert! conn :comment
-                              {:id (uuid/next)
-                               :thread-id thread-id
-                               :owner-id profile-id
-                               :created-at created-at
-                               :modified-at created-at
-                               :content content})]
+
+        thread    (-> (db/insert! conn :comment-thread
+                                  {:id thread-id
+                                   :file-id file-id
+                                   :page-name (:page-name file)
+                                   :page-id (:page-id file)
+                                   :owner-id profile-id
+                                   :participants (db/tjson #{profile-id})
+                                   :created-at request-at
+                                   :modified-at request-at
+                                   :seqn seqn
+                                   :frame-id frame-id
+                                   :position (db/pgpoint position)
+                                   :mentions (db/encode-pgarray mentions conn "uuid")})
+                      (decode-row))
+        comment   (-> (db/insert! conn :comment
+                                  {:id (uuid/next)
+                                   :thread-id thread-id
+                                   :owner-id profile-id
+                                   :created-at request-at
+                                   :modified-at request-at
+                                   :mentions (db/encode-pgarray mentions conn "uuid")
+                                   :content content})
+                      (decode-row))]
 
     ;; Make the current thread as read.
-    (upsert-comment-thread-status! conn profile-id thread-id created-at)
+    (upsert-comment-thread-status! conn profile-id thread-id request-at)
 
     ;; Optimistic update of current seq number on file.
-    (db/update! conn :file
-                {:comment-thread-seqn seqn}
-                {:id file-id}
-                {::db/return-keys false})
+    (update-thread-seqn conn file-id seqn)
+
+    ;; Send mentions emails
+    (send-comment-emails! conn profile comment thread file)
 
     (-> thread
-        (select-keys [:id :file-id :page-id])
+        (add-owner profile)
         (assoc :comment-id (:id comment)))))
 
 ;; --- COMMAND: Update Comment Thread Status
@@ -392,12 +553,12 @@
 
 (sv/defmethod ::update-comment-thread-status
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-status}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (upsert-comment-thread-status! conn profile-id id)))))
+   ::sm/params schema:update-comment-thread-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (upsert-comment-thread-status! conn profile-id id)))
 
 ;; --- COMMAND: Update Comment Thread
 
@@ -410,77 +571,85 @@
 
 (sv/defmethod ::update-comment-thread
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread}
-  [cfg {:keys [::rpc/profile-id id is-resolved share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:is-resolved is-resolved}
-                                  {:id id})
-                      nil))))
-
+   ::sm/params schema:update-comment-thread
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id is-resolved share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:is-resolved is-resolved}
+                {:id id})
+    nil))
 
 ;; --- COMMAND: Add Comment
 
-(declare ^:private get-comment-thread)
-
 (def ^:private
   schema:create-comment
   [:map {:title "create-comment"}
    [:thread-id ::sm/uuid]
-   [:content [:string {:max 250}]]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:content [:string {:max comment-max-length}]]
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
 
 (sv/defmethod ::create-comment
   {::doc/added "1.15"
    ::webhooks/event? true
-   ::sm/params schema:create-comment}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)
-                      {:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
+   ::sm/params schema:create-comment
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content mentions]}]
+  (let [{:keys [file-id page-id] :as thread}
+        (get-comment-thread conn thread-id ::sql/for-update true)
 
-                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                  (quotes/check! cfg {::quotes/id ::quotes/comments-per-file
-                                      ::quotes/profile-id profile-id
-                                      ::quotes/team-id team-id
-                                      ::quotes/project-id project-id
-                                      ::quotes/file-id file-id})
+        {:keys [team-id project-id] :as file}
+        (get-file cfg file-id page-id)]
 
-                  ;; Update the page-name cached attribute on comment thread table.
-                  (when (not= page-name (:page-name thread))
-                    (db/update! conn :comment-thread
-                                {:page-name page-name}
-                                {:id thread-id}))
+    (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (let [comment (db/insert! conn :comment
-                                            {:id (uuid/next)
-                                             :created-at request-at
-                                             :modified-at request-at
-                                             :thread-id thread-id
-                                             :owner-id profile-id
-                                             :content content})
-                        comment (decode-row comment)
-                        props   {:file-id file-id
-                                 :share-id nil}]
+    (quotes/check! cfg {::quotes/id ::quotes/comments-per-file
+                        ::quotes/profile-id profile-id
+                        ::quotes/team-id team-id
+                        ::quotes/project-id project-id
+                        ::quotes/file-id file-id})
 
-                    ;; Update thread modified-at attribute and assoc the current
-                    ;; profile to the participant set.
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :participants (-> (:participants thread #{})
-                                                   (conj profile-id)
-                                                   (db/tjson))}
-                                {:id thread-id})
+    (let [profile  (profile/get-profile conn profile-id)
+          mentions (into #{} mentions)
+          params   {:id (uuid/next)
+                    :created-at request-at
+                    :modified-at request-at
+                    :thread-id thread-id
+                    :owner-id profile-id
+                    :content content
+                    :mentions (db/encode-pgarray mentions conn "uuid")}
 
-                    ;; Update the current profile status in relation to the
-                    ;; current thread.
-                    (upsert-comment-thread-status! conn profile-id thread-id request-at)
+          comment (-> (db/insert! conn :comment params)
+                      (decode-row)
+                      (assoc :file-id file-id)
+                      (add-owner profile))]
 
-                    (vary-meta comment assoc ::audit/props props))))))
+      ;; Update thread modified-at attribute and assoc the current
+      ;; profile to the participant set.
+      (let [mentions     (into (:mentions thread) mentions)
+            participants (-> (:participants thread #{})
+                             (conj profile-id))
+            params       {:modified-at request-at
+                          :participants (db/tjson participants)
+                          :mentions (db/encode-pgarray mentions conn "uuid")}
 
+            ;; Update the page-name cached attribute on comment thread table.
+            params       (cond-> params
+                           (not= (:page-name file) (:page-name thread))
+                           (assoc :page-name (:page-name file)))]
+
+        (db/update! conn :comment-thread params
+                    {:id thread-id}
+                    {::db/return-keys false}))
+
+      ;; Update the current profile status in relation to the current thread
+      (upsert-comment-thread-status! conn profile-id thread-id)
+
+      (send-comment-emails! conn profile comment thread file)
+
+      (vary-meta comment assoc ::audit/props comment))))
 
 ;; --- COMMAND: Update Comment
 
@@ -488,36 +657,47 @@
   schema:update-comment
   [:map {:title "update-comment"}
    [:id ::sm/uuid]
-   [:content [:string {:max 250}]]
-   [:share-id {:optional true} [:maybe ::sm/uuid]]])
+   [:content [:string {:max comment-max-length}]]
+   [:share-id {:optional true} [:maybe ::sm/uuid]]
+   [:mentions {:optional true} [::sm/set ::sm/uuid]]])
+
+;; TODO: Check if there are new mentions, if there are send the new emails.
 
 (sv/defmethod ::update-comment
   {::doc/added "1.15"
-   ::sm/params schema:update-comment}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::sql/for-update true)
-                      {:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::sql/for-update true)]
+   ::sm/params schema:update-comment
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content mentions]}]
+  (let [{:keys [thread-id owner-id] :as comment}
+        (get-comment conn id ::sql/for-update true)
 
-                  (files/check-comment-permissions! conn profile-id file-id share-id)
+        {:keys [file-id page-id] :as thread}
+        (get-comment-thread conn thread-id ::sql/for-update true)]
 
-                  ;; Don't allow edit comments to not owners
-                  (when-not (= owner-id profile-id)
-                    (ex/raise :type :validation
-                              :code :not-allowed))
+    (files/check-comment-permissions! conn profile-id file-id share-id)
 
-                  (let [{:keys [page-name]} (get-file cfg file-id page-id)]
-                    (db/update! conn :comment
-                                {:content content
-                                 :modified-at request-at}
-                                {:id id})
+    ;; Don't allow edit comments to not owners
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
 
-                    (db/update! conn :comment-thread
-                                {:modified-at request-at
-                                 :page-name page-name}
-                                {:id thread-id})
-                    nil)))))
+    (let [{:keys [page-name]} (get-file cfg file-id page-id)]
+      (db/update! conn :comment
+                  {:content content
+                   :modified-at request-at
+                   :mentions (db/encode-pgarray mentions conn "uuid")}
+                  {:id id})
+
+      (db/update! conn :comment-thread
+                  {:modified-at request-at
+                   :page-name page-name
+                   :mentions
+                   (-> (:mentions thread)
+                       (into mentions)
+                       (db/encode-pgarray conn "uuid"))}
+                  {:id thread-id}
+                  {::db/return-keys false})
+      nil)))
 
 ;; --- COMMAND: Delete Comment Thread
 
@@ -529,17 +709,18 @@
 
 (sv/defmethod ::delete-comment-thread
   {::doc/added "1.15"
-   ::sm/params schema:delete-comment-thread}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (when-not (= owner-id profile-id)
-                        (ex/raise :type :validation
-                                  :code :not-allowed))
+   ::sm/params schema:delete-comment-thread
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
 
-                      (db/delete! conn :comment-thread {:id id})
-                      nil))))
+    (db/delete! conn :comment-thread {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Delete comment
 
@@ -551,17 +732,23 @@
 
 (sv/defmethod ::delete-comment
   {::doc/added "1.15"
-   ::sm/params schema:delete-comment}
-  [cfg {:keys [::rpc/profile-id id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [owner-id thread-id] :as comment} (get-comment conn id ::sql/for-update true)
-                          {:keys [file-id] :as thread} (get-comment-thread conn thread-id)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (when-not (= owner-id profile-id)
-                        (ex/raise :type :validation
-                                  :code :not-allowed))
-                      (db/delete! conn :comment {:id id})
-                      nil))))
+   ::sm/params schema:delete-comment
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id share-id]}]
+  (let [{:keys [owner-id thread-id] :as comment}
+        (get-comment conn id ::sql/for-update true)
+
+        {:keys [file-id]}
+        (get-comment-thread conn thread-id)]
+
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (when-not (= owner-id profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
+
+    (db/delete! conn :comment {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Update comment thread position
 
@@ -575,17 +762,18 @@
 
 (sv/defmethod ::update-comment-thread-position
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-position}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:modified-at request-at
-                                   :position (db/pgpoint position)
-                                   :frame-id frame-id}
-                                  {:id (:id thread)})
-                      nil))))
+   ::sm/params schema:update-comment-thread-position
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:modified-at request-at
+                 :position (db/pgpoint position)
+                 :frame-id frame-id}
+                {:id id}
+                {::db/return-keys false})
+    nil))
 
 ;; --- COMMAND: Update comment frame
 
@@ -598,13 +786,29 @@
 
 (sv/defmethod ::update-comment-thread-frame
   {::doc/added "1.15"
-   ::sm/params schema:update-comment-thread-frame}
-  [cfg {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (let [{:keys [file-id] :as thread} (get-comment-thread conn id ::sql/for-update true)]
-                      (files/check-comment-permissions! conn profile-id file-id share-id)
-                      (db/update! conn :comment-thread
-                                  {:modified-at request-at
-                                   :frame-id frame-id}
-                                  {:id id})
-                      nil))))
+   ::sm/params schema:update-comment-thread-frame
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
+  (let [{:keys [file-id]} (get-comment-thread conn id ::sql/for-update true)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (db/update! conn :comment-thread
+                {:modified-at request-at
+                 :frame-id frame-id}
+                {:id id}
+                {::db/return-keys false})
+    nil))
+
+(def ^:private
+  schema:mark-all-threads-as-read
+  [:map {:title "mark-all-threads-as-read"}
+   [:threads [:vector ::sm/uuid]]])
+
+(sv/defmethod ::mark-all-threads-as-read
+  {::doc/added "1.15"
+   ::sm/params schema:mark-all-threads-as-read}
+  [cfg {:keys [::rpc/profile-id threads] :as params}]
+  (db/tx-run!
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (doseq [thread-id threads]
+       (upsert-comment-thread-status! conn profile-id thread-id)))))

backend/src/app/rpc/commands/demo.clj

@@ -27,7 +27,7 @@
   {::rpc/auth false
    ::doc/added "1.15"
    ::doc/changes ["1.15" "This method is migrated from mutations to commands."]}
-  [{:keys [::db/pool] :as cfg} _]
+  [cfg _]
 
   (when-not (contains? cf/flags :demo-users)
     (ex/raise :type :validation
@@ -49,9 +49,11 @@
                   :password (profile/derive-password cfg password)
                   :props {}}]
 
-    (db/with-atomic [conn pool]
-      (let [profile (->> (auth/create-profile! conn params)
-                         (auth/create-profile-rels! conn))]
-        (with-meta {:email email
-                    :password password}
-          {::audit/profile-id (:id profile)})))))
+
+    (let [profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                                    (->> (auth/create-profile! conn params)
+                                         (auth/create-profile-rels! conn))))]
+      (with-meta {:email email
+                  :password password}
+        {::audit/profile-id (:id profile)}))))
+

backend/src/app/rpc/commands/files.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -22,6 +23,8 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -188,7 +191,7 @@
    [:is-shared ::sm/boolean]
    [:project-id ::sm/uuid]
    [:created-at ::dt/instant]
-   [:data {:optional true} :any]])
+   [:data {:optional true} ::sm/any]])
 
 (def schema:permissions-mixin
   [:map {:title "PermissionsMixin"}
@@ -207,10 +210,11 @@
    [:project-id {:optional true} ::sm/uuid]])
 
 (defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
             pmap/*tracked* (pmap/create-tracked)]
-    (let [;; For avoid unnecesary overhead of creating multiple pointers and
+    (let [libs (delay (bfc/get-resolved-file-libraries cfg file))
+          ;; For avoid unnecesary overhead of creating multiple pointers and
           ;; handly internally with objects map in their worst case (when
           ;; probably all shapes and all pointers will be readed in any
           ;; case), we just realize/resolve them before applying the
@@ -218,42 +222,45 @@
           file (-> file
                    (update :data feat.fdata/process-pointers deref)
                    (update :data feat.fdata/process-objects (partial into {}))
-                   (fmg/migrate-file))
+                   (fmg/migrate-file libs))]
 
-          ;; When file is migrated, we break the rule of no perform
-          ;; mutations on get operations and update the file with all
-          ;; migrations applied
-          ;;
-          ;; WARN: he following code will not work on read-only mode,
-          ;; it is a known issue; we keep is not implemented until we
-          ;; really need this.
-          file (if (contains? (:features file) "fdata/objects-map")
-                 (feat.fdata/enable-objects-map file)
-                 file)
-          file (if (contains? (:features file) "fdata/pointer-map")
-                 (feat.fdata/enable-pointer-map file)
-                 file)]
+      (if (or read-only? (db/read-only? conn))
+        file
+        (let [;; When file is migrated, we break the rule of no perform
+              ;; mutations on get operations and update the file with all
+              ;; migrations applied
+              file (if (contains? (:features file) "fdata/objects-map")
+                     (feat.fdata/enable-objects-map file)
+                     file)
+              file (if (contains? (:features file) "fdata/pointer-map")
+                     (feat.fdata/enable-pointer-map file)
+                     file)]
 
-      (db/update! conn :file
-                  {:data (blob/encode (:data file))
-                   :version (:version file)
-                   :features (db/create-array conn "text" (:features file))}
-                  {:id id})
+          (db/update! conn :file
+                      {:data (blob/encode (:data file))
+                       :version (:version file)
+                       :features (db/create-array conn "text" (:features file))}
+                      {:id id}
+                      {::db/return-keys false})
 
-      (when (contains? (:features file) "fdata/pointer-map")
-        (feat.fdata/persist-pointers! cfg id))
+          (when (contains? (:features file) "fdata/pointer-map")
+            (feat.fdata/persist-pointers! cfg id))
 
-      file)))
+          (feat.fmigr/upsert-migrations! conn file)
+          (feat.fmigr/resolve-applied-migrations cfg file))))))
 
 (defn get-file
   [{:keys [::db/conn ::wrk/executor] :as cfg} id
    & {:keys [project-id
              migrate?
              include-deleted?
-             lock-for-update?]
+             lock-for-update?
+             preload-pointers?]
       :or {include-deleted? false
            lock-for-update? false
-           migrate? true}}]
+           migrate? true
+           preload-pointers? false}
+      :as options}]
 
   (assert (db/connection? conn) "expected cfg with valid connection")
 
@@ -264,16 +271,23 @@
                             {::db/check-deleted (not include-deleted?)
                              ::db/remove-deleted (not include-deleted?)
                              ::sql/for-update lock-for-update?})
+                    (feat.fmigr/resolve-applied-migrations cfg)
                     (feat.fdata/resolve-file-data cfg))
 
         ;; NOTE: we perform the file decoding in a separate thread
         ;; because it has heavy and synchronous operations for
         ;; decoding file body that are not very friendly with virtual
         ;; threads.
-        file   (px/invoke! executor #(decode-row file))]
+        file   (px/invoke! executor #(decode-row file))
+
+        file   (if (and migrate? (fmg/need-migration? file))
+                 (migrate-file cfg file options)
+                 file)]
+
+    (if preload-pointers?
+      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+        (update file :data feat.fdata/process-pointers deref))
 
-    (if (and migrate? (fmg/need-migration? file))
-      (migrate-file cfg file)
       file)))
 
 (defn get-minimal-file
@@ -289,7 +303,7 @@
 
 (defn get-file-etag
   [{:keys [::rpc/profile-id]} {:keys [modified-at revn vern permissions]}]
-  (str profile-id "/" revn "/" vern "/"
+  (str profile-id "/" revn "/" vern "/" (hash fmg/available-migrations) "/"
        (dt/format-instant modified-at :iso)
        "/"
        (uri/map->query-string permissions)))
@@ -320,11 +334,12 @@
 
           file (-> (get-file cfg id :project-id project-id)
                    (assoc :permissions perms)
+                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
           (cfeat/check-client-features! (:features params))
-          (cfeat/check-file-features! (:features file) (:features params)))
+          (cfeat/check-file-features! (:features file)))
 
       ;; This operation is needed for backward comapatibility with frontends that
       ;; does not support pointer-map resolution mechanism; this just resolves the
@@ -381,8 +396,10 @@
           f.revn,
           f.vern,
           f.is_shared,
-          ft.media_id AS thumbnail_id
+          ft.media_id AS thumbnail_id,
+          p.team_id
      from file as f
+     inner join project as p on (p.id = f.project_id)
      left join file_thumbnail as ft on (ft.file_id = f.id
                                         and ft.revn = f.revn
                                         and ft.deleted_at is null)
@@ -468,7 +485,7 @@
   (update page :objects update-vals #(dissoc % :thumbnail)))
 
 (defn get-page
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id share-id] :as params}]
 
   (when (and (uuid? object-id)
              (not (uuid? page-id)))
@@ -476,22 +493,30 @@
               :code :params-validation
               :hint "page-id is required when object-id is provided"))
 
-  (let [team (teams/get-team conn
-                             :profile-id profile-id
-                             :file-id file-id)
+  (let [perms (get-permissions conn profile-id file-id share-id)
 
-        file (get-file cfg file-id)
+        file  (get-file cfg file-id :read-only? true)
 
-        _    (-> (cfeat/get-team-enabled-features cf/flags team)
-                 (cfeat/check-client-features! (:features params))
-                 (cfeat/check-file-features! (:features file) (:features params)))
+        proj  (db/get conn :project {:id (:project-id file)})
 
-        page (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-               (let [page-id (or page-id (-> file :data :pages first))
-                     page    (dm/get-in file [:data :pages-index page-id])]
-                 (if (pmap/pointer-map? page)
-                   (deref page)
-                   page)))]
+        team  (-> (db/get conn :team {:id (:team-id proj)})
+                  (teams/decode-row))
+
+        _     (-> (cfeat/get-team-enabled-features cf/flags team)
+                  (cfeat/check-client-features! (:features params))
+                  (cfeat/check-file-features! (:features file)))
+
+        page  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+                (let [page-id (or page-id (-> file :data :pages first))
+                      page    (dm/get-in file [:data :pages-index page-id])]
+                  (if (pmap/pointer-map? page)
+                    (deref page)
+                    page)))]
+
+    (when-not perms
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "object not found"))
 
     (cond-> (prune-thumbnails page)
       (some? object-id)
@@ -534,9 +559,13 @@
           f.project_id,
           f.created_at,
           f.modified_at,
+          f.data_backend,
+          f.data_ref_id,
           f.name,
+          f.version,
           f.is_shared,
-          ft.media_id
+          ft.media_id,
+          p.team_id
      from file as f
     inner join project as p on (p.id = f.project_id)
      left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn and ft.deleted_at is null)
@@ -546,7 +575,6 @@
       and p.team_id = ?
     order by f.modified_at desc")
 
-
 (defn- get-library-summary
   [cfg {:keys [id data] :as file}]
   (letfn [(assets-sample [assets limit]
@@ -570,12 +598,16 @@
   (teams/check-read-permissions! conn profile-id team-id)
   (->> (db/exec! conn [sql:team-shared-files team-id])
        (into #{} (comp
-                  (map decode-row)
+                  ;; NOTE: this decode operation is a workaround for a
+                  ;; fast fix, this should be approached with a more
+                  ;; efficient implementation, for now it loads all
+                  ;; the files in memory.
+                  (map (partial bfc/decode-file cfg))
                   (map (fn [row]
                          (if-let [media-id (:media-id row)]
                            (-> row
                                (dissoc :media-id)
-                               (assoc :thumbnail-uri (resolve-public-uri media-id)))
+                               (assoc :thumbnail-id media-id))
                            (dissoc row :media-id))))
                   (map #(assoc % :library-summary (get-library-summary cfg %)))
                   (map #(dissoc % :data))))))
@@ -593,41 +625,6 @@
 
 ;; --- COMMAND QUERY: get-file-libraries
 
-(def ^:private sql:get-file-libraries
-  "WITH RECURSIVE libs AS (
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-      WHERE flr.file_id = ?::uuid
-    UNION
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-       JOIN libs AS l ON (flr.file_id = l.id)
-   )
-   SELECT l.id,
-          l.features,
-          l.project_id,
-          l.created_at,
-          l.modified_at,
-          l.deleted_at,
-          l.name,
-          l.revn,
-          l.vern,
-          l.synced_at
-     FROM libs AS l
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
-
-(defn get-file-libraries
-  [conn file-id]
-  (into []
-        (comp
-         ;; FIXME: :is-indirect set to false to all rows looks
-         ;; completly useless
-         (map #(assoc % :is-indirect false))
-         (map decode-row))
-        (db/exec! conn [sql:get-file-libraries file-id])))
-
 (def ^:private schema:get-file-libraries
   [:map {:title "get-file-libraries"}
    [:file-id ::sm/uuid]])
@@ -639,7 +636,7 @@
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id file-id)
-    (get-file-libraries conn file-id)))
+    (bfc/get-file-libraries conn file-id)))
 
 
 ;; --- COMMAND QUERY: Files that use this File library
@@ -682,7 +679,8 @@
             f.name,
             f.is_shared,
             ft.media_id AS thumbnail_id,
-            row_number() over w as row_num
+            row_number() over w as row_num,
+            p.team_id
        from file as f
       inner join project as p on (p.id = f.project_id)
        left join file_thumbnail as ft on (ft.file_id = f.id
@@ -723,11 +721,13 @@
                              :project-id project-id
                              :file-id id)
 
-        file (get-file cfg id :project-id project-id)]
+        file (get-file cfg id
+                       :project-id project-id
+                       :read-only? true)]
 
     (-> (cfeat/get-team-enabled-features cf/flags team)
         (cfeat/check-client-features! (:features params))
-        (cfeat/check-file-features! (:features file) (:features params)))
+        (cfeat/check-file-features! (:features file)))
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
       {:name             (:name file)
@@ -796,17 +796,17 @@
     [:id ::sm/uuid]
     [:name [:string {:max 250}]]
     [:created-at ::dt/instant]
-    [:modified-at ::dt/instant]]}
+    [:modified-at ::dt/instant]]
 
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [file (rename-file conn params)]
-      (rph/with-meta
-        (select-keys file [:id :name :created-at :modified-at])
-        {::audit/props {:project-id (:project-id file)
-                        :created-at (:created-at file)
-                        :modified-at (:modified-at file)}}))))
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [file (rename-file conn params)]
+    (rph/with-meta
+      (select-keys file [:id :name :created-at :modified-at])
+      {::audit/props {:project-id (:project-id file)
+                      :created-at (:created-at file)
+                      :modified-at (:modified-at file)}})))
 
 ;; --- MUTATION COMMAND: set-file-shared
 
@@ -942,12 +942,13 @@
 ;; --- MUTATION COMMAND: delete-file
 
 (defn- mark-file-deleted
-  [conn file-id]
-  (let [file (db/update! conn :file
-                         {:deleted-at (dt/now)}
-                         {:id file-id}
-                         {::db/return-keys [:id :name :is-shared :deleted-at
-                                            :project-id :created-at :modified-at]})]
+  [conn team file-id]
+  (let [delay (ldel/get-deletion-delay team)
+        file  (db/update! conn :file
+                          {:deleted-at (dt/in-future delay)}
+                          {:id file-id}
+                          {::db/return-keys [:id :name :is-shared :deleted-at
+                                             :project-id :created-at :modified-at]})]
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :file
@@ -963,7 +964,11 @@
 (defn- delete-file
   [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
   (check-edition-permissions! conn profile-id id)
-  (let [file (mark-file-deleted conn id)]
+  (let [team (teams/get-team conn
+                             :profile-id profile-id
+                             :file-id id)
+        file (mark-file-deleted conn team id)]
+
     (rph/with-meta (rph/wrap)
       {::audit/props {:project-id (:project-id file)
                       :name (:name file)
@@ -998,15 +1003,17 @@
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:link-file-to-library}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id library-id] :as params}]
+  [cfg {:keys [::rpc/profile-id file-id library-id] :as params}]
   (when (= file-id library-id)
     (ex/raise :type :validation
               :code :invalid-library
               :hint "A file cannot be linked to itself"))
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (check-edition-permissions! conn profile-id library-id)
-    (link-file-to-library conn params)))
+
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn]}]
+                (check-edition-permissions! conn profile-id file-id)
+                (check-edition-permissions! conn profile-id library-id)
+                (link-file-to-library conn params))))
 
 ;; --- MUTATION COMMAND: unlink-file-from-library
 
@@ -1024,12 +1031,12 @@
 (sv/defmethod ::unlink-file-from-library
   {::doc/added "1.17"
    ::webhooks/event? true
-   ::sm/params schema:unlink-file-to-library}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (unlink-file-from-library conn params)
-    nil))
+   ::sm/params schema:unlink-file-to-library
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (unlink-file-from-library conn params)
+  nil)
 
 ;; --- MUTATION COMMAND: update-sync
 
@@ -1049,12 +1056,11 @@
 (sv/defmethod ::update-file-library-sync-status
   "Update the synchronization status of a file->library link"
   {::doc/added "1.17"
-   ::sm/params schema:update-file-library-sync-status}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (update-sync conn params)))
-
+   ::sm/params schema:update-file-library-sync-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (update-sync conn params))
 
 ;; --- MUTATION COMMAND: ignore-sync
 
@@ -1075,9 +1081,9 @@
 (sv/defmethod ::ignore-file-library-sync-status
   "Ignore updates in linked files"
   {::doc/added "1.17"
-   ::sm/params schema:ignore-file-library-sync-status}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (->  (ignore-sync conn params)
-         (update :features db/decode-pgarray #{}))))
+   ::sm/params schema:ignore-file-library-sync-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (->  (ignore-sync conn params)
+       (update :features db/decode-pgarray #{})))

backend/src/app/rpc/commands/files_create.clj

@@ -6,13 +6,13 @@
 
 (ns app.rpc.commands.files-create
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
    [app.common.types.file :as ctf]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -21,7 +21,6 @@
    [app.rpc.doc :as-alias doc]
    [app.rpc.permissions :as perms]
    [app.rpc.quotes :as quotes]
-   [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -48,34 +47,19 @@
 
   (binding [pmap/*tracked* (pmap/create-tracked)
             cfeat/*current* features]
-    (let [file     (ctf/make-file {:id id
-                                   :project-id project-id
-                                   :name name
-                                   :revn revn
-                                   :is-shared is-shared
-                                   :features features
-                                   :ignore-sync-until ignore-sync-until
-                                   :modified-at modified-at
-                                   :deleted-at deleted-at
-                                   :create-page create-page
-                                   :page-id page-id})
-
-          file     (if (contains? features "fdata/objects-map")
-                     (feat.fdata/enable-objects-map file)
-                     file)
-
-          file     (if (contains? features "fdata/pointer-map")
-                     (feat.fdata/enable-pointer-map file)
-                     file)]
-
-      (db/insert! conn :file
-                  (-> file
-                      (update :data blob/encode)
-                      (update :features db/encode-pgarray conn "text"))
-                  {::db/return-keys false})
-
-      (when (contains? features "fdata/pointer-map")
-        (feat.fdata/persist-pointers! cfg (:id file)))
+    (let [file (ctf/make-file {:id id
+                               :project-id project-id
+                               :name name
+                               :revn revn
+                               :is-shared is-shared
+                               :features features
+                               :ignore-sync-until ignore-sync-until
+                               :modified-at modified-at
+                               :deleted-at deleted-at}
+                              {:create-page create-page
+                               :page-id page-id})
+          file (-> (bfc/insert-file! cfg file)
+                   (bfc/decode-row))]
 
       (->> (assoc params :file-id (:id file) :role :owner)
            (create-file-role! conn))
@@ -107,9 +91,6 @@
                                  :project-id project-id)
         team-id  (:id team)
 
-        ;; When we create files, we only need to respect the team
-        ;; features, because some features can be enabled
-        ;; globally, but the team is still not migrated properly.
         features (-> (cfeat/get-team-enabled-features cf/flags team)
                      (cfeat/check-client-features! (:features params)))
 
@@ -123,25 +104,28 @@
 
         params   (-> params
                      (assoc :profile-id profile-id)
-                     (assoc :features (set/difference features cfeat/frontend-only-features)))]
+                     (assoc :features features))]
 
     (quotes/check! cfg {::quotes/id ::quotes/files-per-project
                         ::quotes/team-id team-id
                         ::quotes/profile-id profile-id
                         ::quotes/project-id project-id})
 
-    ;; FIXME: IMPORTANT: this code can have race
-    ;; conditions, because we have no locks for updating
-    ;; team so, creating two files concurrently can lead
-    ;; to lost team features updating
+    ;; FIXME: IMPORTANT: this code can have race conditions, because
+    ;; we have no locks for updating team so, creating two files
+    ;; concurrently can lead to lost team features updating
 
-    ;; When newly computed features does not match exactly with
-    ;; the features defined on team row, we update it.
-    (when (not= features (:features team))
-      (let [features (db/create-array conn "text" features)]
+    (when-let [features (-> features
+                            (set/difference (:features team))
+                            (set/difference cfeat/no-team-inheritable-features)
+                            (not-empty))]
+      (let [features (->> features
+                          (set/union (:features team))
+                          (db/create-array conn "text"))]
         (db/update! conn :team
                     {:features features}
-                    {:id team-id})))
+                    {:id (:id team)}
+                    {::db/return-keys false})))
 
     (-> (create-file cfg params)
         (vary-meta assoc ::audit/props {:team-id team-id}))))

backend/src/app/rpc/commands/files_share.clj

@@ -33,11 +33,11 @@
   pages of a file with specific permissions (who-comment and who-inspect)."
   {::doc/added "1.18"
    ::doc/module :files
-   ::sm/params schema:create-share-link}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (create-share-link conn (assoc params :profile-id profile-id))))
+   ::sm/params schema:create-share-link
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (create-share-link conn (assoc params :profile-id profile-id)))
 
 (defn create-share-link
   [conn {:keys [profile-id file-id pages who-comment who-inspect]}]
@@ -61,10 +61,10 @@
 (sv/defmethod ::delete-share-link
   {::doc/added "1.18"
    ::doc/module ::files
-   ::sm/params schema:delete-share-link}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [slink (db/get-by-id conn :share-link id)]
-      (files/check-edition-permissions! conn profile-id (:file-id slink))
-      (db/delete! conn :share-link {:id id})
-      nil)))
+   ::sm/params schema:delete-share-link
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
+  (let [slink (db/get-by-id conn :share-link id)]
+    (files/check-edition-permissions! conn profile-id (:file-id slink))
+    (db/delete! conn :share-link {:id id})
+    nil))

backend/src/app/rpc/commands/files_snapshot.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files-snapshot
   (:require
+   [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
@@ -22,7 +23,6 @@
    [app.rpc.quotes :as quotes]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
@@ -58,26 +58,6 @@
                  (files/check-read-permissions! conn profile-id file-id)
                  (get-file-snapshots conn file-id))))
 
-(def ^:private sql:get-file
-  "SELECT f.*,
-          p.id AS project_id,
-          p.team_id AS team_id
-     FROM file AS f
-    INNER JOIN project AS p ON (p.id = f.project_id)
-    WHERE f.id = ?")
-
-(defn- get-file
-  [cfg file-id]
-  (let [file (->> (db/exec-one! cfg [sql:get-file file-id])
-                  (feat.fdata/resolve-file-data cfg))]
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-      (-> file
-          (update :data blob/decode)
-          (update :data feat.fdata/process-pointers deref)
-          (update :data feat.fdata/process-objects (partial into {}))
-          (update :data assoc ::id file-id)
-          (update :data blob/encode)))))
-
 (defn- generate-snapshot-label
   []
   (let [ts (-> (dt/now)
@@ -87,49 +67,53 @@
     (str "snapshot-" ts)))
 
 (defn create-file-snapshot!
-  [cfg profile-id file-id label]
-  (let [file (get-file cfg file-id)
+  [cfg file & {:keys [label created-by deleted-at profile-id]
+               :or {deleted-at :default
+                    created-by :system}}]
+
+  (assert (#{:system :user :admin} created-by)
+          "expected valid keyword for created-by")
+
+  (let [conn
+        (db/get-connection cfg)
 
-        ;; NOTE: final user never can provide label as `:system`
-        ;; keyword because the validator implies label always as
-        ;; string; keyword is used for signal a special case
         created-by
-        (if (= label :system)
-          "system"
-          "user")
+        (name created-by)
 
         deleted-at
-        (if (= label :system)
+        (cond
+          (= deleted-at :default)
           (dt/plus (dt/now) (cf/get-deletion-delay))
+
+          (dt/instant? deleted-at)
+          deleted-at
+
+          :else
           nil)
 
         label
-        (if (= label :system)
-          (str "internal/snapshot/" (:revn file))
-          (or label (generate-snapshot-label)))
+        (or label (generate-snapshot-label))
 
         snapshot-id
-        (uuid/next)]
+        (uuid/next)
 
-    (-> cfg
-        (assoc ::quotes/profile-id profile-id)
-        (assoc ::quotes/project-id (:project-id file))
-        (assoc ::quotes/team-id (:team-id file))
-        (assoc ::quotes/file-id (:id file))
-        (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
-                       {::quotes/id ::quotes/snapshots-per-team}))
+        data
+        (blob/encode (:data file))
+
+        features
+        (db/encode-pgarray (:features file) conn "text")]
 
     (l/debug :hint "creating file snapshot"
-             :file-id (str file-id)
+             :file-id (str (:id file))
              :id (str snapshot-id)
              :label label)
 
     (db/insert! cfg :file-change
                 {:id snapshot-id
                  :revn (:revn file)
-                 :data (:data file)
+                 :data data
                  :version (:version file)
-                 :features (:features file)
+                 :features features
                  :profile-id profile-id
                  :file-id (:id file)
                  :label label
@@ -146,12 +130,25 @@
 
 (sv/defmethod ::create-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:create-file-snapshot}
-  [cfg {:keys [::rpc/profile-id file-id label]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (files/check-edition-permissions! conn profile-id file-id)
-                (create-file-snapshot! cfg profile-id file-id label))))
+   ::sm/params schema:create-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id label]}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (let [file    (bfc/get-file cfg file-id)
+        project (db/get-by-id cfg :project (:project-id file))]
+
+    (-> cfg
+        (assoc ::quotes/profile-id profile-id)
+        (assoc ::quotes/project-id (:project-id file))
+        (assoc ::quotes/team-id (:team-id project))
+        (assoc ::quotes/file-id (:id file))
+        (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
+                       {::quotes/id ::quotes/snapshots-per-team}))
+
+    (create-file-snapshot! cfg file
+                           {:label label
+                            :profile-id profile-id
+                            :created-by :user})))
 
 (defn restore-file-snapshot!
   [{:keys [::db/conn ::mbus/msgbus] :as cfg} file-id snapshot-id]
@@ -237,8 +234,11 @@
   (db/tx-run! cfg
               (fn [{:keys [::db/conn] :as cfg}]
                 (files/check-edition-permissions! conn profile-id file-id)
-                (create-file-snapshot! cfg profile-id file-id :system)
-                (restore-file-snapshot! cfg file-id id))))
+                (let [file (bfc/get-file cfg file-id)]
+                  (create-file-snapshot! cfg file
+                                         {:profile-id profile-id
+                                          :created-by :system})
+                  (restore-file-snapshot! cfg file-id id)))))
 
 (def ^:private schema:update-file-snapshot
   [:map {:title "update-file-snapshot"}

backend/src/app/rpc/commands/files_temp.clj

@@ -14,7 +14,6 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.components-v2 :as feat.compv2]
    [app.features.fdata :as fdata]
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
@@ -110,7 +109,7 @@
 ;; --- MUTATION COMMAND: persist-temp-file
 
 (defn persist-temp-file
-  [{:keys [::db/conn] :as cfg} {:keys [id ::rpc/profile-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
   (let [file (files/get-file cfg id
                              :migrate? false
                              :lock-for-update? true)]
@@ -119,7 +118,6 @@
       (ex/raise :type :validation
                 :code :cant-persist-already-persisted-file))
 
-
     (let [changes (->> (db/cursor conn
                                   (sql/select :file-change {:file-id id}
                                               {:order-by [[:revn :asc]]})
@@ -147,19 +145,6 @@
                    :revn 1
                    :data (blob/encode (:data file))}
                   {:id id})
-
-      (let [team          (teams/get-team conn :profile-id profile-id :project-id (:project-id file))
-            file-features (:features file)
-            team-features (cfeat/get-team-enabled-features cf/flags team)]
-        (when (and (contains? team-features "components/v2")
-                   (not (contains? file-features "components/v2")))
-          ;; Migrate components v2
-          (feat.compv2/migrate-file! cfg
-                                     (:id file)
-                                     :max-procs 2
-                                     :validate? true
-                                     :throw-on-validate? true)))
-
       nil)))
 
 (def ^:private schema:persist-temp-file

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -10,7 +10,6 @@
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
    [app.common.files.helpers :as cfh]
-   [app.common.files.migrations :as fmg]
    [app.common.geom.shapes :as gsh]
    [app.common.schema :as sm]
    [app.common.thumbnails :as thc]
@@ -18,7 +17,6 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -180,8 +178,7 @@
 (def ^:private
   schema:get-file-data-for-thumbnail
   [:map {:title "get-file-data-for-thumbnail"}
-   [:file-id ::sm/uuid]
-   [:features {:optional true} ::cfeat/features]])
+   [:file-id ::sm/uuid]])
 
 (def ^:private
   schema:partial-file
@@ -201,18 +198,16 @@
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
                  (files/check-read-permissions! conn profile-id file-id)
 
-                 (let [team     (teams/get-team conn
-                                                :profile-id profile-id
-                                                :file-id file-id)
+                 (let [team (teams/get-team conn
+                                            :profile-id profile-id
+                                            :file-id file-id)
 
-                       file     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                                  (-> (files/get-file cfg file-id :migrate? false)
-                                      (update :data feat.fdata/process-pointers deref)
-                                      (fmg/migrate-file)))]
+                       file (files/get-file cfg file-id
+                                            :preload-pointers? true
+                                            :read-only? true)]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
-                       (cfeat/check-client-features! (:features params))
-                       (cfeat/check-file-features! (:features file) (:features params)))
+                       (cfeat/check-file-features! (:features file)))
 
                    {:file-id file-id
                     :revn (:revn file)

backend/src/app/rpc/commands/files_update.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files-update
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
@@ -18,6 +19,8 @@
    [app.config :as cf]
    [app.db :as db]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
+   [app.features.logical-deletion :as ldel]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
@@ -118,6 +121,7 @@
 (sv/defmethod ::update-file
   {::climit/id [[:update-file/by-profile ::rpc/profile-id]
                 [:update-file/global]]
+
    ::webhooks/event? true
    ::webhooks/batch-timeout (dt/duration "2m")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
@@ -139,7 +143,7 @@
 
                           features (-> (cfeat/get-team-enabled-features cf/flags team)
                                        (cfeat/check-client-features! (:features params))
-                                       (cfeat/check-file-features! (:features file) (:features params)))
+                                       (cfeat/check-file-features! (:features file)))
 
                           changes  (if changes-with-metadata
                                      (->> changes-with-metadata (mapcat :changes) vec)
@@ -174,12 +178,19 @@
                                             :stored-revn (:revn file)}))
 
                       ;; When newly computed features does not match exactly with
-                      ;; the features defined on team row, we update it.
-                      (when (not= features (:features team))
-                        (let [features (db/create-array conn "text" features)]
+                      ;; the features defined on team row, we update it
+                      (when-let [features (-> features
+                                              (set/difference (:features team))
+                                              (set/difference cfeat/no-team-inheritable-features)
+                                              (not-empty))]
+                        (let [features (->> features
+                                            (set/union (:features team))
+                                            (db/create-array conn "text"))]
                           (db/update! conn :team
                                       {:features features}
-                                      {:id (:id team)})))
+                                      {:id (:id team)}
+                                      {::db/return-keys false})))
+
 
                       (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
 
@@ -199,39 +210,30 @@
 
   Only intended for internal use on this module."
   [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
-   {:keys [profile-id file features changes session-id skip-validate] :as params}]
+   {:keys [profile-id file team features changes session-id skip-validate] :as params}]
 
   (let [;; Retrieve the file data
-        file (feat.fdata/resolve-file-data cfg file)
+        file  (feat.fmigr/resolve-applied-migrations cfg file)
+        file  (feat.fdata/resolve-file-data cfg file)
+        file  (assoc file :features
+                     (-> features
+                         (set/difference cfeat/frontend-only-features)
+                         (set/union (:features file))))]
 
-        file (assoc file :features
-                    (-> features
-                        (set/difference cfeat/frontend-only-features)
-                        (set/union (:features file))))
+    ;; We create a new lexycal scope for clearly delimit the result of
+    ;; executing this update file operation and all its side effects
+    (let [file (px/invoke! executor
+                           (fn []
+                             ;; Process the file data on separated thread for avoid to do
+                             ;; the CPU intensive operation on vthread.
+                             (binding [cfeat/*current*  features
+                                       cfeat/*previous* (:features file)]
+                               (update-file-data! cfg file
+                                                  process-changes-and-validate
+                                                  changes skip-validate))))]
 
-        ;; Process the file data on separated thread for avoid to do
-        ;; the CPU intensive operation on vthread.
-        file  (px/invoke! executor
-                          (fn []
-                            (binding [cfeat/*current*  features
-                                      cfeat/*previous* (:features file)]
-                              (update-file-data! cfg file
-                                                 process-changes-and-validate
-                                                 changes skip-validate))))]
-
-    (when (feat.fdata/offloaded? file)
-      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-        (some->> (:data-ref-id file) (sto/touch-object! storage))))
-
-    (persist-file! cfg file)
-
-    (let [params   (assoc params :file file)
-          response {:revn (:revn file)
-                    :lagged (get-lagged-changes conn params)}
-          features (db/create-array conn "text" (:features file))
-          deleted-at (if (::snapshot-data file)
-                       (dt/plus timestamp (cf/get-deletion-delay))
-                       (dt/plus timestamp (dt/duration {:hours 1})))]
+      (feat.fmigr/upsert-migrations! conn file)
+      (persist-file! cfg file)
 
       ;; Insert change (xlog) with deleted_at in a future data for
       ;; make them automatically eleggible for GC once they expires
@@ -241,19 +243,27 @@
                    :profile-id profile-id
                    :created-at timestamp
                    :updated-at timestamp
-                   :deleted-at deleted-at
+                   :deleted-at (if (::snapshot-data file)
+                                 (dt/plus timestamp (ldel/get-deletion-delay team))
+                                 (dt/plus timestamp (dt/duration {:hours 1})))
                    :file-id (:id file)
                    :revn (:revn file)
                    :version (:version file)
-                   :features features
+                   :features (:features file)
                    :label (::snapshot-label file)
                    :data (::snapshot-data file)
                    :changes (blob/encode changes)}
                   {::db/return-keys false})
 
       ;; Send asynchronous notifications
-      (send-notifications! cfg params)
+      (send-notifications! cfg params file))
 
+    (when (feat.fdata/offloaded? file)
+      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
+        (some->> (:data-ref-id file) (sto/touch-object! storage))))
+
+    (let [response {:revn (:revn file)
+                    :lagged (get-lagged-changes conn params)}]
       (vary-meta response assoc ::audit/replace-props
                  {:id         (:id file)
                   :name       (:name file)
@@ -263,9 +273,10 @@
 
 (defn update-file!
   "A public api that allows apply a transformation to a file with all context setup."
-  [cfg file-id update-fn & args]
+  [{:keys [::db/conn] :as cfg} file-id update-fn & args]
   (let [file (get-file cfg file-id)
         file (apply update-file-data! cfg file update-fn args)]
+    (feat.fmigr/upsert-migrations! conn file)
     (persist-file! cfg file)))
 
 (def ^:private sql:get-file
@@ -293,8 +304,7 @@
 
   It also updates the project modified-at attr."
   [{:keys [::db/conn ::timestamp]} file]
-  (let [features (db/create-array conn "text" (:features file))
-        ;; The timestamp can be nil because this function is also
+  (let [;; The timestamp can be nil because this function is also
         ;; intended to be used outside of this module
         modified-at (or timestamp (dt/now))]
 
@@ -307,7 +317,7 @@
                 {:revn (:revn file)
                  :data (:data file)
                  :version (:version file)
-                 :features features
+                 :features (:features file)
                  :data-backend nil
                  :data-ref-id nil
                  :modified-at modified-at
@@ -331,6 +341,7 @@
                                     (-> data
                                         (blob/decode)
                                         (assoc :id (:id file)))))
+          libs (delay (bfc/get-resolved-file-libraries cfg file))
 
           ;; For avoid unnecesary overhead of creating multiple pointers
           ;; and handly internally with objects map in their worst
@@ -341,7 +352,7 @@
                  (-> file
                      (update :data feat.fdata/process-pointers deref)
                      (update :data feat.fdata/process-objects (partial into {}))
-                     (fmg/migrate-file))
+                     (fmg/migrate-file libs))
                  file)
 
           file (apply update-fn cfg file args)
@@ -366,39 +377,10 @@
                    (-> file
                        (assoc ::snapshot-data snapshot)
                        (assoc ::snapshot-label label)))
-                 file)
+                 file)]
 
-          file (cond-> file
-                 (contains? cfeat/*current* "fdata/objects-map")
-                 (feat.fdata/enable-objects-map)
+      (bfc/encode-file cfg file))))
 
-                 (contains? cfeat/*current* "fdata/pointer-map")
-                 (feat.fdata/enable-pointer-map)
-
-                 :always
-                 (update :data blob/encode))]
-
-      (feat.fdata/persist-pointers! cfg id)
-
-      file)))
-
-(defn- get-file-libraries
-  "A helper for preload file libraries, mainly used for perform file
-  semantical and structural validation"
-  [{:keys [::db/conn] :as cfg} file]
-  (->> (files/get-file-libraries conn (:id file))
-       (into [file] (map (fn [{:keys [id]}]
-                           (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-                                     pmap/*tracked* nil]
-                             ;; We do not resolve the objects maps here
-                             ;; because there is a lower probability that all
-                             ;; shapes needed to be loded into memory, so we
-                             ;; leeave it on lazy status
-                             (-> (files/get-file cfg id :migrate? false)
-                                 (update :data feat.fdata/process-pointers deref) ; ensure all pointers resolved
-                                 (update :data feat.fdata/process-objects (partial into {}))
-                                 (fmg/migrate-file))))))
-       (d/index-by :id)))
 
 (defn- soft-validate-file-schema!
   [file]
@@ -415,19 +397,37 @@
       (l/error :hint "file validation error"
                :cause cause))))
 
+
 (defn- process-changes-and-validate
   [cfg file changes skip-validate]
   (let [;; WARNING: this ruins performance; maybe we need to find
         ;; some other way to do general validation
-        libs (when (and (or (contains? cf/flags :file-validation)
-                            (contains? cf/flags :soft-file-validation))
-                        (not skip-validate))
-               (get-file-libraries cfg file))
+        libs
+        (when (and (or (contains? cf/flags :file-validation)
+                       (contains? cf/flags :soft-file-validation))
+                   (not skip-validate))
+          (bfc/get-resolved-file-libraries cfg file))
 
-        file (-> (files/check-version! file)
-                 (update :revn inc)
-                 (update :data cpc/process-changes changes)
-                 (update :data d/without-nils))]
+
+        ;; The main purpose of this atom is provide a contextual state
+        ;; for the changes subsystem where optionally some hints can
+        ;; be provided for the changes processing. Right now we are
+        ;; using it for notify about the existence of media refs when
+        ;; a new shape is added.
+        state
+        (atom {})
+
+        file
+        (binding [cpc/*state* state]
+          (-> (files/check-version! file)
+              (update :revn inc)
+              (update :data cpc/process-changes changes)
+              (update :data d/without-nils)))
+
+        file
+        (if-let [media-refs (-> @state :media-refs not-empty)]
+          (bfc/update-media-references! cfg file media-refs)
+          file)]
 
     (binding [pmap/*tracked* nil]
       (when (contains? cf/flags :soft-file-validation)
@@ -474,7 +474,7 @@
        (vec)))
 
 (defn- send-notifications!
-  [cfg {:keys [file team changes session-id] :as params}]
+  [cfg {:keys [team changes session-id] :as params} file]
   (let [lchanges (filter library-change? changes)
         msgbus   (::mbus/msgbus cfg)]
 

backend/src/app/rpc/commands/fonts.clj

@@ -12,6 +12,7 @@
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.db.sql :as-alias sql]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -80,9 +81,9 @@
 (def ^:private schema:create-font-variant
   [:map {:title "create-font-variant"}
    [:team-id ::sm/uuid]
-   [:data [:map-of :string :any]]
+   [:data [:map-of ::sm/text ::sm/any]]
    [:font-id ::sm/uuid]
-   [:font-family :string]
+   [:font-family ::sm/text]
    [:font-weight [::sm/one-of {:format "number"} valid-weight]]
    [:font-style [::sm/one-of {:format "string"} valid-style]]])
 
@@ -202,32 +203,40 @@
 (sv/defmethod ::delete-font
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font}
-  [cfg {:keys [::rpc/profile-id id team-id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (teams/check-edition-permissions! conn profile-id team-id)
-                (let [fonts   (db/query conn :team-font-variant
-                                        {:team-id team-id
-                                         :font-id id
-                                         :deleted-at nil}
-                                        {::sql/for-update true})
-                      tnow    (dt/now)]
+   ::sm/params schema:delete-font
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
+  (let [team  (teams/get-team conn
+                              :profile-id profile-id
+                              :team-id team-id)
 
-                  (when-not (seq fonts)
-                    (ex/raise :type :not-found
-                              :code :object-not-found))
+        fonts (db/query conn :team-font-variant
+                        {:team-id team-id
+                         :font-id id
+                         :deleted-at nil}
+                        {::sql/for-update true})
 
-                  (doseq [font fonts]
-                    (db/update! conn :team-font-variant
-                                {:deleted-at tnow}
-                                {:id (:id font)}))
+        delay (ldel/get-deletion-delay team)
+        tnow  (dt/in-future delay)]
 
-                  (rph/with-meta (rph/wrap)
-                    {::audit/props {:id id
-                                    :team-id team-id
-                                    :name (:font-family (peek fonts))
-                                    :profile-id profile-id}})))))
+    (teams/check-edition-permissions! (:permissions team))
+
+    (when-not (seq fonts)
+      (ex/raise :type :not-found
+                :code :object-not-found))
+
+
+    (doseq [font fonts]
+      (db/update! conn :team-font-variant
+                  {:deleted-at tnow}
+                  {:id (:id font)}
+                  {::db/return-keys false}))
+
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:id id
+                      :team-id team-id
+                      :name (:font-family (peek fonts))
+                      :profile-id profile-id}})))
 
 ;; --- DELETE FONT VARIANT
 
@@ -239,19 +248,23 @@
 (sv/defmethod ::delete-font-variant
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font-variant}
-  [cfg {:keys [::rpc/profile-id id team-id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (teams/check-edition-permissions! conn profile-id team-id)
-                (let [variant (db/get conn :team-font-variant
-                                      {:id id :team-id team-id}
-                                      {::sql/for-update true})]
+   ::sm/params schema:delete-font-variant
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
+  (let [team    (teams/get-team conn
+                                :profile-id profile-id
+                                :team-id team-id)
+        variant (db/get conn :team-font-variant
+                        {:id id :team-id team-id}
+                        {::sql/for-update true})
+        delay   (ldel/get-deletion-delay team)]
 
-                  (db/update! conn :team-font-variant
-                              {:deleted-at (dt/now)}
-                              {:id (:id variant)})
+    (teams/check-edition-permissions! (:permissions team))
+    (db/update! conn :team-font-variant
+                {:deleted-at (dt/in-future delay)}
+                {:id (:id variant)}
+                {::db/return-keys false})
 
-                  (rph/with-meta (rph/wrap)
-                    {::audit/props {:font-family (:font-family variant)
-                                    :font-id (:font-id variant)}})))))
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:font-family (:font-family variant)
+                      :font-id (:font-id variant)}})))

backend/src/app/rpc/commands/management.clj

@@ -9,6 +9,7 @@
   (:require
    [app.binfile.common :as bfc]
    [app.binfile.v1 :as bf.v1]
+   [app.binfile.v3 :as bf.v3]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
@@ -25,6 +26,7 @@
    [app.rpc.doc :as-alias doc]
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
+   [app.storage.tmp :as tmp]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
@@ -54,8 +56,8 @@
     (vswap! bfc/*state* update :index bfc/update-index fmeds :id)
 
     ;; Process and persist file
-    (let [file (->> (bfc/process-file file)
-                    (bfc/persist-file! cfg))]
+    (let [file (bfc/process-file cfg file)]
+      (bfc/insert-file! cfg file ::db/return-keys false)
 
       ;; The file profile creation is optional, so when no profile is
       ;; present (when this function is called from profile less
@@ -394,32 +396,49 @@
 ;; --- COMMAND: Clone Template
 
 (defn clone-template
-  [cfg {:keys [project-id profile-id] :as params} template]
-  (db/tx-run! cfg (fn [{:keys [::db/conn ::wrk/executor] :as cfg}]
-                    ;; NOTE: the importation process performs some operations
-                    ;; that are not very friendly with virtual threads, and for
-                    ;; avoid unexpected blocking of other concurrent operations
-                    ;; we dispatch that operation to a dedicated executor.
-                    (let [cfg    (-> cfg
-                                     (assoc ::bf.v1/project-id project-id)
-                                     (assoc ::bf.v1/profile-id profile-id)
-                                     (assoc ::bf.v1/input template))
-                          result (px/invoke! executor (partial bf.v1/import-files! cfg))]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [project-id profile-id] :as params} template]
 
-                      (db/update! conn :project
-                                  {:modified-at (dt/now)}
-                                  {:id project-id})
+  ;; NOTE: the importation process performs some operations
+  ;; that are not very friendly with virtual threads, and for
+  ;; avoid unexpected blocking of other concurrent operations
+  ;; we dispatch that operation to a dedicated executor.
+  (let [template (tmp/tempfile-from template
+                                    :prefix "penpot.template."
+                                    :suffix ""
+                                    :min-age "30m")
 
-                      (let [props (audit/clean-props params)]
-                        (doseq [file-id result]
-                          (let [props (assoc props :id file-id)
-                                event (-> (audit/event-from-rpc-params params)
-                                          (assoc ::audit/profile-id profile-id)
-                                          (assoc ::audit/name "create-file")
-                                          (assoc ::audit/props props))]
-                            (audit/submit! cfg event))))
+        format   (bfc/parse-file-format template)
+        team     (teams/get-team pool
+                                 :profile-id profile-id
+                                 :project-id project-id)
 
-                      result))))
+        cfg      (-> cfg
+                     (assoc ::bfc/project-id project-id)
+                     (assoc ::bfc/profile-id profile-id)
+                     (assoc ::bfc/input template)
+                     (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
+
+        result   (if (= format :binfile-v3)
+                   (px/invoke! executor (partial bf.v3/import-files! cfg))
+                   (px/invoke! executor (partial bf.v1/import-files! cfg)))]
+
+    (db/tx-run! cfg
+                (fn [{:keys [::db/conn] :as cfg}]
+                  (db/update! conn :project
+                              {:modified-at (dt/now)}
+                              {:id project-id}
+                              {::db/return-keys false})
+
+                  (let [props (audit/clean-props params)]
+                    (doseq [file-id result]
+                      (let [props (assoc props :id file-id)
+                            event (-> (audit/event-from-rpc-params params)
+                                      (assoc ::audit/profile-id profile-id)
+                                      (assoc ::audit/name "create-file")
+                                      (assoc ::audit/props props))]
+                        (audit/submit! cfg event))))))
+
+    result))
 
 (def ^:private
   schema:clone-template

backend/src/app/rpc/commands/media.clj

@@ -273,15 +273,14 @@
 
 (sv/defmethod ::clone-file-media-object
   {::doc/added "1.17"
-   ::sm/params schema:clone-file-media-object}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (-> (assoc cfg :conn conn)
-        (clone-file-media-object params))))
+   ::sm/params schema:clone-file-media-object
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (clone-file-media-object cfg params))
 
 (defn clone-file-media-object
-  [{:keys [conn]} {:keys [id file-id is-local]}]
+  [{:keys [::db/conn]} {:keys [id file-id is-local]}]
   (let [mobj (db/get-by-id conn :file-media-object id)]
     (db/insert! conn :file-media-object
                 {:id (uuid/next)

backend/src/app/rpc/commands/profile.clj

@@ -41,6 +41,12 @@
 (declare strip-private-attrs)
 (declare verify-password)
 
+(def schema:props-notifications
+  [:map {:title "props-notifications"}
+   [:dashboard-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-invites [::sm/one-of #{:all :none}]]])
+
 (def schema:props
   [:map {:title "ProfileProps"}
    [:plugins {:optional true} schema:plugin-registry]
@@ -51,7 +57,9 @@
    [:v2-info-shown {:optional true} ::sm/boolean]
    [:welcome-file-id {:optional true} [:maybe ::sm/boolean]]
    [:release-notes-viewed {:optional true}
-    [::sm/text {:max 100}]]])
+    [::sm/text {:max 100}]]
+   [:notifications {:optional true} schema:props-notifications]
+   [:workspace-visited {:optional true} ::sm/boolean]])
 
 (def schema:profile
   [:map {:title "Profile"}
@@ -117,32 +125,32 @@
 (sv/defmethod ::update-profile
   {::doc/added "1.0"
    ::sm/params schema:update-profile
-   ::sm/result schema:profile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id fullname lang theme] :as params}]
-  (db/with-atomic [conn pool]
-    ;; NOTE: we need to retrieve the profile independently if we use
-    ;; it or not for explicit locking and avoid concurrent updates of
-    ;; the same row/object.
-    (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
-                      (decode-row))
+   ::sm/result schema:profile
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id fullname lang theme] :as params}]
+  ;; NOTE: we need to retrieve the profile independently if we use
+  ;; it or not for explicit locking and avoid concurrent updates of
+  ;; the same row/object.
+  (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
+                    (decode-row))
 
-          ;; Update the profile map with direct params
-          profile (-> profile
-                      (assoc :fullname fullname)
-                      (assoc :lang lang)
-                      (assoc :theme theme))]
+        ;; Update the profile map with direct params
+        profile (-> profile
+                    (assoc :fullname fullname)
+                    (assoc :lang lang)
+                    (assoc :theme theme))]
 
-      (db/update! conn :profile
-                  {:fullname fullname
-                   :lang lang
-                   :theme theme
-                   :props (db/tjson (:props profile))}
-                  {:id profile-id})
+    (db/update! conn :profile
+                {:fullname fullname
+                 :lang lang
+                 :theme theme
+                 :props (db/tjson (:props profile))}
+                {:id profile-id})
 
-      (-> profile
-          (strip-private-attrs)
-          (d/without-nils)
-          (rph/with-meta {::audit/props (audit/profile->props profile)})))))
+    (-> profile
+        (strip-private-attrs)
+        (d/without-nils)
+        (rph/with-meta {::audit/props (audit/profile->props profile)}))))
 
 
 ;; --- MUTATION: Update Password
@@ -161,21 +169,20 @@
 (sv/defmethod ::update-profile-password
   {::doc/added "1.0"
    ::sm/params schema:update-profile-password
-   ::climit/id :auth/global}
+   ::climit/id :auth/global
+   ::db/transaction true}
   [cfg {:keys [::rpc/profile-id password] :as params}]
+  (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
+        session-id (::session/id params)]
 
-  (db/tx-run! cfg (fn [cfg]
-                    (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
-                          session-id (::session/id params)]
+    (when (= (:email profile) (str/lower (:password params)))
+      (ex/raise :type :validation
+                :code :email-as-password
+                :hint "you can't use your email as password"))
 
-                      (when (= (:email profile) (str/lower (:password params)))
-                        (ex/raise :type :validation
-                                  :code :email-as-password
-                                  :hint "you can't use your email as password"))
-
-                      (update-profile-password! cfg (assoc profile :password password))
-                      (invalidate-profile-session! cfg profile-id session-id)
-                      nil))))
+    (update-profile-password! cfg (assoc profile :password password))
+    (invalidate-profile-session! cfg profile-id session-id)
+    nil))
 
 (defn- invalidate-profile-session!
   "Removes all sessions except the current one."
@@ -200,6 +207,44 @@
                 {:id id})
     nil))
 
+
+;; --- MUTATION: Update notifications
+
+(def ^:private
+  schema:update-profile-notifications
+  [:map {:title "update-profile-notifications"}
+   [:dashboard-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-comments [::sm/one-of #{:all :partial :none}]]
+   [:email-invites [::sm/one-of #{:all :none}]]])
+
+(declare update-notifications!)
+
+(sv/defmethod ::update-profile-notifications
+  {::doc/added "2.4.0"
+   ::sm/params schema:update-profile-notifications
+   ::climit/id :auth/global}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg update-notifications! (assoc params :profile-id profile-id)))
+
+(defn- update-notifications!
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
+  (let [profile (get-profile conn profile-id)
+
+        notifications
+        {:dashboard-comments dashboard-comments
+         :email-comments email-comments
+         :email-invites email-invites}]
+
+    (db/update!
+     conn :profile
+     {:props
+      (-> (:props profile)
+          (assoc :notifications notifications)
+          (db/tjson))}
+     {:id (:id profile)})
+
+    nil))
+
 ;; --- MUTATION: Update Photo
 
 (declare upload-photo)
@@ -395,37 +440,36 @@
 (declare ^:private get-owned-teams)
 
 (sv/defmethod ::delete-profile
-  {::doc/added "1.0"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [teams      (get-owned-teams conn profile-id)
-          deleted-at (dt/now)]
+  {::doc/added "1.0"
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (let [teams      (get-owned-teams conn profile-id)
+        deleted-at (dt/now)]
 
-      ;; If we found owned teams with participants, we don't allow
-      ;; delete profile until the user properly transfer ownership or
-      ;; explicitly removes all participants from the team
-      (when (some pos? (map :participants teams))
-        (ex/raise :type :validation
-                  :code :owner-teams-with-people
-                  :hint "The user need to transfer ownership of owned teams."
-                  :context {:teams (mapv :id teams)}))
+    ;; If we found owned teams with participants, we don't allow
+    ;; delete profile until the user properly transfer ownership or
+    ;; explicitly removes all participants from the team
+    (when (some pos? (map :participants teams))
+      (ex/raise :type :validation
+                :code :owner-teams-with-people
+                :hint "The user need to transfer ownership of owned teams."
+                :context {:teams (mapv :id teams)}))
 
-      ;; Mark profile deleted immediatelly
-      (db/update! conn :profile
-                  {:deleted-at deleted-at}
-                  {:id profile-id})
+    ;; Mark profile deleted immediatelly
+    (db/update! conn :profile
+                {:deleted-at deleted-at}
+                {:id profile-id})
 
-      ;; Schedule cascade deletion to a worker
-      (wrk/submit! {::db/conn conn
-                    ::wrk/task :delete-object
-                    ::wrk/params {:object :profile
-                                  :deleted-at deleted-at
-                                  :id profile-id}})
+    ;; Schedule cascade deletion to a worker
+    (wrk/submit! {::db/conn conn
+                  ::wrk/task :delete-object
+                  ::wrk/params {:object :profile
+                                :deleted-at deleted-at
+                                :id profile-id}})
 
 
-      (-> (rph/wrap nil)
-          (rph/with-transform (session/delete-fn cfg))))))
-
+    (-> (rph/wrap nil)
+        (rph/with-transform (session/delete-fn cfg)))))
 
 ;; --- HELPERS
 
@@ -436,8 +480,7 @@
         JOIN team AS t ON (t.id = tpr.team_id)
        WHERE tpr.is_owner IS TRUE
          AND tpr.profile_id = ?
-         AND (t.deleted_at IS NULL OR
-              t.deleted_at > now())
+         AND t.deleted_at IS NULL
    )
    SELECT tpr.team_id AS id,
           count(tpr.profile_id) - 1 AS participants

backend/src/app/rpc/commands/projects.clj

@@ -11,6 +11,7 @@
    [app.common.schema :as sm]
    [app.db :as db]
    [app.db.sql :as-alias sql]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as webhooks]
    [app.rpc :as-alias rpc]
@@ -219,12 +220,12 @@
    ::sm/params schema:update-project-pin
    ::webhooks/batch-timeout (dt/duration "5s")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id team-id is-pinned] :as params}]
-  (db/with-atomic [conn pool]
-    (check-read-permissions! conn profile-id id)
-    (db/exec-one! conn [sql:update-project-pin team-id id profile-id is-pinned is-pinned])
-    nil))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id team-id is-pinned] :as params}]
+  (check-read-permissions! conn profile-id id)
+  (db/exec-one! conn [sql:update-project-pin team-id id profile-id is-pinned is-pinned])
+  nil)
 
 ;; --- MUTATION: Rename Project
 
@@ -238,24 +239,25 @@
 (sv/defmethod ::rename-project
   {::doc/added "1.18"
    ::sm/params schema:rename-project
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [project (db/get-by-id conn :project id ::sql/for-update true)]
-      (db/update! conn :project
-                  {:name name}
-                  {:id id})
-      (rph/with-meta (rph/wrap)
-        {::audit/props {:team-id (:team-id project)
-                        :prev-name (:name project)}}))))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id name] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [project (db/get-by-id conn :project id ::sql/for-update true)]
+    (db/update! conn :project
+                {:name name}
+                {:id id})
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:team-id (:team-id project)
+                      :prev-name (:name project)}})))
 
 ;; --- MUTATION: Delete Project
 
 (defn- delete-project
-  [conn project-id]
-  (let [project (db/update! conn :project
-                            {:deleted-at (dt/now)}
+  [conn team project-id]
+  (let [delay   (ldel/get-deletion-delay team)
+        project (db/update! conn :project
+                            {:deleted-at (dt/in-future delay)}
                             {:id project-id}
                             {::db/return-keys true})]
 
@@ -272,7 +274,6 @@
 
     project))
 
-
 (def ^:private schema:delete-project
   [:map {:title "delete-project"}
    [:id ::sm/uuid]])
@@ -280,13 +281,16 @@
 (sv/defmethod ::delete-project
   {::doc/added "1.18"
    ::sm/params schema:delete-project
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [project (delete-project conn id)]
-      (rph/with-meta (rph/wrap)
-        {::audit/props {:team-id (:team-id project)
-                        :name (:name project)
-                        :created-at (:created-at project)
-                        :modified-at (:modified-at project)}}))))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [team    (teams/get-team conn
+                                :profile-id profile-id
+                                :project-id id)
+        project (delete-project conn team id)]
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:team-id (:team-id project)
+                      :name (:name project)
+                      :created-at (:created-at project)
+                      :modified-at (:modified-at project)}})))

backend/src/app/rpc/commands/search.clj

@@ -9,7 +9,6 @@
    [app.common.schema :as sm]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :refer [resolve-public-uri]]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]))
 
@@ -61,7 +60,7 @@
                (if-let [media-id (:media-id row)]
                  (-> row
                      (dissoc :media-id)
-                     (assoc :thumbnail-uri (resolve-public-uri media-id)))
+                     (assoc :thumbnail-id media-id))
                  (dissoc row :media-id))))))
 
 (def ^:private schema:search-files

backend/src/app/rpc/commands/teams.clj

@@ -17,6 +17,7 @@
    [app.db :as db]
    [app.db.sql :as sql]
    [app.email :as eml]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.media :as media]
@@ -76,9 +77,10 @@
   (perms/make-check-fn has-read-permissions?))
 
 (defn decode-row
-  [{:keys [features] :as row}]
+  [{:keys [features subscription] :as row}]
   (cond-> row
-    (some? features) (assoc :features (db/decode-pgarray features #{}))))
+    (some? features) (assoc :features (db/decode-pgarray features #{}))
+    (some? subscription) (assoc :subscription (db/decode-transit-pgobject subscription))))
 
 ;; FIXME: move
 
@@ -113,29 +115,42 @@
 
 ;; --- Query: Teams
 
-(declare get-teams)
-
-(def ^:private schema:get-teams
-  [:map {:title "get-teams"}])
-
-(sv/defmethod ::get-teams
-  {::doc/added "1.17"
-   ::sm/params schema:get-teams}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (dm/with-open [conn (db/open pool)]
-    (get-teams conn profile-id)))
-
 (def sql:get-teams-with-permissions
-  "select t.*,
+  "SELECT t.*,
           tp.is_owner,
           tp.is_admin,
           tp.can_edit,
-          (t.id = ?) as is_default
-     from team_profile_rel as tp
-     join team as t on (t.id = tp.team_id)
-    where t.deleted_at is null
-      and tp.profile_id = ?
-    order by tp.created_at asc")
+          (t.id = ?) AS is_default
+     FROM team_profile_rel AS tp
+     JOIN team AS t ON (t.id = tp.team_id)
+    WHERE t.deleted_at IS null
+      AND tp.profile_id = ?
+    ORDER BY tp.created_at ASC")
+
+(def sql:get-teams-with-permissions-and-subscription
+  "SELECT t.*,
+          tp.is_owner,
+          tp.is_admin,
+          tp.can_edit,
+          (t.id = ?) AS is_default,
+
+          jsonb_build_object(
+            '~:type', COALESCE(p.props->'~:subscription'->>'~:type', 'professional'),
+            '~:status', CASE COALESCE(p.props->'~:subscription'->>'~:type', 'professional')
+                          WHEN 'professional' THEN 'active'
+                          ELSE COALESCE(p.props->'~:subscription'->>'~:status', 'incomplete')
+                       END,
+            '~:seats', p.props->'~:quantity'
+          ) AS subscription
+     FROM team_profile_rel AS tp
+     JOIN team AS t ON (t.id = tp.team_id)
+     JOIN team_profile_rel AS tpr
+       ON (tpr.team_id = t.id AND tpr.is_owner IS true)
+     JOIN profile AS p
+       ON (tpr.profile_id = p.id)
+    WHERE t.deleted_at IS null
+      AND tp.profile_id = ?
+    ORDER BY tp.created_at ASC")
 
 (defn process-permissions
   [team]
@@ -150,13 +165,52 @@
         (dissoc :is-owner :is-admin :can-edit)
         (assoc :permissions permissions))))
 
+(def ^:private
+  xform:process-teams
+  (comp
+   (map decode-row)
+   (map process-permissions)))
+
 (defn get-teams
   [conn profile-id]
-  (let [profile (profile/get-profile conn profile-id)]
-    (->> (db/exec! conn [sql:get-teams-with-permissions (:default-team-id profile) profile-id])
-         (map decode-row)
-         (map process-permissions)
-         (vec))))
+  (let [profile (profile/get-profile conn profile-id)
+        sql     (if (contains? cf/flags :subscriptions)
+                  sql:get-teams-with-permissions-and-subscription
+                  sql:get-teams-with-permissions)]
+
+    (->> (db/exec! conn [sql (:default-team-id profile) profile-id])
+         (into [] xform:process-teams))))
+
+(def ^:private schema:get-teams
+  [:map {:title "get-teams"}])
+
+(sv/defmethod ::get-teams
+  {::doc/added "1.17"
+   ::sm/params schema:get-teams}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (get-teams conn profile-id)))
+
+(def ^:private sql:get-owned-teams
+  "SELECT t.id, t.name,
+          (SELECT count(*) FROM team_profile_rel WHERE team_id=t.id) AS total_members
+     FROM team AS t
+     JOIN team_profile_rel AS tpr ON (tpr.team_id = t.id)
+    WHERE t.is_default IS false
+      AND tpr.is_owner IS true
+      AND tpr.profile_id = ?
+      AND t.deleted_at IS NULL")
+
+(defn- get-owned-teams
+  [cfg profile-id]
+  (->> (db/exec! cfg [sql:get-owned-teams profile-id])
+       (into [] (map decode-row))))
+
+(sv/defmethod ::get-owned-teams
+  {::doc/added "2.8.0"
+   ::sm/params schema:get-teams}
+  [cfg {:keys [::rpc/profile-id]}]
+  (get-owned-teams cfg profile-id))
 
 ;; --- Query: Team (by ID)
 
@@ -181,39 +235,43 @@
 (defn get-team
   [conn & {:keys [profile-id team-id project-id file-id] :as params}]
 
-  (dm/assert!
-   "connection or pool is mandatory"
-   (or (db/connection? conn)
-       (db/pool? conn)))
+  (assert (uuid? profile-id) "profile-id is mandatory")
+  (assert (or (db/connection? conn)
+              (db/pool? conn))
+          "connection or pool is mandatory")
 
-  (dm/assert!
-   "profile-id is mandatory"
-   (uuid? profile-id))
+  (let [{:keys [default-team-id] :as profile}
+        (profile/get-profile conn profile-id)
 
-  (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
-        result (cond
-                 (some? team-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
-                                ") SELECT * FROM teams WHERE id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id team-id]))
+        sql
+        (if (contains? cf/flags :subscriptions)
+          sql:get-teams-with-permissions-and-subscription
+          sql:get-teams-with-permissions)
 
-                 (some? project-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                " WHERE p.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id project-id]))
+        result
+        (cond
+          (some? team-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT * FROM teams WHERE id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id team-id]))
 
-                 (some? file-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                "  JOIN file AS f ON (f.project_id = p.id) "
-                                " WHERE f.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id file-id]))
+          (some? project-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT t.* FROM teams AS t "
+                         "  JOIN project AS p ON (p.team_id = t.id) "
+                         " WHERE p.id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id project-id]))
 
-                 :else
-                 (throw (IllegalArgumentException. "invalid arguments")))]
+          (some? file-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT t.* FROM teams AS t "
+                         "  JOIN project AS p ON (p.team_id = t.id) "
+                         "  JOIN file AS f ON (f.project_id = p.id) "
+                         " WHERE f.id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id file-id]))
+
+          :else
+          (throw (IllegalArgumentException. "invalid arguments")))]
 
     (when-not result
       (ex/raise :type :not-found
@@ -286,18 +344,18 @@
 ;; implemented in UI)
 
 (def sql:team-users
-  "select pf.id, pf.fullname, pf.photo_id
+  "select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join team_profile_rel as tpr on (tpr.profile_id = pf.id)
     where tpr.team_id = ?
     union
-   select pf.id, pf.fullname, pf.photo_id
+   select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join project_profile_rel as ppr on (ppr.profile_id = pf.id)
     inner join project as p on (ppr.project_id = p.id)
     where p.team_id = ?
    union
-   select pf.id, pf.fullname, pf.photo_id
+   select pf.id, pf.fullname, pf.photo_id, pf.email
      from profile as pf
     inner join file_profile_rel as fpr on (fpr.profile_id = pf.id)
     inner join file as f on (fpr.file_id = f.id)
@@ -308,6 +366,30 @@
   [conn team-id]
   (db/exec! conn [sql:team-users team-id team-id team-id]))
 
+;; Get the users but add the props property
+(def sql:team-users+props
+  "select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join team_profile_rel as tpr on (tpr.profile_id = pf.id)
+    where tpr.team_id = ?
+    union
+   select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join project_profile_rel as ppr on (ppr.profile_id = pf.id)
+    inner join project as p on (ppr.project_id = p.id)
+    where p.team_id = ?
+   union
+   select pf.id, pf.fullname, pf.photo_id, pf.email, pf.props
+     from profile as pf
+    inner join file_profile_rel as fpr on (fpr.profile_id = pf.id)
+    inner join file as f on (fpr.file_id = f.id)
+    inner join project as p on (f.project_id = p.id)
+    where p.team_id = ?")
+
+(defn get-users+props
+  [conn team-id]
+  (db/exec! conn [sql:team-users+props team-id team-id team-id]))
+
 (def sql:get-team-by-file
   "SELECT t.*
      FROM team AS t
@@ -503,14 +585,14 @@
 
 (sv/defmethod ::update-team
   {::doc/added "1.17"
-   ::sm/params schema:update-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (db/update! conn :team
-                {:name name}
-                {:id id})
-    nil))
+   ::sm/params schema:update-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id name]}]
+  (check-edition-permissions! conn profile-id id)
+  (db/update! conn :team
+              {:name name}
+              {:id id})
+  nil)
 
 
 ;; --- Mutation: Leave Team
@@ -568,22 +650,22 @@
 
 (sv/defmethod ::leave-team
   {::doc/added "1.17"
-   ::sm/params schema:leave-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (leave-team conn (assoc params :profile-id profile-id))))
+   ::sm/params schema:leave-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (leave-team conn (assoc params :profile-id profile-id)))
 
 ;; --- Mutation: Delete Team
 
 (defn- delete-team
   "Mark a team for deletion"
-  [conn team-id]
+  [conn {:keys [id] :as team}]
 
-  (let [deleted-at (dt/now)
-        team       (db/update! conn :team
-                               {:deleted-at deleted-at}
-                               {:id team-id}
-                               {::db/return-keys true})]
+  (let [delay (ldel/get-deletion-delay team)
+        team  (db/update! conn :team
+                          {:deleted-at (dt/in-future delay)}
+                          {:id id}
+                          {::db/return-keys true})]
 
     (when (:is-default team)
       (ex/raise :type :validation
@@ -593,8 +675,8 @@
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :team
-                                :deleted-at deleted-at
-                                :id team-id}})
+                                :deleted-at (:deleted-at team)
+                                :id id}})
     team))
 
 (def ^:private schema:delete-team
@@ -603,16 +685,18 @@
 
 (sv/defmethod ::delete-team
   {::doc/added "1.17"
-   ::sm/params schema:delete-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (get-permissions conn profile-id id)]
-      (when-not (:is-owner perms)
-        (ex/raise :type :validation
-                  :code :only-owner-can-delete-team))
+   ::sm/params schema:delete-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (let [team  (get-team conn :profile-id profile-id :team-id id)
+        perms (get team :permissions)]
 
-      (delete-team conn id)
-      nil)))
+    (when-not (:is-owner perms)
+      (ex/raise :type :validation
+                :code :only-owner-can-delete-team))
+
+    (delete-team conn team)
+    nil))
 
 ;; --- Mutation: Team Update Role
 
@@ -690,31 +774,30 @@
 
 (sv/defmethod ::delete-team-member
   {::doc/added "1.17"
-   ::sm/params schema:delete-team-member}
-  [{:keys [::db/pool ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id team-id member-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [team  (get-team pool :profile-id profile-id :team-id team-id)
-          perms (get-permissions conn profile-id team-id)]
-      (when-not (or (:is-owner perms)
-                    (:is-admin perms))
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+   ::sm/params schema:delete-team-member
+   ::db/transaction true}
+  [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id team-id member-id] :as params}]
+  (let [team  (get-team conn :profile-id profile-id :team-id team-id)
+        perms (get-permissions conn profile-id team-id)]
+    (when-not (or (:is-owner perms)
+                  (:is-admin perms))
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (when (= member-id profile-id)
-        (ex/raise :type :validation
-                  :code :cant-remove-yourself))
+    (when (= member-id profile-id)
+      (ex/raise :type :validation
+                :code :cant-remove-yourself))
 
-      (db/delete! conn :team-profile-rel {:profile-id member-id
-                                          :team-id team-id})
+    (db/delete! conn :team-profile-rel {:profile-id member-id
+                                        :team-id team-id})
+    (mbus/pub! msgbus
+               :topic member-id
+               :message {:type :team-membership-change
+                         :change :removed
+                         :team-id team-id
+                         :team-name (:name team)})
 
-      (mbus/pub! msgbus
-                 :topic member-id
-                 :message {:type :team-membership-change
-                           :change :removed
-                           :team-id team-id
-                           :team-name (:name team)})
-
-      nil)))
+    nil))
 
 ;; --- Mutation: Update Team Photo
 
@@ -740,16 +823,16 @@
   (let [team  (get-team pool :profile-id profile-id :team-id team-id)
         photo (profile/upload-photo cfg params)]
 
-    (db/with-atomic [conn pool]
-      (check-admin-permissions! conn profile-id team-id)
-      ;; Mark object as touched for make it ellegible for tentative
-      ;; garbage collection.
-      (when-let [id (:photo-id team)]
-        (sto/touch-object! storage id))
+    (check-admin-permissions! pool profile-id team-id)
 
-      ;; Save new photo
-      (db/update! pool :team
-                  {:photo-id (:id photo)}
-                  {:id team-id})
+    ;; Mark object as touched for make it ellegible for tentative
+    ;; garbage collection.
+    (when-let [id (:photo-id team)]
+      (sto/touch-object! storage id))
 
-      (assoc team :photo-id (:id photo)))))
+    ;; Save new photo
+    (db/update! pool :team
+                {:photo-id (:id photo)}
+                {:id team-id})
+
+    (assoc team :photo-id (:id photo))))

backend/src/app/rpc/commands/teams_invitations.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.teams-invitations
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
@@ -15,7 +16,6 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.db.sql :as sql]
    [app.email :as eml]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
@@ -34,7 +34,6 @@
 
 ;; --- Mutation: Create Team Invitation
 
-
 (def sql:upsert-team-invitation
   "insert into team_invitation(id, team_id, email_to, created_by, role, valid_until)
    values (?, ?, ?, ?, ?, ?)
@@ -79,27 +78,23 @@
    [:role ::types.team/role]
    [:email ::sm/email]])
 
-(def ^:private check-create-invitation-params!
+(def ^:private check-create-invitation-params
   (sm/check-fn schema:create-invitation))
 
+(defn- allow-invitation-emails?
+  [member]
+  (let [notifications (dm/get-in member [:props :notifications])]
+    (not= :none (:email-invites notifications))))
+
 (defn- create-invitation
   [{:keys [::db/conn] :as cfg} {:keys [team profile role email] :as params}]
 
-  (dm/assert!
-   "expected valid connection on cfg parameter"
-   (db/connection? conn))
-
-  (dm/assert!
-   "expected valid params for `create-invitation` fn"
-   (check-create-invitation-params! params))
+  (assert (db/connection? conn) "expected valid connection on cfg parameter")
+  (assert (check-create-invitation-params params))
 
   (let [email  (profile/clean-email email)
         member (profile/get-profile-by-email conn email)]
 
-    (teams/check-profile-muted conn member)
-    (teams/check-email-bounce conn email true)
-    (teams/check-email-spam conn email true)
-
     ;; When we have email verification disabled and invitation user is
     ;; already present in the database, we proceed to add it to the
     ;; team as-is, without email roundtrip.
@@ -125,62 +120,65 @@
 
         nil)
 
-      (let [id         (uuid/next)
-            expire     (dt/in-future "168h") ;; 7 days
-            invitation (db/exec-one! conn [sql:upsert-team-invitation id
-                                           (:id team) (str/lower email)
-                                           (:id profile)
-                                           (name role) expire
-                                           (name role) expire])
-            updated?   (not= id (:id invitation))
-            profile-id (:id profile)
-            tprops     {:profile-id profile-id
-                        :invitation-id (:id invitation)
-                        :valid-until expire
-                        :team-id (:id team)
-                        :member-email (:email-to invitation)
-                        :member-id (:id member)
-                        :role role}
-            itoken     (create-invitation-token cfg tprops)
-            ptoken     (create-profile-identity-token cfg profile-id)]
+      (do
+        (some->> member (teams/check-profile-muted conn))
+        (teams/check-email-bounce conn email true)
+        (teams/check-email-spam conn email true)
 
-        (when (contains? cf/flags :log-invitation-tokens)
-          (l/info :hint "invitation token" :token itoken))
+        (let [id         (uuid/next)
+              expire     (dt/in-future "168h") ;; 7 days
+              invitation (db/exec-one! conn [sql:upsert-team-invitation id
+                                             (:id team) (str/lower email)
+                                             (:id profile)
+                                             (name role) expire
+                                             (name role) expire])
+              updated?   (not= id (:id invitation))
+              profile-id (:id profile)
+              tprops     {:profile-id profile-id
+                          :invitation-id (:id invitation)
+                          :valid-until expire
+                          :team-id (:id team)
+                          :member-email (:email-to invitation)
+                          :member-id (:id member)
+                          :role role}
+              itoken     (create-invitation-token cfg tprops)
+              ptoken     (create-profile-identity-token cfg profile-id)]
 
-        (let [props  (-> (dissoc tprops :profile-id)
-                         (audit/clean-props))
-              evname (if updated?
-                       "update-team-invitation"
-                       "create-team-invitation")
-              event (-> (audit/event-from-rpc-params params)
-                        (assoc ::audit/name evname)
-                        (assoc ::audit/props props))]
-          (audit/submit! cfg event))
+          (when (contains? cf/flags :log-invitation-tokens)
+            (l/info :hint "invitation token" :token itoken))
 
-        (eml/send! {::eml/conn conn
-                    ::eml/factory eml/invite-to-team
-                    :public-uri (cf/get :public-uri)
-                    :to email
-                    :invited-by (:fullname profile)
-                    :team (:name team)
-                    :token itoken
-                    :extra-data ptoken})
+          (let [props  (-> (dissoc tprops :profile-id)
+                           (audit/clean-props))
+                evname (if updated?
+                         "update-team-invitation"
+                         "create-team-invitation")
+                event (-> (audit/event-from-rpc-params params)
+                          (assoc ::audit/name evname)
+                          (assoc ::audit/props props))]
+            (audit/submit! cfg event))
 
-        itoken))))
+          (when (allow-invitation-emails? member)
+            (eml/send! {::eml/conn conn
+                        ::eml/factory eml/invite-to-team
+                        :public-uri (cf/get :public-uri)
+                        :to email
+                        :invited-by (:fullname profile)
+                        :team (:name team)
+                        :token itoken
+                        :extra-data ptoken}))
 
-(defn- add-user-to-team
-  [conn profile team role email]
+          itoken)))))
+
+(defn- add-member-to-team
+  [conn profile team role member]
 
   (let [team-id (:id team)
-        member  (db/get* conn :profile
-                         {:email (str/lower email)}
-                         {::sql/columns [:id :email]})
         params  (merge
                  {:team-id team-id
                   :profile-id (:id member)}
                  (get types.team/permissions-for-role role))]
 
-      ;; Do not allow blocked users to join teams.
+    ;; Do not allow blocked users to join teams.
     (when (:is-blocked member)
       (ex/raise :type :restriction
                 :code :profile-blocked))
@@ -205,29 +203,33 @@
     (eml/send! {::eml/conn conn
                 ::eml/factory eml/join-team
                 :public-uri (cf/get :public-uri)
-                :to email
+                :to (:email member)
                 :invited-by (:fullname profile)
                 :team (:name team)
                 :team-id (:id team)})))
 
-(def sql:valid-requests-email
-  "SELECT p.email
+(def ^:private sql:valid-access-request-profiles
+  "SELECT p.id, p.email, p.is_blocked
      FROM team_access_request AS tr
      JOIN profile AS p ON (tr.requester_id = p.id)
     WHERE tr.team_id = ?
-      AND tr.auto_join_until > now()")
+      AND tr.auto_join_until > now()
+      AND (p.deleted_at IS NULL OR
+           p.deleted_at > now())")
 
-(defn- get-valid-requests-email
+(defn- get-valid-access-request-profiles
   [conn team-id]
-  (db/exec! conn [sql:valid-requests-email team-id]))
+  (db/exec! conn [sql:valid-access-request-profiles team-id]))
 
-(def ^:private xf:map-email
-  (map :email))
+(def ^:private xf:map-email (map :email))
 
 (defn- create-team-invitations
   [{:keys [::db/conn] :as cfg} {:keys [profile team role emails] :as params}]
-  (let [join-requests    (into #{} xf:map-email
-                               (get-valid-requests-email conn (:id team)))
+  (let [emails           (set emails)
+
+        join-requests    (->> (get-valid-access-request-profiles conn (:id team))
+                              (d/index-by :email))
+
         team-members     (into #{} xf:map-email
                                (teams/get-team-members conn (:id team)))
 
@@ -245,8 +247,10 @@
 
     ;; For requested invitations, do not send invitation emails, add
     ;; the user directly to the team
-    (->> (filter join-requests emails)
-         (run! (partial add-user-to-team conn profile team role)))
+    (->> join-requests
+         (filter #(contains? emails (key %)))
+         (map val)
+         (run! (partial add-member-to-team conn profile team role)))
 
     invitations))
 
@@ -404,20 +408,20 @@
 (sv/defmethod ::update-team-invitation-role
   {::doc/added "1.17"
    ::doc/module :teams
-   ::sm/params schema:update-team-invitation-role}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id email role] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (teams/get-permissions conn profile-id team-id)]
+   ::sm/params schema:update-team-invitation-role
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email role] :as params}]
+  (let [perms (teams/get-permissions conn profile-id team-id)]
 
-      (when-not (:is-admin perms)
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+    (when-not (:is-admin perms)
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (db/update! conn :team-invitation
-                  {:role (name role) :updated-at (dt/now)}
-                  {:team-id team-id :email-to (profile/clean-email email)})
+    (db/update! conn :team-invitation
+                {:role (name role) :updated-at (dt/now)}
+                {:team-id team-id :email-to (profile/clean-email email)})
 
-      nil)))
+    nil))
 
 ;; --- Mutation: Delete invitation
 
@@ -428,20 +432,20 @@
 
 (sv/defmethod ::delete-team-invitation
   {::doc/added "1.17"
-   ::sm/params schema:delete-team-invition}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id email] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (teams/get-permissions conn profile-id team-id)]
+   ::sm/params schema:delete-team-invition
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email] :as params}]
+  (let [perms (teams/get-permissions conn profile-id team-id)]
 
-      (when-not (:is-admin perms)
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+    (when-not (:is-admin perms)
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (let [invitation (db/delete! conn :team-invitation
-                                   {:team-id team-id
-                                    :email-to (profile/clean-email email)}
-                                   {::db/return-keys true})]
-        (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}})))))
+    (let [invitation (db/delete! conn :team-invitation
+                                 {:team-id team-id
+                                  :email-to (profile/clean-email email)}
+                                 {::db/return-keys true})]
+      (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}}))))
 
 
 ;; --- Mutation: Request Team Invitation
@@ -572,5 +576,3 @@
 
       (with-meta {:request request}
         {::audit/props {:request 1}}))))
-
-

backend/src/app/rpc/commands/viewer.clj

@@ -6,18 +6,19 @@
 
 (ns app.rpc.commands.viewer
   (:require
+   [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
    [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.comments :as comments]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [cuerdas.core :as str]))
 
 ;; --- QUERY: View Only Bundle
 
@@ -27,6 +28,27 @@
       (update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
       (update :pages-index select-keys allowed)))
 
+(defn obfuscate-email
+  [email]
+  (let [[name domain]
+        (str/split email "@" 2)
+
+        [_ rest]
+        (str/split domain "." 2)
+
+        name
+        (if (> (count name) 3)
+          (str (subs name 0 1) (apply str (take (dec (count name)) (repeat "*"))))
+          "****")]
+
+    (str name "@****." rest)))
+
+(defn anonymize-member
+  [member]
+  (-> (select-keys member [:id :email :name :fullname :photo-id])
+      (update :email obfuscate-email)
+      (assoc :can-read true)))
+
 (defn- get-view-only-bundle
   [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
   (let [file    (files/get-file cfg file-id)
@@ -38,10 +60,13 @@
         team    (-> (db/get conn :team {:id (:team-id project)})
                     (teams/decode-row))
 
-        members (into #{} (->> (teams/get-team-members conn (:team-id project))
-                               (map :id)))
+        members    (cond->> (teams/get-team-members conn (:team-id project))
+                     (= :share-link (:type perms))
+                     (mapv anonymize-member))
 
-        perms   (assoc perms :in-team (contains? members profile-id))
+        member-ids (into #{} (map :id) members)
+
+        perms   (assoc perms :in-team (contains? member-ids profile-id))
 
         _       (-> (cfeat/get-team-enabled-features cf/flags team)
                     (cfeat/check-client-features! (:features params))
@@ -54,8 +79,10 @@
                   :always
                   (update :data select-keys [:id :options :pages :pages-index :components]))
 
-        libs    (files/get-file-libraries conn file-id)
-        users   (comments/get-file-comments-users conn file-id profile-id)
+        libs    (->> (bfc/get-file-libraries conn file-id)
+                     (mapv (fn [{:keys [id] :as lib}]
+                             (merge lib (files/get-file cfg id)))))
+
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]
                              (-> row
@@ -71,13 +98,14 @@
                           {:team-id (:id team)
                            :deleted-at nil})]
 
-    {:users users
+    {:users members
+     :profiles members
      :fonts fonts
      :project project
      :share-links links
      :libraries libs
      :file file
-     :team team
+     :team (assoc team :permissions perms)
      :permissions perms}))
 
 (def schema:get-view-only-bundle

backend/src/app/rpc/commands/webhooks.clj

@@ -144,20 +144,20 @@
 
 (sv/defmethod ::delete-webhook
   {::doc/added "1.17"
-   ::sm/params schema:delete-webhook}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
-  (db/with-atomic [conn pool]
-    (let [whook (-> (db/get conn :webhook {:id id}) decode-row)]
-      (check-webhook-edition-permissions! conn profile-id (:team-id whook) (:profile-id whook))
-      (db/delete! conn :webhook {:id id})
-      nil)))
+   ::sm/params schema:delete-webhook
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (let [whook (-> (db/get conn :webhook {:id id}) decode-row)]
+    (check-webhook-edition-permissions! conn profile-id (:team-id whook) (:profile-id whook))
+    (db/delete! conn :webhook {:id id})
+    nil))
 
 ;; --- Query: Webhooks
 
 (def sql:get-webhooks
-  "SELECT id, uri, mtype, is_active, error_code, error_count, profile_id 
-     FROM webhook 
-    WHERE team_id = ? 
+  "SELECT id, uri, mtype, is_active, error_code, error_count, profile_id
+     FROM webhook
+    WHERE team_id = ?
     ORDER BY uri")
 
 (def ^:private schema:get-webhooks

backend/src/app/rpc/doc.clj

@@ -9,6 +9,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.json :as json]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as smdj]
@@ -19,7 +20,6 @@
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
-   [app.util.json :as json]
    [app.util.services :as sv]
    [app.util.template :as tmpl]
    [clojure.java.io :as io]
@@ -86,7 +86,7 @@
     (fn [request]
       (let [params  (:query-params request)
             pstyle  (:type params "js")
-            context (assoc context :param-style pstyle)]
+            context (assoc @context :param-style pstyle)]
 
         {::yres/status 200
          ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
@@ -178,8 +178,7 @@
     (fn [_]
       {::yres/status 200
        ::yres/headers {"content-type" "application/json; charset=utf-8"}
-       ::yres/body (json/encode context)})
-
+       ::yres/body (json/encode @context)})
     (fn [_]
       {::yres/status 404})))
 
@@ -209,7 +208,7 @@
 
 (defmethod ig/init-key ::routes
   [_ {:keys [::rpc/methods] :as cfg}]
-  [(let [context (prepare-doc-context methods)]
+  [(let [context (delay (prepare-doc-context methods))]
      [["/_doc"
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]
@@ -217,7 +216,7 @@
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]])
 
-   (let [context (prepare-openapi-context methods)]
+   (let [context (delay (prepare-openapi-context methods))]
      [["/openapi"
        {:handler (openapi-handler)
         :allowed-methods #{:get}}]

backend/src/app/setup.clj

@@ -78,19 +78,19 @@
 
 (defmethod ig/init-key ::props
   [_ {:keys [::db/pool ::key] :as cfg}]
-  (db/with-atomic [conn pool]
-    (db/xact-lock! conn 0)
-    (when-not key
-      (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
-                         "all sessions on each restart, it is hightly recommended setting up the "
-                         "PENPOT_SECRET_KEY environment variable")))
 
-    (let [secret (or key (generate-random-key))]
-      (-> (get-all-props conn)
-          (assoc :secret-key secret)
-          (assoc :tokens-key (keys/derive secret :salt "tokens"))
-          (update :instance-id handle-instance-id conn (db/read-only? pool))))))
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (db/xact-lock! conn 0)
+                    (when-not key
+                      (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
+                                         "all sessions on each restart, it is highly recommended setting up the "
+                                         "PENPOT_SECRET_KEY environment variable")))
 
+                    (let [secret (or key (generate-random-key))]
+                      (-> (get-all-props conn)
+                          (assoc :secret-key secret)
+                          (assoc :tokens-key (keys/derive secret :salt "tokens"))
+                          (update :instance-id handle-instance-id conn (db/read-only? pool)))))))
 
 ;; FIXME
 (sm/register! ::props :any)

backend/src/app/setup/templates.clj

@@ -54,6 +54,7 @@
                               (::setup/templates cfg))]
     (let [dest (fs/join fs/*cwd* "builtin-templates")
           path (or (:path template) (fs/join dest template-id))]
+
       (if (fs/exists? path)
         (io/input-stream path)
         (let [resp (http/req! cfg