Merge pull request #6724 from penpot/elenatorro-improve-create-profile-command

🔧 Add option to skip tutorial/walkthrough when creating a profile from the script

.circleci/config.yml

@@ -1,5 +1,53 @@
 version: 2.1
+
 jobs:
+  lint:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    steps:
+      - checkout
+
+      - run:
+          name: "fmt check"
+          working_directory: "."
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+
+      - run:
+          name: "lint clj common"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:common
+
+      - run:
+          name: "lint clj frontend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:frontend
+
+      - run:
+          name: "lint clj backend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:backend
+
+      - run:
+          name: "lint clj exporter"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:exporter
+
+      - run:
+          name: "lint clj library"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:library
+
   test-common:
     docker:
       - image: penpotapp/devenv:latest
@@ -17,15 +65,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "common/deps.edn"}}
-
-      - run:
-          name: "fmt check & linter"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
+            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
 
       - run:
           name: "JVM tests"
@@ -37,12 +77,16 @@ jobs:
           name: "NODE tests"
           working_directory: "./common"
           command: |
+            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-          key: v1-dependencies-{{ checksum "common/deps.edn"}}
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
 
   test-frontend:
     docker:
@@ -61,36 +105,68 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
       - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
+          name: "install dependencies"
+          working_directory: "./frontend"
+          # We install playwright here because the dependent tasks
+          # uses the same cache as this task so we prepopulate it
           command: |
             yarn install
-            yarn run lint:clj
+            yarn run playwright install chromium
 
       - run:
-          name: "fmt check & linter"
+          name: "lint scss on frontend"
           working_directory: "./frontend"
           command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run fmt:js:check
             yarn run lint:scss
-            yarn run lint:clj
 
       - run:
           name: "unit tests"
           working_directory: "./frontend"
           command: |
-            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+  test-library:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: Install dependencies and build
+          working_directory: "./library"
+          command: |
+            yarn install
+
+      - run:
+          name: Build and Test
+          working_directory: "./library"
+          command: |
+            ./scripts/build
+            yarn run test
 
   test-components:
     docker:
@@ -109,14 +185,14 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
       - run:
           name: Install dependencies
           working_directory: "./frontend"
           command: |
-            yarn
-            npx playwright install --with-deps
+            yarn install
+            yarn run playwright install chromium
 
       - run:
           name: Build Storybook
@@ -148,7 +224,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
       - run:
           name: "integration tests"
@@ -158,7 +234,7 @@ jobs:
             yarn run build:app:assets
             yarn run build:app
             yarn run build:app:libs
-            yarn run playwright install --with-deps chromium
+            yarn run playwright install chromium
             yarn run test:e2e -x --workers=4
 
   test-backend:
@@ -185,21 +261,6 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "backend/deps.edn" }}
 
-      - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run lint:clj
-
-      - run:
-          name: "fmt check & linter"
-          working_directory: "./backend"
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
-
       - run:
           name: "tests"
           working_directory: "./backend"
@@ -215,37 +276,9 @@ jobs:
       - save_cache:
           paths:
             - ~/.m2
+            - ~/.gitlibs
           key: v1-dependencies-{{ checksum "backend/deps.edn" }}
 
-  test-exporter:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      - run:
-          name: "prepopulate linter cache"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run lint:clj
-
-      - run:
-          name: "fmt check & linter"
-          working_directory: "./exporter"
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-            yarn run lint:clj
-
   test-render-wasm:
     docker:
       - image: penpotapp/devenv:latest
@@ -263,6 +296,12 @@ jobs:
           command: |
             cargo fmt --check
 
+      - run:
+          name: "lint"
+          working_directory: "./render-wasm"
+          command: |
+            ./lint
+
       - run:
           name: "cargo tests"
           working_directory: "./render-wasm"
@@ -272,10 +311,32 @@ jobs:
 workflows:
   penpot:
     jobs:
-      - test-frontend
-      - test-components
-      - test-integration
-      - test-backend
-      - test-common
-      - test-exporter
+      - lint
+      - test-frontend:
+          requires:
+            - lint: success
+
+      - test-library:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-components:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-integration:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-backend:
+          requires:
+            - lint: success
+
+      - test-common:
+          requires:
+            - lint: success
+
       - test-render-wasm

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,29 +1,19 @@
-<!--
+### Related Ticket
 
-Some key notes before you open a PR:
+<!-- Reference the related GitHub/Taiga ticket. -->
 
- 1. Select which branch should this PR be merged in? By default, you should always merge to the develop branch.
- 2. PR name follows [convention](http://karma-runner.github.io/4.0/dev/git-commit-msg.html)
- 3. All tests pass locally, UI and Unit tests
- 4. All business logic and validations must be on the server-side
- 5. Update necessary Documentation
- 6. Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes
+### Summary
 
+### Steps to reproduce 
 
-Also, if you're new here
+### Checklist
 
-- Contribution Guide => https://github.com/uxbox/uxbox/blob/develop/CONTRIBUTING.md
+- [ ] Choose the correct target branch; use `develop` by default.
+- [ ] Provide a brief summary of the changes introduced.
+- [ ] Add a detailed explanation of how to reproduce the issue and/or verify the fix, if applicable.
+- [ ] Include screenshots or videos, if applicable.
+- [ ] Add or modify existing integration tests in case of bugs or new features, if applicable.
+- [ ] Check CI passes successfully.
+- [ ] Update the `CHANGES.md` file, referencing the related GitHub issue, if applicable.
 
--->
-
-> Please provide enough information so that others can review your pull request:
-
-<!-- You can skip this if you're fixing a typo or updating existing documentation -->
-
-> Explain the **details** for making this change. What existing problem does the pull request solve?
-
-<!-- Example: When "Adding a function to do X", explain why it is necessary to have a way to do X. -->
-
-> Screenshots/GIFs
-
-<!-- Add images/recordings to better visualize the change: expected/current behviour -->
+<!-- For more details, check the contribution guidelines: https://github.com/penpot/penpot/blob/develop/CONTRIBUTING.md -->

.github/workflows/build-bundles.yml

@@ -0,0 +1,129 @@
+name: Build and Upload Penpot Bundles non-prod
+
+on:
+  # Create bundler for every tag
+  push:
+    tags:
+      - '**' # Pattern matched against refs/tags
+  # Create bundler every hour between 5:00 and 20:00 on working days
+  schedule:
+    - cron: '0 5-20 * * 1-5'
+  # Create bundler from manual action
+  workflow_dispatch:
+    inputs:
+      zip_mode:
+        # zip_mode defines how the build artifacts are packaged:
+        # - 'individual': creates one ZIP file per component (frontend, backend, exporter)
+        # - 'all': creates a single ZIP containing all components
+        # - null: for the rest of cases (non-manual events)
+        description: 'Bundle packaging mode'
+        required: false
+        default: 'individual'
+        type: choice
+        options:
+          - individual
+          - all
+
+jobs:
+  build-bundles:
+    name: Build and Upload Penpot Bundles
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Extract somer useful variables
+        id: vars
+        run: |
+          echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "gh_branch=${{ github.base_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      # Set up Docker Buildx for multi-arch build
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run manage.sh build-bundle from host
+        run: ./manage.sh build-bundle
+
+      - name: Prepare directories for zipping
+        run: |
+          mkdir zips
+          mv bundles penpot
+
+      - name: Create zip bundles for zip_mode == 'all'
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        run: |
+          echo "📦 Packaging Penpot 'all' bundles..."
+          zip -r zips/penpot-all-bundles.zip penpot
+
+      - name: Create zip bundles for zip_mode != 'all'
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        run: |
+          echo "📦 Packaging Penpot 'individual' bundles..."
+          zip -r zips/penpot-frontend.zip penpot/frontend
+          zip -r zips/penpot-backend.zip penpot/backend
+          zip -r zips/penpot-exporter.zip penpot/exporter
+
+      - name: Upload unified 'all' bundle
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: penpot-all-bundles
+          path: zips/penpot-all-bundles.zip
+
+      - name: Upload individual bundles
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: penpot-individual-bundles
+          path: |
+            zips/penpot-frontend.zip
+            zips/penpot-backend.zip
+            zips/penpot-exporter.zip
+
+      - name: Upload unified 'all' bundle to S3
+        if: ${{ github.event.inputs.zip_mode == 'all' }}
+        run: |
+          aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.gh_branch}}.zip
+          aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.commit_hash }}.zip
+
+      - name: Upload 'individual' bundles to S3
+        if: ${{ github.event.inputs.zip_mode != 'all' }}
+        run: |
+            for name in penpot-frontend penpot-backend penpot-exporter; do
+              aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.gh_branch }}-latest.zip
+              aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.commit_hash }}.zip
+            done
+
+      - name: Notify Mattermost about automatic bundles
+        if: github.event_name == 'pull_request'
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          TEXT: |
+            📦 *Penpot bundle automatically generated*
+            📄 PR: ${{ github.event.pull_request.title }}
+            🔁 From: \`${{ github.head_ref }}\` to \`{{ github.base_ref }}\`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Notify Mattermost about manual bundles
+        if: github.event_name == 'workflow_dispatch'
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          TEXT: |
+            📦 *Penpot bundle manually generated*
+            📄 Triggered from branch: `${{ github.ref_name}}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Print artifact summary URL
+        run: |
+          echo "📦 Artifacts available at:"
+          echo "🔗 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):\s[A-Z].*[^.]$'
+          pattern: '^(Merge|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.gitignore

@@ -30,6 +30,7 @@
 /*.zip
 /.clj-kondo/.cache
 /_dump
+/notes
 /backend/*.md
 /backend/*.sql
 /backend/*.txt
@@ -68,6 +69,8 @@
 /vendor/**/target
 /vendor/svgclean/bundle*.js
 /web
+/library/target/
+
 clj-profiler/
 node_modules
 /test-results/

CHANGES.md

@@ -1,6 +1,6 @@
 # CHANGELOG
 
-## 2.5.1
+## 2.9.0 (Unreleased)
 
 ### :rocket: Epics and highlights
 
@@ -8,16 +8,282 @@
 
 ### :heart: Community contributions (Thank you!)
 
+### :sparkles: New features & Enhancements
+- Add visual indicator for new comments in the workspace [Taiga #11328](https://tree.taiga.io/project/penpot/issue/11328)
+- On components overrides, separate the content of the text from the rest of properties [Taiga #7434](https://tree.taiga.io/project/penpot/us/7434)
+- Improve dashboard's sidebar [Taiga #10700](https://tree.taiga.io/project/penpot/us/10700)
+- Change "Save color" button to primary button [Taiga #9410](https://tree.taiga.io/project/penpot/issue/9410)
+- Support for exif rotated images [GitHub #6767](https://github.com/penpot/penpot/issues/6767)
+- Display Blend Mode and Layer Opacity properties in the Inspect tab [Taiga #11283](https://tree.taiga.io/project/penpot/issue/11283)
+- Add the option to import tokens in a .zip file. [Taiga #11378](https://tree.taiga.io/project/penpot/us/11378)
+- New typography token type - font size token [Taiga #10938](https://tree.taiga.io/project/penpot/us/10938)
+
+### :bug: Bugs fixed
+- Copying font size does not copy the unit [Taiga #11143](https://tree.taiga.io/project/penpot/issue/11143)
+- Fix text-decoration line-through that displays a wrong property value [Taiga #11145](https://tree.taiga.io/project/penpot/issue/11145)
+- Fix display error message on register form [Taiga #11444](https://tree.taiga.io/project/penpot/issue/11444)
+- Fix toggle focus mode did not restore viewport and selection upon exit [GitHub #6280](https://github.com/penpot/penpot/issues/6820)
+- Fix problem when creating a layout from an existing layout [Taiga #11554](https://tree.taiga.io/project/penpot/issue/11554)
+- Fix title button from Title Case to Capitalize [Taiga #11476](https://tree.taiga.io/project/penpot/issue/11476)
+
+## 2.8.0
+
+### :rocket: Epics and highlights
+
+### :boom: Breaking changes & Deprecations
+
+**Penpot Library**
+
+The initial prototype is completly reworked for provide a more consistent API
+and to have proper validation and params decoding. All the details can be found
+on [its own changelog](library/CHANGES.md)
+
+**Penpot migrate from Redis to Valkey**
+
+As [Valkey](https://valkey.io/) is an opne-souce fork of [Redis](https://redis.io/)
+version 7.2.4, this version of Penpot will be compatible with Redis but may diverge
+in future versions. Therefore, **migration from Redis to ValKey is recommended for all
+on-premises instances** that want to keep up to date.
+
+### :heart: Community contributions (Thank you!)
+- Add Serbian language [GitHub #5002](https://github.com/penpot/penpot/issues/5002) by [crnobog69](https://github.com/crnobog69)
+
+### :sparkles: New features & Enhancements
+
+- Optimize profile setup flow for better user experience [Taiga #10028](https://tree.taiga.io/project/penpot/us/10028)
+- Rewrite path shape data PathData encoding [Taiga #8542](https://tree.taiga.io/project/penpot/us/8542?milestone=441308)
+- Update base image for Docker Backend and Exporter to Ubuntu 24.04
+- Update base image for Docker Frontend to Nginx 1.28.0
+- Allow multi file token import [Github #27](https://github.com/tokens-studio/penpot/issues/27)
+- Create `input*` wrapper component, and `label*`, `input-field*` and `hint-message*` components [Taiga #10713](https://tree.taiga.io/project/penpot/us/10713)
+- Deselect layers (and path nodes) with Ctrl+Shift+Drag [Github #2509](https://github.com/penpot/penpot/issues/2509)
+- Copy to SVG from contextual menu [Github #838](https://github.com/penpot/penpot/issues/838)
+- Add styles for Inkeep Chat at workspace [Taiga #10708](https://tree.taiga.io/project/penpot/us/10708)
+- Add configuration for air gapped installations with Docker
+- Support system color scheme [Github #5030](https://github.com/penpot/penpot/issues/5030)
+- Persist ruler visibility across files and reloads [GitHub #4586](https://github.com/penpot/penpot/issues/4586)
+- Update google fonts (at 2025/05/19) [Taiga 10792](https://tree.taiga.io/project/penpot/us/10792)
+- Add tooltip component to DS [Taiga 9220](https://tree.taiga.io/project/penpot/us/9220)
+- Allow multi file token export [Taiga #10144](https://tree.taiga.io/project/penpot/us/10144)
+- Fix problem when double click on hidden shapes [Taiga #11314](https://tree.taiga.io/project/penpot/issue/11314)
+
+### :bug: Bugs fixed
+
+- Fix getCurrentUser for plugins api [Taiga #11057](https://tree.taiga.io/project/penpot/issue/11057)
+- Fix spacing / sizes of different elements in the measurements section of the design tab [Taiga #11076](https://tree.taiga.io/project/penpot/issue/11076)
+- Fix selection of short paths [Github #4472](https://github.com/penpot/penpot/issues/4472)
+- Fix element positioning on the right side to adjust to grid [#11073](https://tree.taiga.io/project/penpot/issue/11073)
+- Fix palette is over sidebar [#11160](https://tree.taiga.io/project/penpot/issue/11160)
+- Fix font size input not displaying "mixed" when multiple texts are selected [Taiga #11177](https://tree.taiga.io/project/penpot/issue/11177)
+- Misalignments at Create account [Taiga #11315](https://tree.taiga.io/project/penpot/issue/11315)
+- Fix issue with importing files where flex/grid is used [Taiga #11334](https://tree.taiga.io/project/penpot/issue/11334)
+- Fix wrong color in the export progress bar [Taiga #11299](https://tree.taiga.io/project/penpot/issue/11299)
+- Fix right sidebar width overflow on long layer names [Taiga #11212](https://tree.taiga.io/project/penpot/issue/11212)
+- Fix comment icon fill [Taiga #11388](https://tree.taiga.io/project/penpot/issue/11388)
+- Fix gap on radio-buttons component [Taiga #11360](https://tree.taiga.io/project/penpot/issue/11360)
+- Fix button width [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
+- Fix mixed letter spacing and line height [Taiga #11178](https://tree.taiga.io/project/penpot/issue/11178)
+- Fix snap nodes shortcut [Taiga #11054](https://tree.taiga.io/project/penpot/issue/11054)
+- Fix changing a text property in a text layer does not unapply the previously applied token in the same property [Taiga #11337](https://tree.taiga.io/project/penpot/issue/11337)
+- Fix shortcut error pressing G+W from the View Mode [Taiga #11061](https://tree.taiga.io/project/penpot/issue/11061)
+- Fix entering long project name [Taiga #11417](https://tree.taiga.io/project/penpot/issue/11417)
+- Fix slow color picker [Taiga #11019](https://tree.taiga.io/project/penpot/issue/11019)
+- Fix tooltip position after click [Taiga #11405](https://tree.taiga.io/project/penpot/issue/11405)
+- Fix incorrect media translation on paste text with fill images [Github #6845](https://github.com/penpot/penpot/pull/6845)
+
+## 2.7.2
+
+### :bug: Bugs fixed
+
+- Update plugins runtime [Github #6604](https://github.com/penpot/penpot/pull/6604)
+- Backport from develop a minor fix that enables import of files
+  generated by penpot library [Github #6614](https://github.com/penpot/penpot/pull/6614)
+- Fix copy in error message [GitHub #6615](https://github.com/penpot/penpot/pull/6615)
+- Fix url on invitation link [Taiga #11284](https://tree.taiga.io/project/penpot/issue/11284)
+
+
+## 2.7.1
+
+### :bug: Bugs fixed
+
+- Fix incorrect handling of strokes with images on importing files
+- Fix tokens disappearing after manual additions [Taiga #11063](https://tree.taiga.io/project/penpot/issue/11063)
+
+
+## 2.7.0
+
+### :rocket: Epics and highlights
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+- Design improvements to the Invitations page with an empty state [GitHub #2608](https://github.com/penpot/penpot/issues/2608) by [@iprithvitharun](https://github.com/iprithvitharun)
+
+### :sparkles: New features
+
+- Update board presets with a newer devices [Taiga #10610](https://tree.taiga.io/project/penpot/us/10610)
+- Propagate "sharing a prototype" to editors and viewers [Taiga #8853](https://tree.taiga.io/project/penpot/us/8853)
+- Design improvements to the Invitations page with an empty state [Taiga #4554](https://tree.taiga.io/project/penpot/us/4554)
+- Duplicate token sets [Taiga #10694](https://tree.taiga.io/project/penpot/issue/10694)
+- Add set selection in create Token themes flow [Taiga #10746](https://tree.taiga.io/project/penpot/issue/10746)
+- Display indicator on not active sets [Taiga #10668](https://tree.taiga.io/project/penpot/issue/10668)
+
+### :bug: Bugs fixed
+
+- Fix "at" icon to match all icons on app [Taiga #11136](https://tree.taiga.io/project/penpot/issue/11136)
+- Fix problem in viewer with the back button [Taiga #10907](https://tree.taiga.io/project/penpot/issue/10907)
+- Fix resize bar background on tokens panel [Taiga #10811](https://tree.taiga.io/project/penpot/issue/10811)
+- Fix shortcut for history version panel [Taiga #11006](https://tree.taiga.io/project/penpot/issue/11006)
+- Fix positioning of comment drafts when near the right / bottom edges of viewport [Taiga #10534](https://tree.taiga.io/project/penpot/issue/10534)
+- Fix path having a wrong selrect [Taiga #10257](https://tree.taiga.io/project/penpot/issue/10257)
+- Fix SVG `stroke-linecap` property when importing SVGs [Taiga #9489](https://tree.taiga.io/project/penpot/issue/9489)
+- Fix position problems cutting-pasting a component [Taiga #10677](https://tree.taiga.io/project/penpot/issue/10677)
+- Fix design tab has a horizontal scroll [Taiga #10660](https://tree.taiga.io/project/penpot/issue/10660)
+- Fix long file names being clipped when longer than allowed length [Taiga #10662](https://tree.taiga.io/project/penpot/issue/10662)
+- Fix problem with error detail in toast [Taiga #10519](https://tree.taiga.io/project/penpot/issue/10519)
+- Fix view mode error when an external user tries to export something from a prototype using a shared link [Taiga #10251](https://tree.taiga.io/project/penpot/issue/10251)
+- Fix merge path nodes with only one node selected [Taiga #9626](https://tree.taiga.io/project/penpot/issue/9626)
+- Fix problem with import errors [Taiga #10040](https://tree.taiga.io/project/penpot/issue/10040)
+- Fix color gradient on texts [Taiga Issue #7488](https://tree.taiga.io/project/penpot/issue/7488)
+- Add support for self mentions [Taiga #10809](https://tree.taiga.io/project/penpot/issue/10809)
+- Fix team info settings alignment [Taiga #10869](https://tree.taiga.io/project/penpot/issue/10869)
+- Fix left sidebar horizontal scroll on nested layers [Taiga #10791](https://tree.taiga.io/project/penpot/issue/10791)
+- Improve error message details importing tokens [Taiga Issue #10772](https://tree.taiga.io/project/penpot/issue/10772)
+- Fix no selected set after Drag & Drop [Github #71](https://github.com/tokens-studio/penpot/issues/71)
+- Styledictionary v5 Update [Github #6283](https://github.com/penpot/penpot/pull/6283)
+- Fix Rename a set throws an internal error [Github #78](https://github.com/tokens-studio/penpot/issues/78)
+- Fix Out of Sync Token Value & Color Picker [Github #102](https://github.com/tokens-studio/penpot/issues/102)
+- Fix Color should preserve color space [Github #69](https://github.com/tokens-studio/penpot/issues/69)
+- Fix cannot rename Design Token Sets when group of same name exists [Taiga Issue #10773](https://tree.taiga.io/project/penpot/issue/10773)
+- Fix problem when duplicating grid layout [Github #6391](https://github.com/penpot/penpot/issues/6391)
+- Fix issue that makes workspace shortcuts stop working [Taiga #11062](https://tree.taiga.io/project/penpot/issue/11062)
+- Fix problem while syncing library colors and typographies [Taiga #11068](https://tree.taiga.io/project/penpot/issue/11068)
+- Fix problem with path edition of shapes [Taiga #9496](https://tree.taiga.io/project/penpot/issue/9496)
+- Fix exception on paste invalid html [Taiga #11047](https://tree.taiga.io/project/penpot/issue/11047)
+- Fix share button being displayed with no permissions [Taiga #11086](https://tree.taiga.io/project/penpot/issue/11086)
+- Fix inline styles in code tab [Taiga Issue #7583](https://tree.taiga.io/project/penpot/issue/7583)
+- Fix exception on returning openapi.json
+- Fix json encoding of TokensLib [Taiga #10994](https://tree.taiga.io/project/penpot/issue/10994)
+
+## 2.6.2
+
+### :bug: Bugs fixed
+
+- Increase the height of the right sidebar dropdowns [Taiga #10615](https://tree.taiga.io/project/penpot/issue/10615)
+- Fix scroll on token themes modal [Taiga #10745](https://tree.taiga.io/project/penpot/issue/10745)
+- Fix collapsing grouped sets in "edit Theme" closes the dialog [Taiga #10771](https://tree.taiga.io/project/penpot/issue/10771)
+- Fix unexpected exception on path editor on merge segments when undo stack is empty
+- Fix pricing CTA to be under a config flag [Taiga #10808](https://tree.taiga.io/project/penpot/issue/10808)
+- Fix allow moving a main component into another [Taiga #10818](https://tree.taiga.io/project/penpot/issue/10818)
+- Fix several issues with internal srepl helpers
+- Fix unexpected exception on template import from libraries
+- Fix incorrect uuid parsing from different parts of code
+- Fix update layout on component restore [Taiga #10637](https://tree.taiga.io/project/penpot/issue/10637)
+- Fix horizontal scroll in viewer [Github #6290](https://github.com/penpot/penpot/issues/6290)
+- Fix detach component in a particular case [Taiga #10837](https://tree.taiga.io/project/penpot/issue/10837)
+
+## 2.6.1
+
+### :bug: Bugs fixed
+
+- Fix webhooks not shown in list [Taiga #10763](https://tree.taiga.io/project/penpot/issue/10763)
+- Fix colorpicker scroll when dropdown displayed [Taiga #10696](https://tree.taiga.io/project/penpot/issue/10696)
+- Clean internal workspace state on exit or url changed [Taiga #10619](https://tree.taiga.io/project/penpot/issue/10619)
+
+## 2.6.0
+
+### :rocket: Epics and highlights
+
+- Design Tokens
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+### :sparkles: New features
+
+- [COMMENTS] "Mark All as Read" Functionality in Dashboard [Taiga #9235](https://tree.taiga.io/project/penpot/us/9235)
+- [COMMENTS] Bubble Groups [Taiga #9236](https://tree.taiga.io/project/penpot/us/9236)
+- Change templates carrousel [Taiga #9803](https://tree.taiga.io/project/penpot/us/9803)
+- [DESIGN TOKENS] Tokens CRUD. Types added: Color, Opacity, Border radius, Dimension, Sizing, Spacing, Rotation and Stroke.
+- [DESIGN TOKENS] Create references (alias) that point to other tokens.
+- [DESIGN TOKENS] Math operations in token values.
+- [DESIGN TOKENS] Sets CRUD, grouping and reordering.
+- [DESIGN TOKENS] Multidimensional Themes and Sets management.
+- [DESIGN TOKENS] Apply/Remove tokens to/from elements from the Tokens tab.
+- [DESIGN TOKENS] Integration with components.
+- [DESIGN TOKENS] Import and export tokens from a JSON file.
+- [DESIGN TOKENS] Apply Themes and Sets at document level.
+- Add more descriptive tooltip to boards for first time users [Taiga #9426](https://tree.taiga.io/project/penpot/us/9426)
+- First State of a Project Changes Consolidation [Taia #10605](https://tree.taiga.io/project/penpot/us/10605)
+
+### :bug: Bugs fixed
+
+- Fix opacity in frame containers [Github #5858](https://github.com/penpot/penpot/pull/5858)
+- Avoid resizing on click [Taiga #10213](https://tree.taiga.io/project/penpot/issue/10213)
+- Hide horizontal scroll from dashboard sidebar [Taiga #10422](https://tree.taiga.io/project/penpot/issue/10422)
+- Fix cut and paste a copy a cmponent inside its parent [Taiga #10365](https://tree.taiga.io/project/penpot/us/10365)
+- Fix duplicate page with component over frame [Taiga #8151](https://tree.taiga.io/project/penpot/issue/8151) and [Taiga #9698](https://tree.taiga.io/project/penpot/issue/9698)
+- The plugin list in the navigation menu lacks scrolling, some plugins are not visible when a large number are installed [Taiga #9360](https://tree.taiga.io/project/penpot/us/9360)
+- Fix hidden toolbar click event still available [Taiga #10437](https://tree.taiga.io/project/penpot/us/10437)
+- Fix hovering over templates [Taiga #10545](https://tree.taiga.io/project/penpot/issue/10545)
+- Fix problem with default shadows value in plugins [Plugins #191](https://github.com/penpot/penpot-plugins/issues/191)
+- Fix problem with constraints when creating group [Taiga #10455](https://tree.taiga.io/project/penpot/issue/10455)
+- Fix opening pen with shortcut multiple times breaks toolbar [Taiga #10566](https://tree.taiga.io/project/penpot/issue/10566)
+- Fix actions when workspace is visited first time [Taiga #10548](https://tree.taiga.io/project/penpot/issue/10548)
+- Chat icon overlaps "Show" button in carrousel section [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Fix assets name on inspect tab [Taiga #10630](https://tree.taiga.io/project/penpot/issue/10630)
+- Fix chat icon overlaps "Show" button in carrousel section [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Fix incorrect handling of background task result (now task rows are properly marked as completed)
+- Fix available size of resize handler [Taiga #10639](https://tree.taiga.io/project/penpot/issue/10639)
+- Internal error when install a plugin by penpothub - Try plugin [Taiga #10542](https://tree.taiga.io/project/penpot/issue/10542)
+- Add character limitation to asset inputs [Taiga #10669](https://tree.taiga.io/project/penpot/issue/10669)
+- Fix Storybook link 'list of all available icons' wrong path [Taiga #10705](https://tree.taiga.io/project/penpot/issue/10705)
+
+
+## 2.5.4
+
+### :heart: Community contributions (Thank you!)
+
+- Add support for WEBP format on shape export [Github #6053](https://github.com/penpot/penpot/pull/6053) and [Github #6074](https://github.com/penpot/penpot/pull/6074)
+
+### :bug: Bugs fixed
+
+- Fix feature loading on workspace when opening a file in a background
+  tab [Taiga #10377](https://tree.taiga.io/project/penpot/issue/10377)
+- Fix minor inconsistencies on RPC `get-file-libraries` and `get-file`
+  methods (add missing team-id prop)
+- Fix problem with viewer role and inspect mode [Taiga #9751](https://tree.taiga.io/project/penpot/issue/9751)
+- Fix error when clicking on a comment at the viewer's sidebar [Taiga #10465](https://tree.taiga.io/project/penpot/issue/10465)
+
+## 2.5.3
+
+### :bug: Bugs fixed
+
+- Component sync issues with multiple tabs [Taiga #10471](https://tree.taiga.io/project/penpot/issue/10471)
+
+## 2.5.2
+
+### :sparkles: New features
+
+- When the workspace is empty, set default the board creation tool [Taiga #9425](https://tree.taiga.io/project/penpot/us/9425)
+
+### :bug: Bugs fixed
+
+- Fix scroll on storybook docs [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+- Navigate tracking event firing multiple times [Taiga #10415](https://tree.taiga.io/project/penpot/issue/10415)
+- Fix problem with selection colors [Taiga #10376](https://tree.taiga.io/project/penpot/issue/10376)
+- Fix scroll on storybook icons list [taiga #9962](https://tree.taiga.io/project/penpot/issue/9962)
+
+## 2.5.1
+
 ### :sparkles: New features
 
 - Improve Nginx entryponit to get the resolvers dinamically by default
 
-### :bug: Bugs fixed
-
 ## 2.5.0
 
-### :rocket: Epics and highlights
-
 ### :boom: Breaking changes & Deprecations
 
 Although this is not a breaking change, we believe it’s important to highlight it in this
@@ -46,9 +312,6 @@ If you have a big database and many cores available, you can reduce the time of
 all files by increasing paralelizacion changing the `max-jobs` value from 1 to N (where N
 is a number of cores)
 
-
-### :heart: Community contributions (Thank you!)
-
 ### :sparkles: New features
 
 - [GRADIENTS] New gradients UI with multi-stop support. [Taiga #3418](https://tree.taiga.io/project/penpot/epic/3418)
@@ -65,6 +328,7 @@ is a number of cores)
 
 ### :bug: Bugs fixed
 
+- Fix menu shadow color [Taiga #10102](https://tree.taiga.io/project/penpot/issue/10102)
 - Fix missing state refresh on notifications update [Taiga #10253](https://tree.taiga.io/project/penpot/issue/10253)
 - Fix icon visualization on select component [Taiga #8889](https://tree.taiga.io/project/penpot/issue/8889)
 - Fix typo on integration tests docs [Taiga #10112](https://tree.taiga.io/project/penpot/issue/10112)

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Penpot's Code of Conduct
+
+Check it at: https://help.penpot.app/contributing-guide/coc/

CONTRIBUTING.md

@@ -1,62 +1,59 @@
 # Contributing Guide #
 
 Thank you for your interest in contributing to Penpot. This is a
-generic guide that details how to contribute to Penpot in a way that
-is efficient for everyone. If you want a specific documentation for
-different parts of the platform, please refer to `docs/` directory.
-
+generic guide that details how to contribute to the project in a way that
+is efficient for everyone. If you are looking for specific documentation on
+different parts of the platform, please refer to the `docs/` directory,
+or the rendered version at the [Help Center](https://help.penpot.app/).
 
 ## Reporting Bugs ##
 
 We are using [GitHub Issues](https://github.com/penpot/penpot/issues)
-for our public bugs. We keep a close eye on this and try to make it
+for our public bugs. We keep a close eye on them and try to make it
 clear when we have an internal fix in progress. Before filing a new
 task, try to make sure your problem doesn't already exist.
 
-If you found a bug, please report it, as far as possible with:
+If you found a bug, please report it, as far as possible, with:
 
 - a detailed explanation of steps to reproduce the error
-- a browser and the browser version used
-- a dev tools console exception stack trace (if it is available)
+- the browser and browser version used
+- a dev tools console exception stack trace (if available)
 
-If you found a bug that you consider better discuss in private (for
-example: security bugs), consider first send an email to
+If you found a bug which you think is better to discuss in private (for
+example, security bugs), consider first sending an email to
 `support@penpot.app`.
 
-**We don't have formal bug bounty program for security reports; this
-is an open source application and your contribution will be recognized
+**We don't have a formal bug bounty program for security reports; this
+is an open source application, and your contribution will be recognized
 in the changelog.**
 
 
-## Pull requests ##
+## Pull Requests ##
 
-If you want propose a change or bug fix with the Pull-Request system
-firstly you should carefully read the **DCO** section and format your
-commits accordingly.
+If you want to propose a change or bug fix via a pull request (PR),
+you should first carefully read the section **Developer's Certificate of
+Origin**. You must also format your code and commits according to the
+instructions below.
 
-If you intend to fix a bug it's fine to submit a pull request right
-away but we still recommend to file an issue detailing what you're
+If you intend to fix a bug, it's fine to submit a pull request right
+away, but we still recommend filing an issue detailing what you're
 fixing. This is helpful in case we don't accept that specific fix but
 want to keep track of the issue.
 
-If you want to implement or start working in a new feature, please
-open a **question** / **discussion** issue for it. No pull-request
-will be accepted without previous chat about the changes,
-independently if it is a new feature, already planned feature or small
-quick win.
+If you want to implement or start working on a new feature, please
+open a **question*- / **discussion*- issue for it. No PR
+will be accepted without a prior discussion about the changes,
+whether it is a new feature, an already planned one, or a quick win.
 
-If is going to be your first pull request, You can learn how from this
-free video series:
-
-https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github
-
-We will use the `easy fix` mark for tag for indicate issues that are
-easy for beginners.
+If it is your first PR, you can learn how to proceed from
+[this free video
+series](https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github)
 
+We use the `easy fix` tag to indicate issues that are appropriate for beginners.
 
 ## Commit Guidelines ##
 
-We have very precise rules over how our git commit messages can be formatted.
+We have very precise rules on how our git commit messages must be formatted.
 
 The commit message format is:
 
@@ -71,34 +68,37 @@ The commit message format is:
 Where type is:
 
 - :bug: `:bug:` a commit that fixes a bug
-- :sparkles: `:sparkles:` a commit that an improvement
-- :tada: `:tada:` a commit with new feature
+- :sparkles: `:sparkles:` a commit that adds an improvement
+- :tada: `:tada:` a commit with a new feature
 - :recycle: `:recycle:` a commit that introduces a refactor
 - :lipstick: `:lipstick:` a commit with cosmetic changes
-- :ambulance: `:ambulance:` a commit that fixes critical bug
+- :ambulance: `:ambulance:` a commit that fixes a critical bug
 - :books: `:books:` a commit that improves or adds documentation
-- :construction: `:construction:`: a wip commit
+- :construction: `:construction:` a WIP commit
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
-- :whale: `:whale:` a commit for docker related stuff
-- :paperclip: `:paperclip:` a commit with other not relevant changes
-- :arrow_up: `:arrow_up:` a commit with dependencies updates
-- :arrow_down: `:arrow_down:` a commit with dependencies downgrades
+- :whale: `:whale:` a commit for Docker-related stuff
+- :paperclip: `:paperclip:` a commit with other non-relevant changes
+- :arrow_up: `:arrow_up:` a commit with dependency updates
+- :arrow_down: `:arrow_down:` a commit with dependency downgrades
 - :fire: `:fire:` a commit that removes files or code
+- :globe_with_meridians: `:globe_with_meridians:` a commit that adds or updates
+  translations
 
 More info:
+
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
 Each commit should have:
 
-- A concise subject using imperative mood.
-- The subject should have capitalized the first letter, without period
-  at the end and no larger than 65 characters.
+- A concise subject using the imperative mood.
+- The subject should capitalize the first letter, omit the period
+  at the end, and be no longer than 65 characters.
 - A blank line between the subject line and the body.
-- An entry on the CHANGES.md file if applicable, referencing the
-  github or taiga issue/user-story using the these same rules.
+- An entry in the CHANGES.md file if applicable, referencing the
+  GitHub or Taiga issue/user story using these same rules.
 
 Examples of good commit messages:
 
@@ -111,8 +111,30 @@ Examples of good commit messages:
 - `:ambulance: Fix critical bug on user registration process`
 - `:tada: Add new approach for user registration`
 
+## Formatting and Linting ##
 
-## Code of conduct ##
+You will want to make sure your code is formatted and linted before submitting
+a PR. We use [cljfmt](https://github.com/weavejester/cljfmt) and
+[clj-kondo](https://github.com/clj-kondo/clj-kondo) for this. After installing
+them on your system, you can run them with:
+
+```bash
+# Check formatting
+yarn fmt:clj:check
+
+# Check and fix formatting
+yarn fmt:clj
+
+# Run the linter
+yarn lint:clj
+```
+
+There are more choices in `package.json`.
+
+Ideally, you should run these commands as git pre-commit hooks. A convenient way
+of defining them is to use [Husky](https://typicode.github.io/husky/#/).
+
+## Code of Conduct ##
 
 As contributors and maintainers of this project, we pledge to respect
 all people who contribute through reporting issues, posting feature
@@ -132,11 +154,11 @@ unprofessional conduct.
 
 Project maintainers have the right and responsibility to remove, edit,
 or reject comments, commits, code, wiki edits, issues, and other
-contributions that are not aligned to this Code of Conduct. Project
+contributions that are not aligned with this Code of Conduct. Project
 maintainers who do not follow the Code of Conduct may be removed from
 the project team.
 
-This code of conduct applies both within project spaces and in public
+This Code of Conduct applies both within project spaces and in public
 spaces when an individual is representing the project or its
 community.
 
@@ -145,12 +167,11 @@ may be reported by opening an issue or contacting one or more of the
 project maintainers.
 
 This Code of Conduct is adapted from the Contributor Covenant, version
-1.1.0, available from http://contributor-covenant.org/version/1/1/0/
+1.1.0, available from [http://contributor-covenant.org/version/1/1/0/](http://contributor-covenant.org/version/1/1/0/)
 
+## Developer's Certificate of Origin (DCO)
 
-## Developer's Certificate of Origin (DCO) ##
-
-By submitting code you are agree and can certify the below:
+By submitting code you agree to and can certify the following:
 
     Developer's Certificate of Origin 1.1
 
@@ -178,13 +199,15 @@ By submitting code you are agree and can certify the below:
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
 
-Then, all your code patches (**documentation are excluded**) should
+Then, all your code patches (**documentation is excluded**) should
 contain a sign-off at the end of the patch/commit description body. It
-can be automatically added on adding `-s` parameter to `git commit`.
+can be automatically added by adding the `-s` parameter to `git commit`.
 
-This is an example of the aspect of the line:
+This is an example of what the line should look like:
 
-	Signed-off-by: Andrey Antukh <niwi@niwi.nz>
+```
+Signed-off-by: Andrey Antukh <niwi@niwi.nz>
+```
 
 Please, use your real name (sorry, no pseudonyms or anonymous
 contributions are allowed).

README.md

@@ -16,36 +16,38 @@
 </p>
 
 <p align="center">
-    <a href="https://penpot.app/"><b>Website</b></a>  •  
-    <a href="https://help.penpot.app/technical-guide/getting-started/"><b>Getting Started</b></a>  •  
-    <a href="https://help.penpot.app/user-guide/"><b>User Guide</b></a>  •  
-    <a href="https://help.penpot.app/user-guide/introduction/info/"><b>Tutorials & Info</b></a>  •  
+    <a href="https://penpot.app/"><b>Website</b></a>  •
+    <a href="https://help.penpot.app/user-guide/"><b>User Guide</b></a>  •
+    <a href="https://penpot.app/learning-center"><b>Learning Center</b></a>  •
     <a href="https://community.penpot.app/"><b>Community</b></a>
 </p>
 <p align="center">
-    <a href="https://www.youtube.com/@Penpot"><b>Youtube</b></a>  •  
-    <a href="https://peertube.kaleidos.net/a/penpot_app/video-channels"><b>Peertube</b></a>  •  
-    <a href="https://www.linkedin.com/company/penpot/"><b>Linkedin</b></a>  •  
-    <a href="https://instagram.com/penpot.app"><b>Instagram</b></a>  •  
-    <a href="https://fosstodon.org/@penpot/"><b>Mastodon</b></a>  •  
+    <a href="https://www.youtube.com/@Penpot"><b>Youtube</b></a>  •
+    <a href="https://peertube.kaleidos.net/a/penpot_app/video-channels"><b>Peertube</b></a>  •
+    <a href="https://www.linkedin.com/company/penpot/"><b>Linkedin</b></a>  •
+    <a href="https://instagram.com/penpot.app"><b>Instagram</b></a>  •
+    <a href="https://fosstodon.org/@penpot/"><b>Mastodon</b></a>  •
+    <a href="https://bsky.app/profile/penpot.app"><b>Bluesky</b></a>  •
     <a href="https://twitter.com/penpotapp"><b>X</b></a>
 
 </p>
 
 <br />
 
-[Penpot video](https://github.com/penpot/penpot/assets/5446186/b8ad0764-585e-4ddc-b098-9b4090d337cc)
+[Penpot video](https://github.com/user-attachments/assets/7c67fd7c-04d3-4c9b-88ec-b6f5e23f8332
+)
 
 <br />
 
 Penpot is the first **open-source** design tool for design and code collaboration. Designers can create stunning designs, interactive prototypes, design systems at scale, while developers enjoy ready-to-use code and make their workflow easy and fast. And all of this with no handoff drama.
 
-Penpot is available on browser and [self host](https://penpot.app/self-host). It’s web-based and works with open standards (SVG, CSS and HTML). And last but not least, it’s free! 
+Available on browser or self-hosted, Penpot works with open standards like SVG, CSS, HTML and JSON, and it’s free!
 
-Penpot’s latest [huge release 2.0](https://penpot.app/dev-diaries), takes the platform to a whole new level. This update introduces the ground-breaking [CSS Grid Layout feature](https://penpot.app/penpot-2.0), a complete UI redesign, a new Components system, and much more. Plus, it's faster and more accessible. 
+The latest updates take Penpot even further. It’s the first design tool to integrate native [design tokens](https://penpot.dev/collaboration/design-tokens)—a single source of truth to improve efficiency and collaboration between product design and development.
+With the [huge 2.0 release](https://penpot.app/dev-diaries), Penpot took the platform to a whole new level. This update introduces the ground-breaking [CSS Grid Layout feature](https://penpot.app/penpot-2.0), a complete UI redesign, a new Components system, and much more.
+For organizations that need extra service for its teams, [get in touch](https://cal.com/team/penpot/talk-to-us)
 
-
-🎇 **Penpot Fest** is our design, code & Open Source event. Check out the highlights from [Penpot Fest 2023 edition](https://www.youtube.com/watch?v=sOpLZaK5mDc)!
+🎇 Design, code, and Open Source meet at [Penpot Fest](https://penpot.app/penpotfest)! Be part of the 2025 edition in Madrid, Spain, on October 9-10.
 
 ## Table of contents ##
 
@@ -61,7 +63,7 @@ Penpot’s latest [huge release 2.0](https://penpot.app/dev-diaries), takes the
 Penpot expresses designs as code. Designers can do their best work and see it will be beautifully implemented by developers in a two-way collaboration.
 
 ### Plugin system ###
-[Penpot plugins](https://penpot.app/penpothub/plugins) let you expand the platform's capabilities, give you the flexibility to integrate it with other apps, and design custom solutions. 
+[Penpot plugins](https://penpot.app/penpothub/plugins) let you expand the platform's capabilities, give you the flexibility to integrate it with other apps, and design custom solutions.
 
 ### Designed for developers ###
 Penpot was built to serve both designers and developers and create a fluid design-code process. You have the choice to enjoy real-time collaboration or play "solo".
@@ -78,6 +80,10 @@ Penpot offers integration into the development toolchain, thanks to its support
 ### What’s great for design ###
 With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
 
+### Design Tokens ###
+With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
+
+
 <br />
 
 <p align="center">
@@ -88,10 +94,9 @@ With Penpot you can design libraries to share and reuse; turn design elements in
 
 ## Getting started ##
 
-### Install with Elestio ###
-Penpot is the only design & prototype platform that is deployment agnostic. You can use it or deploy it anywhere.
+Penpot is the only design & prototype platform that is deployment agnostic. You can use it in our [SAAS](https://design.penpot.app) or deploy it anywhere.
 
-Learn how to install it with Elestio and Docker, or other options on [our website](https://penpot.app/self-host).
+Learn how to install it with Docker, Kubernetes, Elestio or other options on [our website](https://penpot.app/self-host).
 <br />
 
 <p align="center">
@@ -123,15 +128,21 @@ You will find the following categories:
 </p>
 <br />
 
+### Code of Conduct ###
+
+Anyone who contributes to Penpot, whether through code, in the community, or at an event, must adhere to the
+[code of conduct](https://help.penpot.app/contributing-guide/coc/) and foster a positive and safe environment.
+
+
 ## Contributing ##
 
-Any contribution will make a difference to improve Penpot. How can you get involved? 
+Any contribution will make a difference to improve Penpot. How can you get involved?
 
-Choose your way: 
+Choose your way:
 
 - Create and [share Libraries & Templates](https://penpot.app/libraries-templates.html) that will be helpful for the community
 - Invite your [team to join](https://design.penpot.app/#/auth/register)
-- Star this repo and follow us on Social Media: [Mastodon](https://fosstodon.org/@penpot/), [Youtube](https://www.youtube.com/c/Penpot), [Instagram](https://instagram.com/penpot.app), [Linkedin](https://www.linkedin.com/company/penpotdesign),  [Peertube](https://peertube.kaleidos.net/a/penpot_app) and [X](https://twitter.com/penpotapp).
+- Give this repo a star and follow us on Social Media: [Mastodon](https://fosstodon.org/@penpot/), [Youtube](https://www.youtube.com/c/Penpot), [Instagram](https://instagram.com/penpot.app), [Linkedin](https://www.linkedin.com/company/penpotdesign),  [Peertube](https://peertube.kaleidos.net/a/penpot_app), [X](https://twitter.com/penpotapp) and [BlueSky](https://bsky.app/profile/penpot.app)
 - Participate in the [Community](https://community.penpot.app/) space by asking and answering questions; reacting to others’ articles;  opening your own conversations and following along on decisions affecting the project.
 - Report bugs with our easy [guide for bugs hunting](https://help.penpot.app/contributing-guide/reporting-bugs/) or [GitHub issues](https://github.com/penpot/penpot/issues)
 - Become a [translator](https://help.penpot.app/contributing-guide/translations)

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.0"}
+  org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.6-9"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,8 +17,15 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.5.2.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.7.0.RELEASE"}
+  ;; Minimal dependencies required by lettuce, we need to include them
+  ;; explicitly because clojure dependency management does not support
+  ;; yet the BOM format.
+  io.micrometer/micrometer-core {:mvn/version "1.14.2"}
+  io.micrometer/micrometer-observation {:mvn/version "1.14.2"}
+
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
+  com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
   {:git/tag "v11.4"
@@ -27,15 +34,14 @@
    :exclusions [org.slf4j/slf4j-api]}
 
   com.github.seancorfield/next.jdbc
-  {:mvn/version "1.3.994"}
-  metosin/reitit-core {:mvn/version "0.7.2"}
+  {:mvn/version "1.3.1002"}
+  metosin/reitit-core {:mvn/version "0.9.1"}
   nrepl/nrepl {:mvn/version "1.3.1"}
-  cider/cider-nrepl {:mvn/version "0.52.0"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.5"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.48.0.0"}
+  org.postgresql/postgresql {:mvn/version "42.7.6"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "6.2.1"}
+  com.zaxxer/HikariCP {:mvn/version "6.3.0"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
@@ -44,7 +50,7 @@
 
   com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.18.3"}
+  org.jsoup/jsoup {:mvn/version "1.20.1"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -55,11 +61,11 @@
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
   dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.2"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.3"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.28.26"}}
+  software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -74,7 +80,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.6" :git/sha "52cf7d6"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test

backend/package.json

@@ -4,7 +4,7 @@
   "license": "MPL-2.0",
   "author": "Kaleidos INC",
   "private": true,
-  "packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728",
+  "packageManager": "yarn@4.9.2+sha512.1fc009bc09d13cfd0e19efa44cbfc2b9cf6ca61482725eb35bbc5e257e093ebf4130db6dfe15d604ff4b79efd8e1e8e99b25fa7d0a6197c9f9826358d4d65c3c",
   "repository": {
     "type": "git",
     "url": "https://github.com/penpot/penpot"

backend/resources/app/email/request-file-access/en.txt

@@ -13,7 +13,7 @@ This will automatically include {{requested-by|abbreviate:25}} in the team, so t
 
 Click the link below to provide team access:
 
-{{ public-uri }}/#/dashboard/members?team-id{{team-id}}&invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 

backend/resources/app/onboarding.edn

@@ -1,4 +1,7 @@
-[{:id "wireframing-kit"
+[{:id "tokens-starter-kit"
+  :name "Design tokens starter kit"
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"},
+ {:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}
  {:id "prototype-examples"

backend/resources/app/templates/debug.tmpl

@@ -151,6 +151,78 @@ Debug Main Page
         </div>
 
 
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
+
+  <section class="widget">
+    <h2>Feature Flags</h2>
+    <fieldset>
+      <legend>Enable</legend>
+      <desc>Add a feature flag to a team</desc>
+      <form method="post" action="/dbg/actions/add-team-feature">
+        <div class="row">
+          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
+        </div>
+        <div class="row">
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
+        </div>
+
+        <div class="row">
+          <label for="check-feature">Skip feature check</label>
+          <input id="check-feature" type="checkbox" name="skip-check" />
+          <br />
+          <small>
+            Do not check if the feature is supported
+          </small>
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+    <fieldset>
+      <legend>Disable</legend>
+      <desc>Remove a feature flag from a team</desc>
+      <form method="post" action="/dbg/actions/remove-team-feature">
+        <div class="row">
+          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
+        </div>
+        <div class="row">
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
+        </div>
+
+        <div class="row">
+          <label for="check-feature">Skip feature check</label>
+          <input id="check-feature" type="checkbox" name="skip-check" />
+          <br />
+          <small>
+            Do not check if the feature is supported
+          </small>
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
         <div class="row">
           <input type="submit" value="Submit" />
         </div>

backend/scripts/manage.py

@@ -35,40 +35,35 @@ def get_prepl_conninfo():
 
     return host, port
 
-def send_eval(expr):
+def send(data):
     host, port = get_prepl_conninfo()
+    with socket.create_connection((host, port)) as s:
+        f = s.makefile(mode="rw")
 
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.connect((host, port))
-        s.send(expr.encode("utf-8"))
-        s.send(b":repl/quit\n\n")
+        json.dump(data, f)
+        f.write("\n")
+        f.flush()
 
-        with s.makefile() as f:
-            while True:
-                line = f.readline()
-                result = json.loads(line)
-                tag = result.get("tag", None)
-                if tag == "ret":
-                    return result.get("val", None), result.get("exception", None)
-                elif tag == "out":
-                    print(result.get("val"), end="")
-                else:
-                    raise RuntimeError("unexpected response from PREPL")
+        while True:
+            line = f.readline()
+            result = json.loads(line)
+            tag = result.get("tag", None)
 
-def encode(val):
-    return json.dumps(json.dumps(val))
+            if tag == "ret":
+                return result.get("val", None), result.get("err", None)
+            elif tag == "out":
+                print(result.get("val"), end="")
+            else:
+                raise RuntimeError("unexpected response from PREPL")
 
-def print_error(res):
-    for error in res["via"]:
-        print("ERR:", error["message"])
-        break
+def print_error(error):
+    print("ERR:", error["hint"])
 
 def run_cmd(params):
     try:
-        expr = "(app.srepl.cli/exec {})".format(encode(params))
-        res, failed = send_eval(expr)
-        if failed:
-            print_error(res)
+        res, err = send(params)
+        if err:
+            print_error(err)
             sys.exit(-1)
 
         return res
@@ -76,19 +71,27 @@ def run_cmd(params):
         print("EXC:", str(cause))
         sys.exit(-2)
 
-def create_profile(fullname, email, password):
+def create_profile(fullname, email, password, skip_tutorial=False, skip_walkthrough=False):
+    props = {}
+    if skip_tutorial:
+        props["viewed-tutorial?"] = True
+    if skip_walkthrough:
+        props["viewed-walkthrough?"] = True
+
     params = {
         "cmd": "create-profile",
         "params": {
             "fullname": fullname,
             "email": email,
-            "password": password
+            "password": password,
+            **props
         }
     }
 
     res = run_cmd(params)
     print(f"Created: {res['email']} / {res['id']}")
 
+
 def update_profile(email, fullname, password, is_active):
     params = {
         "cmd": "update-profile",
@@ -96,7 +99,7 @@ def update_profile(email, fullname, password, is_active):
             "email": email,
             "fullname": fullname,
             "password": password,
-            "is_active": is_active
+            "isActive": is_active
         }
     }
 
@@ -138,7 +141,7 @@ def derive_password(password):
     params = {
         "cmd": "derive-password",
         "params": {
-            "password": password,
+            "password": password
         }
     }
 
@@ -175,6 +178,8 @@ parser.add_argument("-n", "--fullname", help="fullname", action="store")
 parser.add_argument("-e", "--email", help="email", action="store")
 parser.add_argument("-p", "--password", help="password", action="store")
 parser.add_argument("-c", "--connect", help="connect to PREPL", action="store", default="tcp://localhost:6063")
+parser.add_argument("--skip-tutorial", help="mark tutorial as viewed", action="store_true")
+parser.add_argument("--skip-walkthrough", help="mark walkthrough as viewed", action="store_true")
 
 args = parser.parse_args()
 

backend/scripts/repl

@@ -30,7 +30,9 @@ export PENPOT_FLAGS="\
        enable-access-tokens \
        enable-tiered-file-data-storage \
        enable-file-validation \
-       enable-file-schema-validation";
+       enable-file-schema-validation \
+       enable-subscriptions \
+       enable-subscriptions-old";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"
@@ -70,15 +72,19 @@ export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
 export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
 
-export JAVA_OPTS="--enable-preview \
+export JAVA_OPTS="\
        -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
        -Djdk.attach.allowAttachSelf \
        -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
        -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
        -XX:+EnableDynamicAgentLoading \
-       -XX:-OmitStackTraceInFastThrow \
+       -XX:-OmitStackTraceInFastThrow  \
        -XX:+UnlockDiagnosticVMOptions \
-       -XX:+DebugNonSafepoints";
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
 export OPTIONS="-A:jmx-remote -A:dev"
 
@@ -101,9 +107,6 @@ export OPTIONS="-A:jmx-remote -A:dev"
 # Setup GC
 # export OPTIONS="$OPTIONS -J-XX:+UseZGC"
 
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 

backend/scripts/run.template.sh

@@ -18,9 +18,9 @@ if [ -f ./environ ]; then
     source ./environ
 fi
 
-export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --enable-preview $JVM_OPTS"
+export JAVA_OPTS="-Dim4java.useV7=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
 
 ENTRYPOINT=${1:-app.main};
 
 set -ex
-exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m $ENTRYPOINT
+exec $JAVA_CMD $JAVA_OPTS -jar penpot.jar -m $ENTRYPOINT

backend/scripts/start-dev

@@ -23,18 +23,9 @@ export PENPOT_FLAGS="\
        enable-access-tokens \
        enable-tiered-file-data-storage \
        enable-file-validation \
-       enable-file-schema-validation";
-
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints"
+       enable-file-schema-validation \
+       enable-subscriptions \
+       enable-subscriptions-old ";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"
@@ -45,9 +36,6 @@ export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
 # Setup default multipart upload size to 300MiB
 export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
 # Initialize MINIO config
 mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
 mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
@@ -65,6 +53,18 @@ export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 
 entrypoint=${1:-app.main};
 
-set -ex
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:-OmitStackTraceInFastThrow  \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
-clojure $OPTIONS -A:dev -M -m $entrypoint;
+export OPTIONS="-A:jmx-remote -A:dev"
+
+set -ex
+clojure $OPTIONS -M -m $entrypoint;

backend/src/app/binfile/cleaner.clj

@@ -0,0 +1,131 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.binfile.cleaner
+  "A collection of helpers for perform cleaning of artifacts; mainly
+  for recently imported shapes."
+  (:require
+   [app.common.data :as d]
+   [app.common.types.shape :as cts]
+   [app.common.uuid :as uuid]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PRE DECODE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- pre-clean-bool-content
+  [shape]
+  (if-let [content (get shape :bool-content)]
+    (-> shape
+        (assoc :content content)
+        (dissoc :bool-content))
+    shape))
+
+(defn- pre-clean-shadow-color
+  [shape]
+  (d/update-when shape :shadow
+                 (fn [shadows]
+                   (mapv (fn [shadow]
+                           (update shadow :color
+                                   (fn [color]
+                                     (let [ref-id   (get color :id)
+                                           ref-file (get color :file-id)]
+                                       (-> (d/without-qualified color)
+                                           (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
+                                           (cond-> ref-id
+                                             (assoc :ref-id ref-id))
+                                           (cond-> ref-file
+                                             (assoc :ref-file ref-file)))))))
+                         shadows))))
+
+(defn clean-shape-pre-decode
+  "Applies a pre-decode phase migration to the shape"
+  [shape]
+  (cond-> shape
+    (= "bool" (:type shape))
+    (pre-clean-bool-content)
+
+    (contains? shape :shadow)
+    (pre-clean-shadow-color)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; POST DECODE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- fix-shape-shadow-color
+  "Some shapes can come with invalid `id` property on shadow colors
+  caused by incorrect uuid parsing bug that should be already fixed;
+  this function removes the invalid id from the data structure."
+  [shape]
+  (let [fix-color
+        (fn [{:keys [id] :as color}]
+          (if (uuid? id)
+            color
+            (if (and (string? id)
+                     (re-matches uuid/regex id))
+              (assoc color :id (uuid/uuid id))
+              (dissoc color :id))))
+
+        fix-shadow
+        (fn [shadow]
+          (d/update-when shadow :color fix-color))
+
+        xform
+        (map fix-shadow)]
+
+    (d/update-when shape :shadow
+                   (fn [shadows]
+                     (into [] xform shadows)))))
+
+(defn- fix-root-shape
+  "Ensure all root objects are well formed shapes"
+  [shape]
+  (if (= (:id shape) uuid/zero)
+    (-> shape
+        (assoc :parent-id uuid/zero)
+        (assoc :frame-id uuid/zero)
+        ;; We explicitly dissoc them and let the shape-setup
+        ;; to regenerate it with valid values.
+        (dissoc :selrect)
+        (dissoc :points)
+        (cts/setup-shape))
+    shape))
+
+(defn- fix-legacy-flex-dir
+  "This operation is only relevant to old data and it is fixed just
+  for convenience."
+  [shape]
+  (d/update-when shape :layout-flex-dir
+                 (fn [dir]
+                   (case dir
+                     :reverse-row :row-reverse
+                     :reverse-column :column-reverse
+                     dir))))
+
+(defn clean-shape-post-decode
+  "A shape procesor that expected to be executed after schema decoding
+  process but before validation."
+  [shape]
+  (-> shape
+      (fix-shape-shadow-color)
+      (fix-root-shape)
+      (fix-legacy-flex-dir)))
+
+(defn- fix-container
+  [container]
+  (-> container
+      ;; Remove possible `nil` keys on objects
+      (d/update-when :objects dissoc nil)
+      (d/update-when :objects d/update-vals clean-shape-post-decode)))
+
+(defn clean-file
+  [file & {:as _opts}]
+  (update file :data
+          (fn [data]
+            (-> data
+                (d/update-when :pages-index d/update-vals fix-container)
+                (d/update-when :components d/update-vals fix-container)
+                (d/without-nils)))))

backend/src/app/binfile/common.clj

@@ -53,6 +53,7 @@
   (* 1024 1024 100))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(declare get-resolved-file-libraries)
 
 (def file-attrs
   #{:id
@@ -143,11 +144,13 @@
    (reduce #(index-object %1 %2 attr) index coll)))
 
 (defn decode-row
-  "A generic decode row helper"
-  [{:keys [data features] :as row}]
-  (cond-> row
-    features (assoc :features (db/decode-pgarray features #{}))
-    data     (assoc :data (blob/decode data))))
+  [{:keys [data changes features] :as row}]
+  (when row
+    (cond-> row
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
+
 
 (defn decode-file
   "A general purpose file decoding function that resolves all external
@@ -156,7 +159,8 @@
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
     (let [file (->> file
                     (feat.fmigr/resolve-applied-migrations cfg)
-                    (feat.fdata/resolve-file-data cfg))]
+                    (feat.fdata/resolve-file-data cfg))
+          libs (delay (get-resolved-file-libraries cfg file))]
 
       (-> file
           (update :features db/decode-pgarray #{})
@@ -164,7 +168,7 @@
           (update :data feat.fdata/process-pointers deref)
           (update :data feat.fdata/process-objects (partial into {}))
           (update :data assoc :id id)
-          (fmg/migrate-file)))))
+          (fmg/migrate-file libs)))))
 
 (defn get-file
   "Get file, resolve all features and apply migrations.
@@ -418,28 +422,35 @@
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
 (defn process-file
-  [{:keys [id] :as file}]
-  (-> file
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (assoc :id id)
-                          (dissoc :recent-colors))))
-      (fmg/migrate-file)
-      (update :data (fn [fdata]
-                      (-> fdata
-                          (update :pages-index relink-shapes)
-                          (update :components relink-shapes)
-                          (update :media relink-media)
-                          (update :colors relink-colors)
-                          (d/without-nils))))))
+  [cfg {:keys [id] :as file}]
+  (let [libs (delay (get-resolved-file-libraries cfg file))]
+    (-> file
+        (update :data (fn [fdata]
+                        (-> fdata
+                            (assoc :id id)
+                            (dissoc :recent-colors))))
+        (update :data (fn [fdata]
+                        (-> fdata
+                            (update :pages-index relink-shapes)
+                            (update :components relink-shapes)
+                            (update :media relink-media)
+                            (update :colors relink-colors)
+                            (d/without-nils))))
+        (fmg/migrate-file libs)
+
+        ;; NOTE: this is necessary because when we just creating a new
+        ;; file from imported artifact or cloned file there are no
+        ;; migrations registered on the database, so we need to persist
+        ;; all of them, not only the applied
+        (vary-meta dissoc ::fmg/migrated))))
 
 (defn encode-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [file (if (contains? (:features file) "fdata/objects-map")
+  [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
+  (let [file (if (contains? features "fdata/objects-map")
                (feat.fdata/enable-objects-map file)
                file)
 
-        file (if (contains? (:features file) "fdata/pointer-map")
+        file (if (contains? features "fdata/pointer-map")
                (binding [pmap/*tracked* (pmap/create-tracked)]
                  (let [file (feat.fdata/enable-pointer-map file)]
                    (feat.fdata/persist-pointers! cfg id)
@@ -522,3 +533,49 @@
           (l/error :hint "file schema validation error" :cause result))))
 
     (insert-file! cfg file opts)))
+
+
+(def ^:private sql:get-file-libraries
+  "WITH RECURSIVE libs AS (
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+      WHERE flr.file_id = ?::uuid
+    UNION
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+       JOIN libs AS l ON (flr.file_id = l.id)
+   )
+   SELECT l.id,
+          l.features,
+          l.project_id,
+          p.team_id,
+          l.created_at,
+          l.modified_at,
+          l.deleted_at,
+          l.name,
+          l.revn,
+          l.vern,
+          l.synced_at,
+          l.is_shared
+     FROM libs AS l
+    INNER JOIN project AS p ON (p.id = l.project_id)
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+
+(defn get-file-libraries
+  [conn file-id]
+  (into []
+        (comp
+         ;; FIXME: :is-indirect set to false to all rows looks
+         ;; completly useless
+         (map #(assoc % :is-indirect false))
+         (map decode-row))
+        (db/exec! conn [sql:get-file-libraries file-id])))
+
+(defn get-resolved-file-libraries
+  "A helper for preload file libraries"
+  [{:keys [::db/conn] :as cfg} file]
+  (->> (get-file-libraries conn (:id file))
+       (into [file] (map #(get-file cfg (:id %))))
+       (d/index-by :id)))

backend/src/app/binfile/migrations.clj

@@ -10,7 +10,6 @@
    [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
-   [app.features.components-v2 :as feat.compv2]
    [clojure.set :as set]
    [cuerdas.core :as str]))
 
@@ -28,17 +27,20 @@
 
 (defn apply-pending-migrations!
   "Apply alredy registered pending migrations to files"
-  [cfg]
-  (doseq [[feature file-id] (-> bfc/*state* deref :pending-to-migrate)]
+  [_cfg]
+  (doseq [[feature _file-id] (-> bfc/*state* deref :pending-to-migrate)]
     (case feature
       "components/v2"
-      (feat.compv2/migrate-file! cfg file-id
-                                 :validate? (::validate cfg true)
-                                 :skip-on-graphic-error? true)
+      nil
 
       "fdata/shape-data-type"
       nil
 
+      ;; There is no migration needed, but we don't want to allow
+      ;; copy paste nor import of variant files into no-variant teams
+      "variants/v1"
+      nil
+
       (ex/raise :type :internal
                 :code :no-migration-defined
                 :hint (str/ffmt "no migation for feature '%' on file importation" feature)

backend/src/app/binfile/v1.clj

@@ -551,8 +551,8 @@
                      (cond-> (and (= idx 0) (some? name))
                        (assoc :name name))
                      (assoc :project-id project-id)
-                     (dissoc :thumbnails)
-                     (bfc/process-file))]
+                     (dissoc :thumbnails))
+            file  (bfc/process-file system file)]
 
         ;; All features that are enabled and requires explicit migration are
         ;; added to the state for a posterior migration step.

backend/src/app/binfile/v2.clj

@@ -281,8 +281,8 @@
 
   (let [file (-> (read-obj cfg :file file-id)
                  (update :id bfc/lookup-index)
-                 (update :project-id bfc/lookup-index)
-                 (bfc/process-file))]
+                 (update :project-id bfc/lookup-index))
+        file (bfc/process-file cfg file)]
 
     (events/tap :progress
                 {:op :import

backend/src/app/binfile/v3.clj

@@ -8,14 +8,16 @@
   "A ZIP based binary file exportation"
   (:refer-clojure :exclude [read])
   (:require
+   [app.binfile.cleaner :as bfl]
    [app.binfile.common :as bfc]
    [app.binfile.migrations :as bfm]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
+   [app.common.files.migrations :as-alias fmg]
    [app.common.json :as json]
    [app.common.logging :as l]
+   [app.common.media :as cmedia]
    [app.common.schema :as sm]
    [app.common.thumbnails :as cth]
    [app.common.types.color :as ctcl]
@@ -51,7 +53,7 @@
   [:map {:title "Manifest"}
    [:version ::sm/int]
    [:type :string]
-
+   [:referer {:optional true} :string]
    [:generated-by {:optional true} :string]
 
    [:files
@@ -71,7 +73,7 @@
    [:size ::sm/int]
    [:content-type :string]
    [:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
-   [:hash :string]])
+   [:hash {:optional true} :string]])
 
 (def ^:private schema:file-thumbnail
   [:map {:title "FileThumbnail"}
@@ -86,13 +88,19 @@
    ctf/schema:file
    [:map [:options {:optional true} ctf/schema:options]]])
 
+;; --- HELPERS
+
+(defn- default-now
+  [o]
+  (or o (dt/now)))
+
 ;; --- ENCODERS
 
 (def encode-file
   (sm/encoder schema:file sm/json-transformer))
 
 (def encode-page
-  (sm/encoder ::ctp/page sm/json-transformer))
+  (sm/encoder ctp/schema:page sm/json-transformer))
 
 (def encode-shape
   (sm/encoder ::cts/shape sm/json-transformer))
@@ -104,7 +112,7 @@
   (sm/encoder ::ctc/component sm/json-transformer))
 
 (def encode-color
-  (sm/encoder ::ctcl/color sm/json-transformer))
+  (sm/encoder ctcl/schema:library-color sm/json-transformer))
 
 (def encode-typography
   (sm/encoder ::cty/typography sm/json-transformer))
@@ -127,13 +135,13 @@
   (sm/decoder schema:manifest sm/json-transformer))
 
 (def decode-media
-  (sm/decoder ::ctf/media sm/json-transformer))
+  (sm/decoder ctf/schema:media sm/json-transformer))
 
 (def decode-component
   (sm/decoder ::ctc/component sm/json-transformer))
 
 (def decode-color
-  (sm/decoder ::ctcl/color sm/json-transformer))
+  (sm/decoder ctcl/schema:library-color sm/json-transformer))
 
 (def decode-file
   (sm/decoder schema:file sm/json-transformer))
@@ -148,7 +156,7 @@
   (sm/decoder ::cty/typography sm/json-transformer))
 
 (def decode-tokens-lib
-  (sm/decoder ::cto/tokens-lib sm/json-transformer))
+  (sm/decoder cto/schema:tokens-lib sm/json-transformer))
 
 (def decode-plugin-data
   (sm/decoder ::ctpg/plugin-data sm/json-transformer))
@@ -177,7 +185,7 @@
   (sm/check-fn ::ctf/media))
 
 (def validate-color
-  (sm/check-fn ::ctcl/color))
+  (sm/check-fn ctcl/schema:library-color))
 
 (def validate-component
   (sm/check-fn ::ctc/component))
@@ -227,27 +235,13 @@
                         :always
                         (bfc/clean-file-features))))))
 
-(defn- resolve-extension
-  [mtype]
-  (case mtype
-    "image/png"     ".png"
-    "image/jpeg"    ".jpg"
-    "image/gif"     ".gif"
-    "image/svg+xml" ".svg"
-    "image/webp"    ".webp"
-    "font/woff"     ".woff"
-    "font/woff2"    ".woff2"
-    "font/ttf"      ".ttf"
-    "font/otf"      ".otf"
-    "application/octet-stream" ".bin"))
-
 (defn- export-storage-objects
   [{:keys [::output] :as cfg}]
   (let [storage (sto/resolve cfg)]
     (doseq [id (-> bfc/*state* deref :storage-objects not-empty)]
       (let [sobject (sto/get-object storage id)
             smeta   (meta sobject)
-            ext     (resolve-extension (:content-type smeta))
+            ext     (cmedia/mtype->extension (:content-type smeta))
             path    (str "objects/" id ".json")
             params  (-> (meta sobject)
                         (assoc :id (:id sobject))
@@ -378,6 +372,7 @@
           params {:type "penpot/export-files"
                   :version 1
                   :generated-by (str "penpot/" (:full cf/version))
+                  :refer "penpot"
                   :files (vec (vals files))
                   :relations rels}]
       (write-entry! output "manifest.json" params))))
@@ -572,7 +567,13 @@
                  (let [object (->> (read-entry input entry)
                                    (decode-media)
                                    (validate-media))
-                       object (assoc object :file-id file-id)]
+                       object (-> object
+                                  (assoc :file-id file-id)
+                                  (update :created-at default-now)
+                                  ;; FIXME: this is set default to true for
+                                  ;; setting a value, this prop is no longer
+                                  ;; relevant;
+                                  (assoc :is-local true))]
                    (if (= id (:id object))
                      (conj result object)
                      result)))
@@ -594,16 +595,34 @@
 
 (defn- read-file-components
   [{:keys [::bfc/input ::file-id ::entries]}]
-  (->> (keep (match-component-entry-fn file-id) entries)
-       (reduce (fn [result {:keys [id entry]}]
-                 (let [object (->> (read-entry input entry)
-                                   (decode-component)
-                                   (validate-component))]
-                   (if (= id (:id object))
-                     (assoc result id object)
-                     result)))
-               {})
-       (not-empty)))
+  (let [clean-component-post-decode
+        (fn [component]
+          (d/update-when component :objects
+                         (fn [objects]
+                           (reduce-kv (fn [objects id shape]
+                                        (assoc objects id (bfl/clean-shape-post-decode shape)))
+                                      objects
+                                      objects))))
+        clean-component-pre-decode
+        (fn [component]
+          (d/update-when component :objects
+                         (fn [objects]
+                           (reduce-kv (fn [objects id shape]
+                                        (assoc objects id (bfl/clean-shape-pre-decode shape)))
+                                      objects
+                                      objects))))]
+
+    (->> (keep (match-component-entry-fn file-id) entries)
+         (reduce (fn [result {:keys [id entry]}]
+                   (let [object (->> (read-entry input entry)
+                                     (clean-component-pre-decode)
+                                     (decode-component)
+                                     (clean-component-post-decode))]
+                     (if (= id (:id object))
+                       (assoc result id object)
+                       result)))
+                 {})
+         (not-empty))))
 
 (defn- read-file-typographies
   [{:keys [::bfc/input ::file-id ::entries]}]
@@ -630,8 +649,9 @@
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
+                                   (bfl/clean-shape-pre-decode)
                                    (decode-shape)
-                                   (validate-shape))]
+                                   (bfl/clean-shape-post-decode))]
                    (if (= id (:id object))
                      (assoc result id object)
                      result)))
@@ -677,7 +697,6 @@
         components   (read-file-components cfg)
         plugin-data  (read-file-plugin-data cfg)
         pages        (read-file-pages cfg)]
-
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
@@ -732,9 +751,10 @@
                    (assoc :data data)
                    (assoc :name file-name)
                    (assoc :project-id project-id)
-                   (dissoc :options)
-                   (bfc/process-file))]
+                   (dissoc :options))
 
+          file  (bfc/process-file cfg file)
+          file  (ctf/check-file file)]
 
       (bfm/register-pending-migrations! cfg file)
       (bfc/save-file! cfg file ::db/return-keys false)
@@ -778,7 +798,7 @@
                     :expected-id (str id)
                     :found-id (str (:id object))))
 
-        (let [ext     (resolve-extension (:content-type object))
+        (let [ext     (cmedia/mtype->extension (:content-type object))
               path    (str "objects/" id ext)
               content (->> path
                            (get-zip-entry input)
@@ -792,13 +812,14 @@
                       :expected-size (:size object)
                       :found-size (sto/get-size content)))
 
-          (when (not= (:hash object) (sto/get-hash content))
-            (ex/raise :type :validation
-                      :code :inconsistent-penpot-file
-                      :hint "found corrupted storage object: hash does not match"
-                      :path path
-                      :expected-hash (:hash object)
-                      :found-hash (sto/get-hash content)))
+          (when-let [hash (get object :hash)]
+            (when (not= hash (sto/get-hash content))
+              (ex/raise :type :validation
+                        :code :inconsistent-penpot-file
+                        :hint "found corrupted storage object: hash does not match"
+                        :path path
+                        :expected-hash (:hash object)
+                        :found-hash (sto/get-hash content))))
 
           (let [params  (-> object
                             (dissoc :id :size)
@@ -857,13 +878,8 @@
 (defn- import-files
   [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
 
-  (dm/assert!
-   "expected zip file"
-   (instance? ZipFile input))
-
-  (dm/assert!
-   "expected valid instant"
-   (dt/instant? timestamp))
+  (assert (instance? ZipFile input) "expected zip file")
+  (assert (dt/instant? timestamp) "expected valid instant")
 
   (let [manifest (-> (read-manifest input)
                      (validate-manifest))
@@ -875,6 +891,7 @@
                 :hint "unexpected type on manifest"
                 :manifest manifest))
 
+
     ;; Check if all files referenced on manifest are present
     (doseq [{file-id :id features :features} (:files manifest)]
       (let [path (str "files/" file-id ".json")]
@@ -935,14 +952,13 @@
 
   [{:keys [::bfc/ids] :as cfg} output]
 
-  (dm/assert!
-   "expected a set of uuid's for `::bfc/ids` parameter"
-   (and (set? ids)
-        (every? uuid? ids)))
+  (assert
+   (and (set? ids) (every? uuid? ids))
+   "expected a set of uuid's for `::bfc/ids` parameter")
 
-  (dm/assert!
-   "expected instance of jio/IOFactory for `input`"
-   (satisfies? jio/IOFactory output))
+  (assert
+   (satisfies? jio/IOFactory output)
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
         tp (dt/tpoint)
@@ -981,14 +997,14 @@
 (defn import-files!
   [{:keys [::bfc/input] :as cfg}]
 
-  (dm/assert!
-   "expected valid profile-id and project-id on `cfg`"
+  (assert
    (and (uuid? (::bfc/profile-id cfg))
-        (uuid? (::bfc/project-id cfg))))
+        (uuid? (::bfc/project-id cfg)))
+   "expected valid profile-id and project-id on `cfg`")
 
-  (dm/assert!
-   "expected instance of jio/IOFactory for `input`"
-   (io/coercible? input))
+  (assert
+   (io/coercible? input)
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
         tp (dt/tpoint)
@@ -1008,3 +1024,9 @@
                 :id (str id)
                 :elapsed (dt/format-duration (tp))
                 :error? (some? @cs))))))
+
+(defn get-manifest
+  [path]
+  (with-open [input (ZipFile. (fs/file path))]
+    (-> (read-manifest input)
+        (validate-manifest))))

backend/src/app/db.clj

@@ -22,7 +22,8 @@
    [clojure.set :as set]
    [integrant.core :as ig]
    [next.jdbc :as jdbc]
-   [next.jdbc.date-time :as jdbc-dt])
+   [next.jdbc.date-time :as jdbc-dt]
+   [next.jdbc.transaction])
   (:import
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
@@ -41,6 +42,8 @@
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
+(def ^:dynamic *conn* nil)
+
 (declare open)
 (declare create-pool)
 
@@ -223,16 +226,6 @@
     (let [^OutputStream os (.getOutputStream ^LargeObject lobj)]
       (io/make-output-stream os opts))))
 
-(defmacro with-atomic
-  [& args]
-  (if (symbol? (first args))
-    (let [cfgs (first args)
-          body (rest args)]
-      `(jdbc/with-transaction [conn# (::pool ~cfgs)]
-         (let [~cfgs (assoc ~cfgs ::conn conn#)]
-           ~@body)))
-    `(jdbc/with-transaction ~@args)))
-
 (defn open
   [system-or-pool]
   (if (pool? system-or-pool)
@@ -535,43 +528,31 @@
      (l/trc :hint "explicit rollback requested (savepoint)")
      (.rollback conn sp))))
 
+(defn transact!
+  "A lower-level function for executing function in a transaction"
+  ([transactable f] (transact! transactable f {}))
+  ([transactable f opts]
+   (binding [next.jdbc.transaction/*nested-tx* :ignore]
+     (jdbc/transact transactable f opts))))
+
 (defn tx-run!
+  "Run a function in a transaction."
   [system f & params]
-  (cond
-    (connection? system)
+  (if (connection? system)
     (tx-run! {::conn system} f)
-
-    (pool? system)
-    (tx-run! {::pool system} f)
-
-    (::conn system)
-    (let [conn (::conn system)
-          sp   (savepoint conn)]
-      (try
-        (let [system' (-> system
-                          (assoc ::savepoint sp)
-                          (dissoc ::rollback))
-              result  (apply f system' params)]
-          (if (::rollback system)
-            (rollback! conn sp)
-            (release! conn sp))
-          result)
-        (catch Throwable cause
-          (.rollback ^Connection conn ^Savepoint sp)
-          (throw cause))))
-
-    (::pool system)
-    (with-atomic [conn (::pool system)]
-      (let [system' (-> system
-                        (assoc ::conn conn)
-                        (dissoc ::rollback))
-            result  (apply f system' params)]
-        (when (::rollback system)
-          (rollback! conn))
-        result))
-
-    :else
-    (throw (IllegalArgumentException. "invalid system/cfg provided"))))
+    (if (pool? system)
+      (tx-run! {::pool system} f)
+      (if-let [conn (or (::conn system)
+                        (::pool system))]
+        (transact! conn
+                   (fn [conn]
+                     (let [system' (-> system
+                                       (dissoc ::rollback)
+                                       (assoc ::conn conn))]
+                       (apply f system' params)))
+                   {:rollback-only (::rollback system)
+                    :read-only (::read-only system)})
+        (throw (IllegalArgumentException. "invalid system/cfg provided"))))))
 
 (defn run!
   [system f & params]

backend/src/app/features/fdata.clj

@@ -9,7 +9,10 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.files.helpers :as cfh]
+   [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
+   [app.common.types.path :as path]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
@@ -30,7 +33,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn enable-objects-map
-  [file]
+  [file & _opts]
   (let [update-page
         (fn [page]
           (if (and (pmap/pointer-map? page)
@@ -136,10 +139,56 @@
 
 (defn enable-pointer-map
   "Enable the fdata/pointer-map feature on the file."
-  [file]
+  [file & _opts]
   (-> file
       (update :data (fn [fdata]
                       (-> fdata
                           (update :pages-index d/update-vals pmap/wrap)
                           (d/update-when :components pmap/wrap))))
       (update :features conj "fdata/pointer-map")))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PATH-DATA
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn enable-path-data
+  "Enable the fdata/path-data feature on the file."
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content path/content)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features conj "fdata/path-data"))))
+
+(defn disable-path-data
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content vec)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (when-let [conn db/*conn*]
+      (db/delete! conn :file-migration {:file-id (:id file)
+                                        :name "0003-convert-path-content"}))
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features disj "fdata/path-data")
+        (update :migrations disj "0003-convert-path-content")
+        (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))

backend/src/app/features/logical_deletion.clj

@@ -0,0 +1,31 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.logical-deletion
+  "A code related to handle logical deletion mechanism"
+  (:require
+   [app.config :as cf]
+   [app.util.time :as dt]))
+
+(defn get-deletion-delay
+  "Calculate the next deleted-at for a resource (file, team, etc) in function
+  of team settings"
+  [team]
+  (if-let [subscription (get team :subscription)]
+    (cond
+      (and (= (:type subscription) "unlimited")
+           (= (:status subscription) "active"))
+      (dt/duration {:days 30})
+
+      (and (= (:type subscription) "enterprise")
+           (= (:status subscription) "active"))
+      (dt/duration {:days 90})
+
+      :else
+      (cf/get-deletion-delay))
+
+    (cf/get-deletion-delay)))
+

backend/src/app/http.clj

@@ -155,10 +155,10 @@
    [["" {:middleware [[mw/server-timing]
                       [mw/params]
                       [mw/format-response]
-                      [mw/parse-request]
-                      [mw/errors errors/handle]
                       [session/soft-auth cfg]
                       [actoken/soft-auth cfg]
+                      [mw/parse-request]
+                      [mw/errors errors/handle]
                       [mw/restrict-methods]]}
 
      (::mtx/routes cfg)

backend/src/app/http/debug.clj

@@ -24,7 +24,7 @@
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
-   [app.srepl.helpers :as srepl]
+   [app.srepl.main :as srepl]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [app.util.blob :as blob]
@@ -430,6 +430,50 @@
      ::yres/body    "OK"}))
 
 
+(defn- add-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id (some-> params :team-id d/parse-uuid)
+        feature (some-> params :feature str)
+        skip-check (contains? params :skip-check)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? team-id)
+      (ex/raise :type :validation
+                :code :invalid-team-id
+                :hint "provided invalid team id"))
+
+    (srepl/enable-team-feature! team-id feature :skip-check skip-check)
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
+(defn- remove-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id   (some-> params :team-id d/parse-uuid)
+        feature   (some-> params :feature str)
+        skip-check (contains? params :skip-check)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? team-id)
+      (ex/raise :type :validation
+                :code :invalid-team-id
+                :hint "provided invalid team id"))
+
+    (srepl/disable-team-feature! team-id feature :skip-check skip-check)
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -500,6 +544,10 @@
      {:handler (partial resend-email-notification cfg)}]
     ["/actions/reset-file-version"
      {:handler (partial reset-file-version cfg)}]
+    ["/actions/add-team-feature"
+     {:handler (partial add-team-feature)}]
+    ["/actions/remove-team-feature"
+     {:handler (partial remove-team-feature)}]
     ["/file/export" {:handler (partial export-handler cfg)}]
     ["/file/import" {:handler (partial import-handler cfg)}]
     ["/file/data" {:handler (partial file-data-handler cfg)}]

backend/src/app/http/errors.clj

@@ -25,7 +25,6 @@
   (let [claims (-> {}
                    (into (::session/token-claims request))
                    (into (::actoken/token-claims request)))]
-
     {:request/path       (:path request)
      :request/method     (:method request)
      :request/params     (:params request)
@@ -62,7 +61,8 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
-        (l/err :hint "restriction error" :data data)
+        (l/err :hint "restriction error"
+               :cause err)
         {::yres/status 400
          ::yres/body data}))))
 
@@ -102,7 +102,7 @@
       (= code :invalid-image)
       (binding [l/*context* (request->context request)]
         (let [cause (or parent-cause err)]
-          (l/warn :hint "unexpected error on processing image" :cause cause)
+          (l/warn :hint "image process error" :cause cause)
           {::yres/status 400 ::yres/body data}))
 
       :else
@@ -177,7 +177,7 @@
   (let [state (.getSQLState ^java.sql.SQLException error)
         cause (or parent-cause error)]
     (binding [l/*context* (request->context request)]
-      (l/error :hint "PSQL error"
+      (l/error :hint "postgresql error"
                :cause cause)
       (cond
         (= state "57014")

backend/src/app/http/session.clj

@@ -337,16 +337,17 @@
        or (updated_at is null and
            created_at < now() - ?::interval)")
 
-(defmethod ig/init-key ::tasks/gc
-  [_ {:keys [::db/pool ::tasks/max-age] :as cfg}]
-  (l/debug :hint "initializing session gc task" :max-age max-age)
-  (fn [_]
-    (db/with-atomic [conn pool]
-      (let [interval (db/interval max-age)
-            result   (db/exec-one! conn [sql:delete-expired interval interval])
-            result   (:next.jdbc/update-count result)]
-        (l/debug :task "gc"
-                 :hint "clean http sessions"
-                 :deleted result)
-        result))))
+(defn- collect-expired-tasks
+  [{:keys [::db/conn ::tasks/max-age]}]
+  (let [interval (db/interval max-age)
+        result   (db/exec-one! conn [sql:delete-expired interval interval])
+        result   (:next.jdbc/update-count result)]
+    (l/debug :task "gc"
+             :hint "clean http sessions"
+             :deleted result)
+    result))
 
+(defmethod ig/init-key ::tasks/gc
+  [_ {:keys [::tasks/max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
+  (fn [_] (db/tx-run! cfg collect-expired-tasks)))

backend/src/app/http/sse.clj

@@ -9,7 +9,6 @@
   (:refer-clojure :exclude [tap])
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.transit :as t]
    [app.http.errors :as errors]
@@ -54,18 +53,20 @@
      ::yres/status 200
      ::yres/body (yres/stream-body
                   (fn [_ output]
-                    (binding [events/*channel* (sp/chan :buf buf :xf (keep encode))]
-                      (let [listener (events/start-listener
-                                      (partial write! output)
-                                      (partial pu/close! output))]
-                        (try
+                    (let [channel  (sp/chan :buf buf :xf (keep encode))
+                          listener (events/start-listener
+                                    channel
+                                    (partial write! output)
+                                    (partial pu/close! output))]
+                      (try
+                        (binding [events/*channel* channel]
                           (let [result (handler)]
-                            (events/tap :end result))
-                          (catch Throwable cause
-                            (events/tap :error (errors/handle' cause request))
-                            (when-not (ex/instance? java.io.EOFException cause)
-                              (binding [l/*context* (errors/request->context request)]
-                                (l/err :hint "unexpected error on processing sse response" :cause cause))))
-                          (finally
-                            (sp/close! events/*channel*)
-                            (px/await! listener)))))))}))
+                            (events/tap :end result)))
+
+                        (catch Throwable cause
+                          (let [result (errors/handle' cause request)]
+                            (events/tap channel :error result)))
+
+                        (finally
+                          (sp/close! channel)
+                          (px/await! listener))))))}))

backend/src/app/http/websocket.clj

@@ -273,7 +273,7 @@
 
 (defn- http-handler
   [cfg {:keys [params ::session/profile-id] :as request}]
-  (let [session-id (some-> params :session-id sm/parse-uuid)]
+  (let [session-id (some-> params :session-id uuid/parse*)]
     (when-not (uuid? session-id)
       (ex/raise :type :validation
                 :code :missing-session-id

backend/src/app/loggers/audit.clj

@@ -108,6 +108,7 @@
    [::ip-addr {:optional true} ::sm/text]
    [::props {:optional true} [:map-of :keyword :any]]
    [::context {:optional true} [:map-of :keyword :any]]
+   [::tracked-at {:optional true} ::sm/inst]
    [::webhooks/event? {:optional true} ::sm/boolean]
    [::webhooks/batch-timeout {:optional true} ::dt/duration]
    [::webhooks/batch-key {:optional true}
@@ -118,12 +119,12 @@
 
 (defn prepare-event
   [cfg mdata params result]
-  (let [resultm    (meta result)
-        request    (-> params meta ::http/request)
-        profile-id (or (::profile-id resultm)
-                       (:profile-id result)
-                       (::rpc/profile-id params)
-                       uuid/zero)
+  (let [resultm      (meta result)
+        request      (-> params meta ::http/request)
+        profile-id   (or (::profile-id resultm)
+                         (:profile-id result)
+                         (::rpc/profile-id params)
+                         uuid/zero)
 
         session-id   (get params ::rpc/external-session-id)
         event-origin (get params ::rpc/external-event-origin)
@@ -135,14 +136,14 @@
 
                          (clean-props))
 
-        token-id  (::actoken/id request)
-        context   (-> (::context resultm)
-                      (assoc :external-session-id session-id)
-                      (assoc :external-event-origin event-origin)
-                      (assoc :access-token-id (some-> token-id str))
-                      (d/without-nils))
+        token-id     (::actoken/id request)
+        context      (-> (::context resultm)
+                         (assoc :external-session-id session-id)
+                         (assoc :external-event-origin event-origin)
+                         (assoc :access-token-id (some-> token-id str))
+                         (d/without-nils))
 
-        ip-addr   (inet/parse-request request)]
+        ip-addr      (inet/parse-request request)]
 
     {::type (or (::type resultm)
                 (::rpc/type cfg))

backend/src/app/loggers/database.clj

@@ -41,7 +41,7 @@
   (if (or (instance? java.util.concurrent.CompletionException cause)
           (instance? java.util.concurrent.ExecutionException cause))
     (-> record
-        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
+        (assoc ::trace (ex/format-throwable cause :data? true :explain? false :header? false :summary? false))
         (assoc ::l/cause (ex-cause cause))
         (record->report))
 
@@ -53,24 +53,29 @@
                    (assoc :logger/name logger)
                    (assoc :logger/level level)
                    (dissoc :request/params :value :params :data))]
+
       (merge
        {:context (-> (into (sorted-map) ctx)
                      (pp/pprint-str :length 50))
         :props   (pp/pprint-str props :length 50)
-        :hint    (or (ex-message cause) @message)
+        :hint    (or (when-let [message (ex-message cause)]
+                       (if-let [props-hint (:hint props)]
+                         (str props-hint ": " message)
+                         message))
+                     @message)
         :trace   (or (::trace record)
-                     (some-> cause (ex/format-throwable :data? false :explain? false :header? false :summary? false)))}
+                     (some-> cause (ex/format-throwable :data? true :explain? false :header? false :summary? false)))}
 
        (when-let [params (or (:request/params context) (:params context))]
-         {:params (pp/pprint-str params :length 30 :level 13)})
+         {:params (pp/pprint-str params :length 20 :level 20)})
 
        (when-let [value (:value context)]
-         {:value (pp/pprint-str value :length 30 :level 12)})
+         {:value (pp/pprint-str value :length 30 :level 13)})
 
        (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-         {:data (pp/pprint-str data :length 30 :level 12)})
+         {:data (pp/pprint-str data :length 30 :level 13)})
 
-       (when-let [explain (ex/explain data :length 30 :level 12)]
+       (when-let [explain (ex/explain data :length 30 :level 13)]
          {:explain explain})))))
 
 (defn error-record?

backend/src/app/loggers/webhooks.clj

@@ -15,6 +15,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
+   [app.loggers.audit :as audit]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.data.json :as json]
@@ -67,18 +68,27 @@
 (defmethod ig/init-key ::process-event-handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (l/dbg :hint "process webhook event" :name (:name props))
 
-    (when-let [items (lookup-webhooks cfg props)]
-      (l/trc :hint "webhooks found for event" :total (count items))
-      (db/tx-run! cfg (fn [cfg]
-                        (doseq [item items]
-                          (wrk/submit! (-> cfg
-                                           (assoc ::wrk/task :run-webhook)
-                                           (assoc ::wrk/queue :webhooks)
-                                           (assoc ::wrk/max-retries 3)
-                                           (assoc ::wrk/params {:event props
-                                                                :config item})))))))))
+    (let [items (lookup-webhooks cfg props)
+          event {::audit/profile-id (:profile-id props)
+                 ::audit/name "webhook"
+                 ::audit/type "trigger"
+                 ::audit/props {:name (get props :name)
+                                :event-id (get props :id)
+                                :total-affected (count items)}}]
+
+      (audit/insert! cfg event)
+
+      (when items
+        (l/trc :hint "webhooks found for event" :total (count items))
+        (db/tx-run! cfg (fn [cfg]
+                          (doseq [item items]
+                            (wrk/submit! (-> cfg
+                                             (assoc ::wrk/task :run-webhook)
+                                             (assoc ::wrk/queue :webhooks)
+                                             (assoc ::wrk/max-retries 3)
+                                             (assoc ::wrk/params {:event props
+                                                                  :config item}))))))))))
 ;; --- RUN
 
 (declare interpret-exception)

backend/src/app/main.clj

@@ -40,7 +40,6 @@
    [app.svgo :as-alias svgo]
    [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [cider.nrepl :refer [cider-nrepl-handler]]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -605,7 +604,7 @@
     (let [p (promise)]
       (when (contains? cf/flags :nrepl-server)
         (l/inf :hint "start nrepl server" :port 6064)
-        (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
+        (nrepl/start-server :bind "0.0.0.0" :port 6064))
 
       (start)
       (deref p))

backend/src/app/media.clj

@@ -8,12 +8,12 @@
   "Media & Font postprocessing."
   (:require
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.schema.openapi :as-alias oapi]
-   [app.common.spec :as us]
-   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
@@ -22,42 +22,42 @@
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.shell :as sh]
-   [clojure.spec.alpha :as s]
+   [clojure.string]
+   [clojure.xml :as xml]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
+   clojure.lang.XMLHandler
+   java.io.InputStream
+   javax.xml.XMLConstants
+   javax.xml.parsers.SAXParserFactory
+   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(s/def ::path fs/path?)
-(s/def ::filename string?)
-(s/def ::size integer?)
-(s/def ::headers (s/map-of string? string?))
-(s/def ::mtype string?)
+(def schema:upload
+  (sm/register!
+   ^{::sm/type ::upload}
+   [:map {:title "Upload"}
+    [:filename :string]
+    [:size ::sm/int]
+    [:path ::fs/path]
+    [:mtype {:optional true} :string]
+    [:headers {:optional true}
+     [:map-of :string :string]]]))
 
-(s/def ::upload
-  (s/keys :req-un [::filename ::size ::path]
-          :opt-un [::mtype ::headers]))
+(def ^:private schema:input
+  [:map {:title "Input"}
+   [:path ::fs/path]
+   [:mtype {:optional true} ::sm/text]])
 
-;; A subset of fields from the ::upload spec
-(s/def ::input
-  (s/keys :req-un [::path]
-          :opt-un [::mtype]))
-
-(sm/register!
- ^{::sm/type ::upload}
- [:map {:title "Upload"}
-  [:filename :string]
-  [:size ::sm/int]
-  [:path ::fs/path]
-  [:mtype {:optional true} :string]
-  [:headers {:optional true}
-   [:map-of :string :string]]])
+(def ^:private check-input
+  (sm/check-fn schema:input))
 
 (defn validate-media-type!
-  ([upload] (validate-media-type! upload cm/valid-image-types))
+  ([upload] (validate-media-type! upload cm/image-types))
   ([upload allowed]
    (when-not (contains? allowed (:mtype upload))
      (ex/raise :type :validation
@@ -97,17 +97,44 @@
     (catch Throwable e
       (process-error e))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SVG PARSING
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- secure-parser-factory
+  [^InputStream input ^XMLHandler handler]
+  (.. (doto (SAXParserFactory/newInstance)
+        (.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
+        (.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
+      (newSAXParser)
+      (parse input handler)))
+
+(defn- strip-doctype
+  [data]
+  (cond-> data
+    (str/includes? data "<!DOCTYPE")
+    (str/replace #"<\!DOCTYPE[^>]*>" "")))
+
+(defn- parse-svg
+  [text]
+  (let [text (strip-doctype text)]
+    (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
+      (xml/parse istream secure-parser-factory))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; IMAGE THUMBNAILS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(s/def ::width integer?)
-(s/def ::height integer?)
-(s/def ::format #{:jpeg :webp :png})
-(s/def ::quality #(< 0 % 101))
+(def ^:private schema:thumbnail-params
+  [:map {:title "ThumbnailParams"}
+   [:input schema:input]
+   [:format [:enum :jpeg :webp :png]]
+   [:quality [:int {:min 1 :max 100}]]
+   [:width :int]
+   [:height :int]])
 
-(s/def ::thumbnail-params
-  (s/keys :req-un [::input ::format ::width ::height]))
+(def ^:private check-thumbnail-params
+  (sm/check-fn schema:thumbnail-params))
 
 ;; Related info on how thumbnails generation
 ;;  http://www.imagemagick.org/Usage/thumbnails/
@@ -129,30 +156,38 @@
            :data   tmp)))
 
 (defmethod process :generic-thumbnail
-  [{:keys [quality width height] :as params}]
-  (us/assert ::thumbnail-params params)
-  (let [op (doto (IMOperation.)
-             (.addImage)
-             (.autoOrient)
-             (.strip)
-             (.thumbnail ^Integer (int width) ^Integer (int height) ">")
-             (.quality (double quality))
-             (.addImage))]
-    (generic-process (assoc params :operation op))))
+  [params]
+  (let [{:keys [quality width height] :as params}
+        (check-thumbnail-params params)
+
+        operation
+        (doto (IMOperation.)
+          (.addImage)
+          (.autoOrient)
+          (.strip)
+          (.thumbnail ^Integer (int width) ^Integer (int height) ">")
+          (.quality (double quality))
+          (.addImage))]
+
+    (generic-process (assoc params :operation operation))))
 
 (defmethod process :profile-thumbnail
-  [{:keys [quality width height] :as params}]
-  (us/assert ::thumbnail-params params)
-  (let [op (doto (IMOperation.)
-             (.addImage)
-             (.autoOrient)
-             (.strip)
-             (.thumbnail ^Integer (int width) ^Integer (int height) "^")
-             (.gravity "center")
-             (.extent (int width) (int height))
-             (.quality (double quality))
-             (.addImage))]
-    (generic-process (assoc params :operation op))))
+  [params]
+  (let [{:keys [quality width height] :as params}
+        (check-thumbnail-params params)
+
+        operation
+        (doto (IMOperation.)
+          (.addImage)
+          (.autoOrient)
+          (.strip)
+          (.thumbnail ^Integer (int width) ^Integer (int height) "^")
+          (.gravity "center")
+          (.extent (int width) (int height))
+          (.quality (double quality))
+          (.addImage))]
+
+    (generic-process (assoc params :operation operation))))
 
 (defn get-basic-info-from-svg
   [{:keys [tag attrs] :as data}]
@@ -182,12 +217,28 @@
                  {:width (int width)
                   :height (int height)})))]))
 
+(defn- get-dimensions-with-orientation [^String path]
+  ;; Image magick doesn't give info about exif rotation so we use the identify command
+  ;; If we are processing an animated gif we use the first frame with -scene 0
+  (let [dim-result (sh/sh "identify" "-format" "%w %h\n" path)
+        orient-result (sh/sh "identify" "-format" "%[EXIF:Orientation]\n" path)]
+    (if (and (= 0 (:exit dim-result))
+             (= 0 (:exit orient-result)))
+      (let [[w h] (-> (:out dim-result)
+                      str/trim
+                      (clojure.string/split #"\s+")
+                      (->> (mapv #(Integer/parseInt %))))
+            orientation (-> orient-result :out str/trim)]
+        (case orientation
+          ("6" "8") {:width h :height w} ; Rotated 90 or 270 degrees
+          {:width w :height h}))         ; Normal or unknown orientation
+      nil)))
+
 (defmethod process :info
   [{:keys [input] :as params}]
-  (us/assert ::input input)
-  (let [{:keys [path mtype]} input]
+  (let [{:keys [path mtype] :as input} (check-input input)]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp parse-svg get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file
@@ -202,13 +253,17 @@
                     :code :media-type-mismatch
                     :hint (str "Seems like you are uploading a file whose content does not match the extension."
                                "Expected: " mtype ". Got: " mtype')))
-        ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
-        ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
-        ;; any frame.
-        (assoc input
-               :width  (.getPageWidth instance)
-               :height (.getPageHeight instance)
-               :ts (dt/now))))))
+        (let [{:keys [width height]}
+              (or (get-dimensions-with-orientation (str path))
+                  (do
+                    (l/warn "Failed to read image dimensions with orientation; falling back to im4java"
+                            {:path path})
+                    {:width  (.getPageWidth instance)
+                     :height (.getPageHeight instance)}))]
+          (assoc input
+                 :width  width
+                 :height height
+                 :ts (dt/now)))))))
 
 (defmethod process-error org.im4java.core.InfoException
   [error]

backend/src/app/rpc/commands/access_token.clj

@@ -43,13 +43,8 @@
     (decode-row token)))
 
 (defn repl:create-access-token
-  [{:keys [::db/pool] :as system} profile-id name expiration]
-  (db/with-atomic [conn pool]
-    (let [props (:app.setup/props system)]
-      (create-access-token {::db/conn conn ::setup/props props}
-                           profile-id
-                           name
-                           expiration))))
+  [cfg profile-id name expiration]
+  (db/tx-run! cfg create-access-token profile-id name expiration))
 
 (def ^:private schema:create-access-token
   [:map {:title "create-access-token"}

backend/src/app/rpc/commands/audit.clj

@@ -92,9 +92,9 @@
      [:string {:max 250}]
      [::sm/one-of {:format "string"} valid-event-types]]]
    [:props
-    [:map-of :keyword :any]]
+    [:map-of :keyword ::sm/any]]
    [:context {:optional true}
-    [:map-of :keyword :any]]])
+    [:map-of :keyword ::sm/any]]])
 
 (def schema:push-audit-events
   [:map {:title "push-audit-events"}

backend/src/app/rpc/commands/auth.clj

@@ -55,7 +55,7 @@
                 (contains? cf/flags :login-with-password))
     (ex/raise :type :restriction
               :code :login-disabled
-              :hint "login is disabled in this instance"))
+              :hint "login is disabled"))
 
   (letfn [(check-password [cfg profile password]
             (if (= (:password profile) "!")
@@ -79,7 +79,8 @@
                         :code :wrong-credentials))
             (when (:is-blocked profile)
               (ex/raise :type :restriction
-                        :code :profile-blocked))
+                        :code :profile-blocked
+                        :hint "profile is marked as blocked"))
             (when-not (check-password cfg profile password)
               (ex/raise :type :validation
                         :code :wrong-credentials))
@@ -149,7 +150,7 @@
 ;; ---- COMMAND: Recover Profile
 
 (defn recover-profile
-  [{:keys [::db/pool] :as cfg} {:keys [token password]}]
+  [{:keys [::db/conn] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
             (let [tdata (tokens/verify (::setup/props cfg) {:token token :iss :password-recovery})]
               (:profile-id tdata)))
@@ -159,10 +160,10 @@
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
-    (db/with-atomic [conn pool]
-      (->> (validate-token token)
-           (update-password conn))
-      nil)))
+    (->> (validate-token token)
+         (update-password conn))
+
+    nil))
 
 (def schema:recover-profile
   [:map {:title "recover-profile"}
@@ -173,7 +174,8 @@
   {::rpc/auth false
    ::doc/added "1.15"
    ::sm/params schema:recover-profile
-   ::climit/id :auth/global}
+   ::climit/id :auth/global
+   ::db/transaction true}
   [cfg params]
   (recover-profile cfg params))
 
@@ -182,11 +184,11 @@
 (defn- validate-register-attempt!
   [cfg params]
 
-  (when (or
-         (not (contains? cf/flags :registration))
-         (not (contains? cf/flags :login-with-password)))
+  (when (or (not (contains? cf/flags :registration))
+            (not (contains? cf/flags :login-with-password)))
     (ex/raise :type :restriction
-              :code :registration-disabled))
+              :code :registration-disabled
+              :hint "registration disabled"))
 
   (when (contains? params :invitation-token)
     (let [invitation (tokens/verify (::setup/props cfg)
@@ -200,12 +202,14 @@
   (when (and (email.blacklist/enabled? cfg)
              (email.blacklist/contains? cfg (:email params)))
     (ex/raise :type :restriction
-              :code :email-domain-is-not-allowed))
+              :code :email-domain-is-not-allowed
+              :hint "email domain in blacklist"))
 
   (when (and (email.whitelist/enabled? cfg)
              (not (email.whitelist/contains? cfg (:email params))))
     (ex/raise :type :restriction
-              :code :email-domain-is-not-allowed))
+              :code :email-domain-is-not-allowed
+              :hint "email domain not in whitelist"))
 
   ;; Perform a basic validation of email & password
   (when (= (str/lower (:email params))
@@ -218,28 +222,30 @@
     (ex/raise :type :restriction
               :code :email-has-permanent-bounces
               :email (:email params)
-              :hint "looks like the email has bounce reports"))
+              :hint "email has bounce reports"))
 
   (when (eml/has-complaint-reports? cfg (:email params))
     (ex/raise :type :restriction
               :code :email-has-complaints
               :email (:email params)
-              :hint "looks like the email has complaint reports")))
+              :hint "email has complaint reports")))
 
 (defn prepare-register
-  [{:keys [::db/pool] :as cfg} {:keys [email] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [fullname email accept-newsletter-updates] :as params}]
 
   (validate-register-attempt! cfg params)
 
   (let [email   (profile/clean-email email)
         profile (profile/get-profile-by-email pool email)
         params  {:email email
+                 :fullname fullname
                  :password (:password params)
                  :invitation-token (:invitation-token params)
                  :backend "penpot"
                  :iss :prepared-register
                  :profile-id (:id profile)
-                 :exp (dt/in-future {:days 7})}
+                 :exp (dt/in-future {:days 7})
+                 :props {:newsletter-updates (or accept-newsletter-updates false)}}
 
         params (d/without-nils params)
         token  (tokens/generate (::setup/props cfg) params)]
@@ -249,8 +255,10 @@
 
 (def schema:prepare-register-profile
   [:map {:title "prepare-register-profile"}
+   [:fullname ::sm/text]
    [:email ::sm/email]
    [:password schema:password]
+   [:create-welcome-file {:optional true} :boolean]
    [:invitation-token {:optional true} schema:token]])
 
 (sv/defmethod ::prepare-register-profile
@@ -354,13 +362,9 @@
                 :extra-data ptoken})))
 
 (defn register-profile
-  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token fullname theme] :as params}]
-  (let [theme      (when (= theme "light") theme)
-        claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
-        params     (-> claims
-                       (into params)
-                       (assoc :fullname fullname)
-                       (assoc :theme theme))
+  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token] :as params}]
+  (let [claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
+        params     (into claims params)
 
         profile    (if-let [profile-id (:profile-id claims)]
                      (profile/get-profile conn profile-id)
@@ -474,10 +478,7 @@
 
 (def schema:register-profile
   [:map {:title "register-profile"}
-   [:token schema:token]
-   [:fullname [::sm/word-string {:max 100}]]
-   [:theme {:optional true} [:string {:max 10}]]
-   [:create-welcome-file {:optional true} :boolean]])
+   [:token schema:token]])
 
 (sv/defmethod ::register-profile
   {::rpc/auth false

backend/src/app/rpc/commands/binfile.clj

@@ -38,7 +38,6 @@
 (def ^:private
   schema:export-binfile
   [:map {:title "export-binfile"}
-   [:name [:string {:max 250}]]
    [:file-id ::sm/uuid]
    [:version {:optional true} ::sm/int]
    [:include-libraries ::sm/boolean]
@@ -78,7 +77,7 @@
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
    ::webhooks/event? true
-   ::sm/result schema:export-binfile}
+   ::sm/params schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
   (fn [_]
@@ -116,7 +115,8 @@
 
     (db/update! pool :project
                 {:modified-at (dt/now)}
-                {:id project-id})
+                {:id project-id}
+                {::db/return-keys false})
 
     result))
 
@@ -134,11 +134,18 @@
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [params (-> params
-                   (assoc :profile-id profile-id)
-                   (assoc :version (or version 1)))]
+  (let [version  (or version 1)
+        params   (-> params
+                     (assoc :profile-id profile-id)
+                     (assoc :version version))
+        manifest (case (int version)
+                   1 nil
+                   3 (bf.v3/get-manifest (:path file)))]
+
     (with-meta
       (sse/response (partial import-binfile cfg params))
-      {::audit/props {:file nil}})))
+      {::audit/props {:file nil
+                      :generated-by (:generated-by manifest)
+                      :referer (:referer manifest)}})))

backend/src/app/rpc/commands/comments.clj

@@ -797,3 +797,18 @@
                 {:id id}
                 {::db/return-keys false})
     nil))
+
+(def ^:private
+  schema:mark-all-threads-as-read
+  [:map {:title "mark-all-threads-as-read"}
+   [:threads [:vector ::sm/uuid]]])
+
+(sv/defmethod ::mark-all-threads-as-read
+  {::doc/added "1.15"
+   ::sm/params schema:mark-all-threads-as-read}
+  [cfg {:keys [::rpc/profile-id threads] :as params}]
+  (db/tx-run!
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (doseq [thread-id threads]
+       (upsert-comment-thread-status! conn profile-id thread-id)))))

backend/src/app/rpc/commands/demo.clj

@@ -27,7 +27,7 @@
   {::rpc/auth false
    ::doc/added "1.15"
    ::doc/changes ["1.15" "This method is migrated from mutations to commands."]}
-  [{:keys [::db/pool] :as cfg} _]
+  [cfg _]
 
   (when-not (contains? cf/flags :demo-users)
     (ex/raise :type :validation
@@ -49,9 +49,11 @@
                   :password (profile/derive-password cfg password)
                   :props {}}]
 
-    (db/with-atomic [conn pool]
-      (let [profile (->> (auth/create-profile! conn params)
-                         (auth/create-profile-rels! conn))]
-        (with-meta {:email email
-                    :password password}
-          {::audit/profile-id (:id profile)})))))
+
+    (let [profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                                    (->> (auth/create-profile! conn params)
+                                         (auth/create-profile-rels! conn))))]
+      (with-meta {:email email
+                  :password password}
+        {::audit/profile-id (:id profile)}))))
+

backend/src/app/rpc/commands/files.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -23,6 +24,7 @@
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
    [app.features.file-migrations :as feat.fmigr]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
@@ -189,7 +191,7 @@
    [:is-shared ::sm/boolean]
    [:project-id ::sm/uuid]
    [:created-at ::dt/instant]
-   [:data {:optional true} :any]])
+   [:data {:optional true} ::sm/any]])
 
 (def schema:permissions-mixin
   [:map {:title "PermissionsMixin"}
@@ -208,10 +210,11 @@
    [:project-id {:optional true} ::sm/uuid]])
 
 (defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
             pmap/*tracked* (pmap/create-tracked)]
-    (let [;; For avoid unnecesary overhead of creating multiple pointers and
+    (let [libs (delay (bfc/get-resolved-file-libraries cfg file))
+          ;; For avoid unnecesary overhead of creating multiple pointers and
           ;; handly internally with objects map in their worst case (when
           ;; probably all shapes and all pointers will be readed in any
           ;; case), we just realize/resolve them before applying the
@@ -219,43 +222,45 @@
           file (-> file
                    (update :data feat.fdata/process-pointers deref)
                    (update :data feat.fdata/process-objects (partial into {}))
-                   (fmg/migrate-file))
+                   (fmg/migrate-file libs))]
 
-          ;; When file is migrated, we break the rule of no perform
-          ;; mutations on get operations and update the file with all
-          ;; migrations applied
-          ;;
-          ;; WARN: he following code will not work on read-only mode,
-          ;; it is a known issue; we keep is not implemented until we
-          ;; really need this.
-          file (if (contains? (:features file) "fdata/objects-map")
-                 (feat.fdata/enable-objects-map file)
-                 file)
-          file (if (contains? (:features file) "fdata/pointer-map")
-                 (feat.fdata/enable-pointer-map file)
-                 file)]
+      (if (or read-only? (db/read-only? conn))
+        file
+        (let [;; When file is migrated, we break the rule of no perform
+              ;; mutations on get operations and update the file with all
+              ;; migrations applied
+              file (if (contains? (:features file) "fdata/objects-map")
+                     (feat.fdata/enable-objects-map file)
+                     file)
+              file (if (contains? (:features file) "fdata/pointer-map")
+                     (feat.fdata/enable-pointer-map file)
+                     file)]
 
-      (db/update! conn :file
-                  {:data (blob/encode (:data file))
-                   :version (:version file)
-                   :features (db/create-array conn "text" (:features file))}
-                  {:id id})
+          (db/update! conn :file
+                      {:data (blob/encode (:data file))
+                       :version (:version file)
+                       :features (db/create-array conn "text" (:features file))}
+                      {:id id}
+                      {::db/return-keys false})
 
-      (when (contains? (:features file) "fdata/pointer-map")
-        (feat.fdata/persist-pointers! cfg id))
+          (when (contains? (:features file) "fdata/pointer-map")
+            (feat.fdata/persist-pointers! cfg id))
 
-      (feat.fmigr/upsert-migrations! conn file)
-      (feat.fmigr/resolve-applied-migrations cfg file))))
+          (feat.fmigr/upsert-migrations! conn file)
+          (feat.fmigr/resolve-applied-migrations cfg file))))))
 
 (defn get-file
   [{:keys [::db/conn ::wrk/executor] :as cfg} id
    & {:keys [project-id
              migrate?
              include-deleted?
-             lock-for-update?]
+             lock-for-update?
+             preload-pointers?]
       :or {include-deleted? false
            lock-for-update? false
-           migrate? true}}]
+           migrate? true
+           preload-pointers? false}
+      :as options}]
 
   (assert (db/connection? conn) "expected cfg with valid connection")
 
@@ -273,10 +278,16 @@
         ;; because it has heavy and synchronous operations for
         ;; decoding file body that are not very friendly with virtual
         ;; threads.
-        file   (px/invoke! executor #(decode-row file))]
+        file   (px/invoke! executor #(decode-row file))
+
+        file   (if (and migrate? (fmg/need-migration? file))
+                 (migrate-file cfg file options)
+                 file)]
+
+    (if preload-pointers?
+      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+        (update file :data feat.fdata/process-pointers deref))
 
-    (if (and migrate? (fmg/need-migration? file))
-      (migrate-file cfg file)
       file)))
 
 (defn get-minimal-file
@@ -292,7 +303,7 @@
 
 (defn get-file-etag
   [{:keys [::rpc/profile-id]} {:keys [modified-at revn vern permissions]}]
-  (str profile-id "/" revn "/" vern "/"
+  (str profile-id "/" revn "/" vern "/" (hash fmg/available-migrations) "/"
        (dt/format-instant modified-at :iso)
        "/"
        (uri/map->query-string permissions)))
@@ -323,11 +334,12 @@
 
           file (-> (get-file cfg id :project-id project-id)
                    (assoc :permissions perms)
+                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
           (cfeat/check-client-features! (:features params))
-          (cfeat/check-file-features! (:features file) (:features params)))
+          (cfeat/check-file-features! (:features file)))
 
       ;; This operation is needed for backward comapatibility with frontends that
       ;; does not support pointer-map resolution mechanism; this just resolves the
@@ -473,7 +485,7 @@
   (update page :objects update-vals #(dissoc % :thumbnail)))
 
 (defn get-page
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id page-id object-id share-id] :as params}]
 
   (when (and (uuid? object-id)
              (not (uuid? page-id)))
@@ -481,22 +493,30 @@
               :code :params-validation
               :hint "page-id is required when object-id is provided"))
 
-  (let [team (teams/get-team conn
-                             :profile-id profile-id
-                             :file-id file-id)
+  (let [perms (get-permissions conn profile-id file-id share-id)
 
-        file (get-file cfg file-id)
+        file  (get-file cfg file-id :read-only? true)
 
-        _    (-> (cfeat/get-team-enabled-features cf/flags team)
-                 (cfeat/check-client-features! (:features params))
-                 (cfeat/check-file-features! (:features file) (:features params)))
+        proj  (db/get conn :project {:id (:project-id file)})
 
-        page (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-               (let [page-id (or page-id (-> file :data :pages first))
-                     page    (dm/get-in file [:data :pages-index page-id])]
-                 (if (pmap/pointer-map? page)
-                   (deref page)
-                   page)))]
+        team  (-> (db/get conn :team {:id (:team-id proj)})
+                  (teams/decode-row))
+
+        _     (-> (cfeat/get-team-enabled-features cf/flags team)
+                  (cfeat/check-client-features! (:features params))
+                  (cfeat/check-file-features! (:features file)))
+
+        page  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+                (let [page-id (or page-id (-> file :data :pages first))
+                      page    (dm/get-in file [:data :pages-index page-id])]
+                  (if (pmap/pointer-map? page)
+                    (deref page)
+                    page)))]
+
+    (when-not perms
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "object not found"))
 
     (cond-> (prune-thumbnails page)
       (some? object-id)
@@ -539,7 +559,10 @@
           f.project_id,
           f.created_at,
           f.modified_at,
+          f.data_backend,
+          f.data_ref_id,
           f.name,
+          f.version,
           f.is_shared,
           ft.media_id,
           p.team_id
@@ -575,7 +598,11 @@
   (teams/check-read-permissions! conn profile-id team-id)
   (->> (db/exec! conn [sql:team-shared-files team-id])
        (into #{} (comp
-                  (map decode-row)
+                  ;; NOTE: this decode operation is a workaround for a
+                  ;; fast fix, this should be approached with a more
+                  ;; efficient implementation, for now it loads all
+                  ;; the files in memory.
+                  (map (partial bfc/decode-file cfg))
                   (map (fn [row]
                          (if-let [media-id (:media-id row)]
                            (-> row
@@ -598,42 +625,6 @@
 
 ;; --- COMMAND QUERY: get-file-libraries
 
-(def ^:private sql:get-file-libraries
-  "WITH RECURSIVE libs AS (
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-      WHERE flr.file_id = ?::uuid
-    UNION
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-       JOIN libs AS l ON (flr.file_id = l.id)
-   )
-   SELECT l.id,
-          l.features,
-          l.project_id,
-          l.created_at,
-          l.modified_at,
-          l.deleted_at,
-          l.name,
-          l.revn,
-          l.vern,
-          l.synced_at,
-          l.is_shared
-     FROM libs AS l
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
-
-(defn get-file-libraries
-  [conn file-id]
-  (into []
-        (comp
-         ;; FIXME: :is-indirect set to false to all rows looks
-         ;; completly useless
-         (map #(assoc % :is-indirect false))
-         (map decode-row))
-        (db/exec! conn [sql:get-file-libraries file-id])))
-
 (def ^:private schema:get-file-libraries
   [:map {:title "get-file-libraries"}
    [:file-id ::sm/uuid]])
@@ -645,7 +636,7 @@
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id file-id)
-    (get-file-libraries conn file-id)))
+    (bfc/get-file-libraries conn file-id)))
 
 
 ;; --- COMMAND QUERY: Files that use this File library
@@ -730,11 +721,13 @@
                              :project-id project-id
                              :file-id id)
 
-        file (get-file cfg id :project-id project-id)]
+        file (get-file cfg id
+                       :project-id project-id
+                       :read-only? true)]
 
     (-> (cfeat/get-team-enabled-features cf/flags team)
         (cfeat/check-client-features! (:features params))
-        (cfeat/check-file-features! (:features file) (:features params)))
+        (cfeat/check-file-features! (:features file)))
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
       {:name             (:name file)
@@ -803,17 +796,17 @@
     [:id ::sm/uuid]
     [:name [:string {:max 250}]]
     [:created-at ::dt/instant]
-    [:modified-at ::dt/instant]]}
+    [:modified-at ::dt/instant]]
 
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [file (rename-file conn params)]
-      (rph/with-meta
-        (select-keys file [:id :name :created-at :modified-at])
-        {::audit/props {:project-id (:project-id file)
-                        :created-at (:created-at file)
-                        :modified-at (:modified-at file)}}))))
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [file (rename-file conn params)]
+    (rph/with-meta
+      (select-keys file [:id :name :created-at :modified-at])
+      {::audit/props {:project-id (:project-id file)
+                      :created-at (:created-at file)
+                      :modified-at (:modified-at file)}})))
 
 ;; --- MUTATION COMMAND: set-file-shared
 
@@ -949,12 +942,13 @@
 ;; --- MUTATION COMMAND: delete-file
 
 (defn- mark-file-deleted
-  [conn file-id]
-  (let [file (db/update! conn :file
-                         {:deleted-at (dt/now)}
-                         {:id file-id}
-                         {::db/return-keys [:id :name :is-shared :deleted-at
-                                            :project-id :created-at :modified-at]})]
+  [conn team file-id]
+  (let [delay (ldel/get-deletion-delay team)
+        file  (db/update! conn :file
+                          {:deleted-at (dt/in-future delay)}
+                          {:id file-id}
+                          {::db/return-keys [:id :name :is-shared :deleted-at
+                                             :project-id :created-at :modified-at]})]
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :file
@@ -970,7 +964,11 @@
 (defn- delete-file
   [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
   (check-edition-permissions! conn profile-id id)
-  (let [file (mark-file-deleted conn id)]
+  (let [team (teams/get-team conn
+                             :profile-id profile-id
+                             :file-id id)
+        file (mark-file-deleted conn team id)]
+
     (rph/with-meta (rph/wrap)
       {::audit/props {:project-id (:project-id file)
                       :name (:name file)
@@ -1005,15 +1003,17 @@
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:link-file-to-library}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id library-id] :as params}]
+  [cfg {:keys [::rpc/profile-id file-id library-id] :as params}]
   (when (= file-id library-id)
     (ex/raise :type :validation
               :code :invalid-library
               :hint "A file cannot be linked to itself"))
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (check-edition-permissions! conn profile-id library-id)
-    (link-file-to-library conn params)))
+
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn]}]
+                (check-edition-permissions! conn profile-id file-id)
+                (check-edition-permissions! conn profile-id library-id)
+                (link-file-to-library conn params))))
 
 ;; --- MUTATION COMMAND: unlink-file-from-library
 
@@ -1031,12 +1031,12 @@
 (sv/defmethod ::unlink-file-from-library
   {::doc/added "1.17"
    ::webhooks/event? true
-   ::sm/params schema:unlink-file-to-library}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (unlink-file-from-library conn params)
-    nil))
+   ::sm/params schema:unlink-file-to-library
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (unlink-file-from-library conn params)
+  nil)
 
 ;; --- MUTATION COMMAND: update-sync
 
@@ -1056,12 +1056,11 @@
 (sv/defmethod ::update-file-library-sync-status
   "Update the synchronization status of a file->library link"
   {::doc/added "1.17"
-   ::sm/params schema:update-file-library-sync-status}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (update-sync conn params)))
-
+   ::sm/params schema:update-file-library-sync-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (update-sync conn params))
 
 ;; --- MUTATION COMMAND: ignore-sync
 
@@ -1082,9 +1081,9 @@
 (sv/defmethod ::ignore-file-library-sync-status
   "Ignore updates in linked files"
   {::doc/added "1.17"
-   ::sm/params schema:ignore-file-library-sync-status}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (->  (ignore-sync conn params)
-         (update :features db/decode-pgarray #{}))))
+   ::sm/params schema:ignore-file-library-sync-status
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (check-edition-permissions! conn profile-id file-id)
+  (->  (ignore-sync conn params)
+       (update :features db/decode-pgarray #{})))

backend/src/app/rpc/commands/files_create.clj

@@ -55,8 +55,8 @@
                                :features features
                                :ignore-sync-until ignore-sync-until
                                :modified-at modified-at
-                               :deleted-at deleted-at
-                               :create-page create-page
+                               :deleted-at deleted-at}
+                              {:create-page create-page
                                :page-id page-id})
           file (-> (bfc/insert-file! cfg file)
                    (bfc/decode-row))]
@@ -91,9 +91,6 @@
                                  :project-id project-id)
         team-id  (:id team)
 
-        ;; When we create files, we only need to respect the team
-        ;; features, because some features can be enabled
-        ;; globally, but the team is still not migrated properly.
         features (-> (cfeat/get-team-enabled-features cf/flags team)
                      (cfeat/check-client-features! (:features params)))
 
@@ -107,25 +104,28 @@
 
         params   (-> params
                      (assoc :profile-id profile-id)
-                     (assoc :features (set/difference features cfeat/frontend-only-features)))]
+                     (assoc :features features))]
 
     (quotes/check! cfg {::quotes/id ::quotes/files-per-project
                         ::quotes/team-id team-id
                         ::quotes/profile-id profile-id
                         ::quotes/project-id project-id})
 
-    ;; FIXME: IMPORTANT: this code can have race
-    ;; conditions, because we have no locks for updating
-    ;; team so, creating two files concurrently can lead
-    ;; to lost team features updating
+    ;; FIXME: IMPORTANT: this code can have race conditions, because
+    ;; we have no locks for updating team so, creating two files
+    ;; concurrently can lead to lost team features updating
 
-    ;; When newly computed features does not match exactly with
-    ;; the features defined on team row, we update it.
-    (when (not= features (:features team))
-      (let [features (db/create-array conn "text" features)]
+    (when-let [features (-> features
+                            (set/difference (:features team))
+                            (set/difference cfeat/no-team-inheritable-features)
+                            (not-empty))]
+      (let [features (->> features
+                          (set/union (:features team))
+                          (db/create-array conn "text"))]
         (db/update! conn :team
                     {:features features}
-                    {:id team-id})))
+                    {:id (:id team)}
+                    {::db/return-keys false})))
 
     (-> (create-file cfg params)
         (vary-meta assoc ::audit/props {:team-id team-id}))))

backend/src/app/rpc/commands/files_share.clj

@@ -33,11 +33,11 @@
   pages of a file with specific permissions (who-comment and who-inspect)."
   {::doc/added "1.18"
    ::doc/module :files
-   ::sm/params schema:create-share-link}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (create-share-link conn (assoc params :profile-id profile-id))))
+   ::sm/params schema:create-share-link
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id file-id] :as params}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (create-share-link conn (assoc params :profile-id profile-id)))
 
 (defn create-share-link
   [conn {:keys [profile-id file-id pages who-comment who-inspect]}]
@@ -61,10 +61,10 @@
 (sv/defmethod ::delete-share-link
   {::doc/added "1.18"
    ::doc/module ::files
-   ::sm/params schema:delete-share-link}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [slink (db/get-by-id conn :share-link id)]
-      (files/check-edition-permissions! conn profile-id (:file-id slink))
-      (db/delete! conn :share-link {:id id})
-      nil)))
+   ::sm/params schema:delete-share-link
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
+  (let [slink (db/get-by-id conn :share-link id)]
+    (files/check-edition-permissions! conn profile-id (:file-id slink))
+    (db/delete! conn :share-link {:id id})
+    nil))

backend/src/app/rpc/commands/files_temp.clj

@@ -14,7 +14,6 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.components-v2 :as feat.compv2]
    [app.features.fdata :as fdata]
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
@@ -110,7 +109,7 @@
 ;; --- MUTATION COMMAND: persist-temp-file
 
 (defn persist-temp-file
-  [{:keys [::db/conn] :as cfg} {:keys [id ::rpc/profile-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
   (let [file (files/get-file cfg id
                              :migrate? false
                              :lock-for-update? true)]
@@ -119,7 +118,6 @@
       (ex/raise :type :validation
                 :code :cant-persist-already-persisted-file))
 
-
     (let [changes (->> (db/cursor conn
                                   (sql/select :file-change {:file-id id}
                                               {:order-by [[:revn :asc]]})
@@ -147,19 +145,6 @@
                    :revn 1
                    :data (blob/encode (:data file))}
                   {:id id})
-
-      (let [team          (teams/get-team conn :profile-id profile-id :project-id (:project-id file))
-            file-features (:features file)
-            team-features (cfeat/get-team-enabled-features cf/flags team)]
-        (when (and (contains? team-features "components/v2")
-                   (not (contains? file-features "components/v2")))
-          ;; Migrate components v2
-          (feat.compv2/migrate-file! cfg
-                                     (:id file)
-                                     :max-procs 2
-                                     :validate? true
-                                     :throw-on-validate? true)))
-
       nil)))
 
 (def ^:private schema:persist-temp-file

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -10,7 +10,6 @@
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
    [app.common.files.helpers :as cfh]
-   [app.common.files.migrations :as fmg]
    [app.common.geom.shapes :as gsh]
    [app.common.schema :as sm]
    [app.common.thumbnails :as thc]
@@ -18,7 +17,6 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -180,8 +178,7 @@
 (def ^:private
   schema:get-file-data-for-thumbnail
   [:map {:title "get-file-data-for-thumbnail"}
-   [:file-id ::sm/uuid]
-   [:features {:optional true} ::cfeat/features]])
+   [:file-id ::sm/uuid]])
 
 (def ^:private
   schema:partial-file
@@ -201,18 +198,16 @@
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
                  (files/check-read-permissions! conn profile-id file-id)
 
-                 (let [team     (teams/get-team conn
-                                                :profile-id profile-id
-                                                :file-id file-id)
+                 (let [team (teams/get-team conn
+                                            :profile-id profile-id
+                                            :file-id file-id)
 
-                       file     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-                                  (-> (files/get-file cfg file-id :migrate? false)
-                                      (update :data feat.fdata/process-pointers deref)
-                                      (fmg/migrate-file)))]
+                       file (files/get-file cfg file-id
+                                            :preload-pointers? true
+                                            :read-only? true)]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
-                       (cfeat/check-client-features! (:features params))
-                       (cfeat/check-file-features! (:features file) (:features params)))
+                       (cfeat/check-file-features! (:features file)))
 
                    {:file-id file-id
                     :revn (:revn file)

backend/src/app/rpc/commands/files_update.clj

@@ -20,6 +20,7 @@
    [app.db :as db]
    [app.features.fdata :as feat.fdata]
    [app.features.file-migrations :as feat.fmigr]
+   [app.features.logical-deletion :as ldel]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
@@ -142,7 +143,7 @@
 
                           features (-> (cfeat/get-team-enabled-features cf/flags team)
                                        (cfeat/check-client-features! (:features params))
-                                       (cfeat/check-file-features! (:features file) (:features params)))
+                                       (cfeat/check-file-features! (:features file)))
 
                           changes  (if changes-with-metadata
                                      (->> changes-with-metadata (mapcat :changes) vec)
@@ -177,12 +178,19 @@
                                             :stored-revn (:revn file)}))
 
                       ;; When newly computed features does not match exactly with
-                      ;; the features defined on team row, we update it.
-                      (when (not= features (:features team))
-                        (let [features (db/create-array conn "text" features)]
+                      ;; the features defined on team row, we update it
+                      (when-let [features (-> features
+                                              (set/difference (:features team))
+                                              (set/difference cfeat/no-team-inheritable-features)
+                                              (not-empty))]
+                        (let [features (->> features
+                                            (set/union (:features team))
+                                            (db/create-array conn "text"))]
                           (db/update! conn :team
                                       {:features features}
-                                      {:id (:id team)})))
+                                      {:id (:id team)}
+                                      {::db/return-keys false})))
+
 
                       (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
 
@@ -202,7 +210,7 @@
 
   Only intended for internal use on this module."
   [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
-   {:keys [profile-id file features changes session-id skip-validate] :as params}]
+   {:keys [profile-id file team features changes session-id skip-validate] :as params}]
 
   (let [;; Retrieve the file data
         file  (feat.fmigr/resolve-applied-migrations cfg file)
@@ -236,7 +244,7 @@
                    :created-at timestamp
                    :updated-at timestamp
                    :deleted-at (if (::snapshot-data file)
-                                 (dt/plus timestamp (cf/get-deletion-delay))
+                                 (dt/plus timestamp (ldel/get-deletion-delay team))
                                  (dt/plus timestamp (dt/duration {:hours 1})))
                    :file-id (:id file)
                    :revn (:revn file)
@@ -333,6 +341,7 @@
                                     (-> data
                                         (blob/decode)
                                         (assoc :id (:id file)))))
+          libs (delay (bfc/get-resolved-file-libraries cfg file))
 
           ;; For avoid unnecesary overhead of creating multiple pointers
           ;; and handly internally with objects map in their worst
@@ -343,7 +352,7 @@
                  (-> file
                      (update :data feat.fdata/process-pointers deref)
                      (update :data feat.fdata/process-objects (partial into {}))
-                     (fmg/migrate-file))
+                     (fmg/migrate-file libs))
                  file)
 
           file (apply update-fn cfg file args)
@@ -372,13 +381,6 @@
 
       (bfc/encode-file cfg file))))
 
-(defn- get-file-libraries
-  "A helper for preload file libraries, mainly used for perform file
-  semantical and structural validation"
-  [{:keys [::db/conn] :as cfg} file]
-  (->> (files/get-file-libraries conn (:id file))
-       (into [file] (map #(bfc/get-file cfg (:id %))))
-       (d/index-by :id)))
 
 (defn- soft-validate-file-schema!
   [file]
@@ -404,7 +406,7 @@
         (when (and (or (contains? cf/flags :file-validation)
                        (contains? cf/flags :soft-file-validation))
                    (not skip-validate))
-          (get-file-libraries cfg file))
+          (bfc/get-resolved-file-libraries cfg file))
 
 
         ;; The main purpose of this atom is provide a contextual state

backend/src/app/rpc/commands/fonts.clj

@@ -12,6 +12,7 @@
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.db.sql :as-alias sql]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -80,9 +81,9 @@
 (def ^:private schema:create-font-variant
   [:map {:title "create-font-variant"}
    [:team-id ::sm/uuid]
-   [:data [:map-of :string :any]]
+   [:data [:map-of ::sm/text ::sm/any]]
    [:font-id ::sm/uuid]
-   [:font-family :string]
+   [:font-family ::sm/text]
    [:font-weight [::sm/one-of {:format "number"} valid-weight]]
    [:font-style [::sm/one-of {:format "string"} valid-style]]])
 
@@ -202,32 +203,40 @@
 (sv/defmethod ::delete-font
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font}
-  [cfg {:keys [::rpc/profile-id id team-id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (teams/check-edition-permissions! conn profile-id team-id)
-                (let [fonts   (db/query conn :team-font-variant
-                                        {:team-id team-id
-                                         :font-id id
-                                         :deleted-at nil}
-                                        {::sql/for-update true})
-                      tnow    (dt/now)]
+   ::sm/params schema:delete-font
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
+  (let [team  (teams/get-team conn
+                              :profile-id profile-id
+                              :team-id team-id)
 
-                  (when-not (seq fonts)
-                    (ex/raise :type :not-found
-                              :code :object-not-found))
+        fonts (db/query conn :team-font-variant
+                        {:team-id team-id
+                         :font-id id
+                         :deleted-at nil}
+                        {::sql/for-update true})
 
-                  (doseq [font fonts]
-                    (db/update! conn :team-font-variant
-                                {:deleted-at tnow}
-                                {:id (:id font)}))
+        delay (ldel/get-deletion-delay team)
+        tnow  (dt/in-future delay)]
 
-                  (rph/with-meta (rph/wrap)
-                    {::audit/props {:id id
-                                    :team-id team-id
-                                    :name (:font-family (peek fonts))
-                                    :profile-id profile-id}})))))
+    (teams/check-edition-permissions! (:permissions team))
+
+    (when-not (seq fonts)
+      (ex/raise :type :not-found
+                :code :object-not-found))
+
+
+    (doseq [font fonts]
+      (db/update! conn :team-font-variant
+                  {:deleted-at tnow}
+                  {:id (:id font)}
+                  {::db/return-keys false}))
+
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:id id
+                      :team-id team-id
+                      :name (:font-family (peek fonts))
+                      :profile-id profile-id}})))
 
 ;; --- DELETE FONT VARIANT
 
@@ -239,19 +248,23 @@
 (sv/defmethod ::delete-font-variant
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font-variant}
-  [cfg {:keys [::rpc/profile-id id team-id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (teams/check-edition-permissions! conn profile-id team-id)
-                (let [variant (db/get conn :team-font-variant
-                                      {:id id :team-id team-id}
-                                      {::sql/for-update true})]
+   ::sm/params schema:delete-font-variant
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
+  (let [team    (teams/get-team conn
+                                :profile-id profile-id
+                                :team-id team-id)
+        variant (db/get conn :team-font-variant
+                        {:id id :team-id team-id}
+                        {::sql/for-update true})
+        delay   (ldel/get-deletion-delay team)]
 
-                  (db/update! conn :team-font-variant
-                              {:deleted-at (dt/now)}
-                              {:id (:id variant)})
+    (teams/check-edition-permissions! (:permissions team))
+    (db/update! conn :team-font-variant
+                {:deleted-at (dt/in-future delay)}
+                {:id (:id variant)}
+                {::db/return-keys false})
 
-                  (rph/with-meta (rph/wrap)
-                    {::audit/props {:font-family (:font-family variant)
-                                    :font-id (:font-id variant)}})))))
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:font-family (:font-family variant)
+                      :font-id (:font-id variant)}})))

backend/src/app/rpc/commands/management.clj

@@ -56,7 +56,7 @@
     (vswap! bfc/*state* update :index bfc/update-index fmeds :id)
 
     ;; Process and persist file
-    (let [file (bfc/process-file file)]
+    (let [file (bfc/process-file cfg file)]
       (bfc/insert-file! cfg file ::db/return-keys false)
 
       ;; The file profile creation is optional, so when no profile is
@@ -396,45 +396,49 @@
 ;; --- COMMAND: Clone Template
 
 (defn clone-template
-  [cfg {:keys [project-id profile-id] :as params} template]
-  (db/tx-run! cfg (fn [{:keys [::db/conn ::wrk/executor] :as cfg}]
-                    ;; NOTE: the importation process performs some operations
-                    ;; that are not very friendly with virtual threads, and for
-                    ;; avoid unexpected blocking of other concurrent operations
-                    ;; we dispatch that operation to a dedicated executor.
-                    (let [template (tmp/tempfile-from template
-                                                      :prefix "penpot.template."
-                                                      :suffix ""
-                                                      :min-age "30m")
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [project-id profile-id] :as params} template]
 
-                          format   (bfc/parse-file-format template)
-                          team     (teams/get-team conn
-                                                   :profile-id profile-id
-                                                   :project-id project-id)
-                          cfg      (-> cfg
-                                       (assoc ::bfc/project-id project-id)
-                                       (assoc ::bfc/profile-id profile-id)
-                                       (assoc ::bfc/input template)
-                                       (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
+  ;; NOTE: the importation process performs some operations
+  ;; that are not very friendly with virtual threads, and for
+  ;; avoid unexpected blocking of other concurrent operations
+  ;; we dispatch that operation to a dedicated executor.
+  (let [template (tmp/tempfile-from template
+                                    :prefix "penpot.template."
+                                    :suffix ""
+                                    :min-age "30m")
 
-                          result   (if (= format :binfile-v3)
-                                     (px/invoke! executor (partial bf.v3/import-files! cfg))
-                                     (px/invoke! executor (partial bf.v1/import-files! cfg)))]
+        format   (bfc/parse-file-format template)
+        team     (teams/get-team pool
+                                 :profile-id profile-id
+                                 :project-id project-id)
 
-                      (db/update! conn :project
-                                  {:modified-at (dt/now)}
-                                  {:id project-id})
+        cfg      (-> cfg
+                     (assoc ::bfc/project-id project-id)
+                     (assoc ::bfc/profile-id profile-id)
+                     (assoc ::bfc/input template)
+                     (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
 
-                      (let [props (audit/clean-props params)]
-                        (doseq [file-id result]
-                          (let [props (assoc props :id file-id)
-                                event (-> (audit/event-from-rpc-params params)
-                                          (assoc ::audit/profile-id profile-id)
-                                          (assoc ::audit/name "create-file")
-                                          (assoc ::audit/props props))]
-                            (audit/submit! cfg event))))
+        result   (if (= format :binfile-v3)
+                   (px/invoke! executor (partial bf.v3/import-files! cfg))
+                   (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
-                      result))))
+    (db/tx-run! cfg
+                (fn [{:keys [::db/conn] :as cfg}]
+                  (db/update! conn :project
+                              {:modified-at (dt/now)}
+                              {:id project-id}
+                              {::db/return-keys false})
+
+                  (let [props (audit/clean-props params)]
+                    (doseq [file-id result]
+                      (let [props (assoc props :id file-id)
+                            event (-> (audit/event-from-rpc-params params)
+                                      (assoc ::audit/profile-id profile-id)
+                                      (assoc ::audit/name "create-file")
+                                      (assoc ::audit/props props))]
+                        (audit/submit! cfg event))))))
+
+    result))
 
 (def ^:private
   schema:clone-template

backend/src/app/rpc/commands/media.clj

@@ -273,15 +273,14 @@
 
 (sv/defmethod ::clone-file-media-object
   {::doc/added "1.17"
-   ::sm/params schema:clone-file-media-object}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (-> (assoc cfg :conn conn)
-        (clone-file-media-object params))))
+   ::sm/params schema:clone-file-media-object
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (clone-file-media-object cfg params))
 
 (defn clone-file-media-object
-  [{:keys [conn]} {:keys [id file-id is-local]}]
+  [{:keys [::db/conn]} {:keys [id file-id is-local]}]
   (let [mobj (db/get-by-id conn :file-media-object id)]
     (db/insert! conn :file-media-object
                 {:id (uuid/next)

backend/src/app/rpc/commands/profile.clj

@@ -125,32 +125,32 @@
 (sv/defmethod ::update-profile
   {::doc/added "1.0"
    ::sm/params schema:update-profile
-   ::sm/result schema:profile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id fullname lang theme] :as params}]
-  (db/with-atomic [conn pool]
-    ;; NOTE: we need to retrieve the profile independently if we use
-    ;; it or not for explicit locking and avoid concurrent updates of
-    ;; the same row/object.
-    (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
-                      (decode-row))
+   ::sm/result schema:profile
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id fullname lang theme] :as params}]
+  ;; NOTE: we need to retrieve the profile independently if we use
+  ;; it or not for explicit locking and avoid concurrent updates of
+  ;; the same row/object.
+  (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
+                    (decode-row))
 
-          ;; Update the profile map with direct params
-          profile (-> profile
-                      (assoc :fullname fullname)
-                      (assoc :lang lang)
-                      (assoc :theme theme))]
+        ;; Update the profile map with direct params
+        profile (-> profile
+                    (assoc :fullname fullname)
+                    (assoc :lang lang)
+                    (assoc :theme theme))]
 
-      (db/update! conn :profile
-                  {:fullname fullname
-                   :lang lang
-                   :theme theme
-                   :props (db/tjson (:props profile))}
-                  {:id profile-id})
+    (db/update! conn :profile
+                {:fullname fullname
+                 :lang lang
+                 :theme theme
+                 :props (db/tjson (:props profile))}
+                {:id profile-id})
 
-      (-> profile
-          (strip-private-attrs)
-          (d/without-nils)
-          (rph/with-meta {::audit/props (audit/profile->props profile)})))))
+    (-> profile
+        (strip-private-attrs)
+        (d/without-nils)
+        (rph/with-meta {::audit/props (audit/profile->props profile)}))))
 
 
 ;; --- MUTATION: Update Password
@@ -169,21 +169,20 @@
 (sv/defmethod ::update-profile-password
   {::doc/added "1.0"
    ::sm/params schema:update-profile-password
-   ::climit/id :auth/global}
+   ::climit/id :auth/global
+   ::db/transaction true}
   [cfg {:keys [::rpc/profile-id password] :as params}]
+  (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
+        session-id (::session/id params)]
 
-  (db/tx-run! cfg (fn [cfg]
-                    (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
-                          session-id (::session/id params)]
+    (when (= (:email profile) (str/lower (:password params)))
+      (ex/raise :type :validation
+                :code :email-as-password
+                :hint "you can't use your email as password"))
 
-                      (when (= (:email profile) (str/lower (:password params)))
-                        (ex/raise :type :validation
-                                  :code :email-as-password
-                                  :hint "you can't use your email as password"))
-
-                      (update-profile-password! cfg (assoc profile :password password))
-                      (invalidate-profile-session! cfg profile-id session-id)
-                      nil))))
+    (update-profile-password! cfg (assoc profile :password password))
+    (invalidate-profile-session! cfg profile-id session-id)
+    nil))
 
 (defn- invalidate-profile-session!
   "Removes all sessions except the current one."
@@ -441,37 +440,36 @@
 (declare ^:private get-owned-teams)
 
 (sv/defmethod ::delete-profile
-  {::doc/added "1.0"}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [teams      (get-owned-teams conn profile-id)
-          deleted-at (dt/now)]
+  {::doc/added "1.0"
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (let [teams      (get-owned-teams conn profile-id)
+        deleted-at (dt/now)]
 
-      ;; If we found owned teams with participants, we don't allow
-      ;; delete profile until the user properly transfer ownership or
-      ;; explicitly removes all participants from the team
-      (when (some pos? (map :participants teams))
-        (ex/raise :type :validation
-                  :code :owner-teams-with-people
-                  :hint "The user need to transfer ownership of owned teams."
-                  :context {:teams (mapv :id teams)}))
+    ;; If we found owned teams with participants, we don't allow
+    ;; delete profile until the user properly transfer ownership or
+    ;; explicitly removes all participants from the team
+    (when (some pos? (map :participants teams))
+      (ex/raise :type :validation
+                :code :owner-teams-with-people
+                :hint "The user need to transfer ownership of owned teams."
+                :context {:teams (mapv :id teams)}))
 
-      ;; Mark profile deleted immediatelly
-      (db/update! conn :profile
-                  {:deleted-at deleted-at}
-                  {:id profile-id})
+    ;; Mark profile deleted immediatelly
+    (db/update! conn :profile
+                {:deleted-at deleted-at}
+                {:id profile-id})
 
-      ;; Schedule cascade deletion to a worker
-      (wrk/submit! {::db/conn conn
-                    ::wrk/task :delete-object
-                    ::wrk/params {:object :profile
-                                  :deleted-at deleted-at
-                                  :id profile-id}})
+    ;; Schedule cascade deletion to a worker
+    (wrk/submit! {::db/conn conn
+                  ::wrk/task :delete-object
+                  ::wrk/params {:object :profile
+                                :deleted-at deleted-at
+                                :id profile-id}})
 
 
-      (-> (rph/wrap nil)
-          (rph/with-transform (session/delete-fn cfg))))))
-
+    (-> (rph/wrap nil)
+        (rph/with-transform (session/delete-fn cfg)))))
 
 ;; --- HELPERS
 
@@ -482,8 +480,7 @@
         JOIN team AS t ON (t.id = tpr.team_id)
        WHERE tpr.is_owner IS TRUE
          AND tpr.profile_id = ?
-         AND (t.deleted_at IS NULL OR
-              t.deleted_at > now())
+         AND t.deleted_at IS NULL
    )
    SELECT tpr.team_id AS id,
           count(tpr.profile_id) - 1 AS participants

backend/src/app/rpc/commands/projects.clj

@@ -11,6 +11,7 @@
    [app.common.schema :as sm]
    [app.db :as db]
    [app.db.sql :as-alias sql]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as webhooks]
    [app.rpc :as-alias rpc]
@@ -219,12 +220,12 @@
    ::sm/params schema:update-project-pin
    ::webhooks/batch-timeout (dt/duration "5s")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id team-id is-pinned] :as params}]
-  (db/with-atomic [conn pool]
-    (check-read-permissions! conn profile-id id)
-    (db/exec-one! conn [sql:update-project-pin team-id id profile-id is-pinned is-pinned])
-    nil))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id team-id is-pinned] :as params}]
+  (check-read-permissions! conn profile-id id)
+  (db/exec-one! conn [sql:update-project-pin team-id id profile-id is-pinned is-pinned])
+  nil)
 
 ;; --- MUTATION: Rename Project
 
@@ -238,24 +239,25 @@
 (sv/defmethod ::rename-project
   {::doc/added "1.18"
    ::sm/params schema:rename-project
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [project (db/get-by-id conn :project id ::sql/for-update true)]
-      (db/update! conn :project
-                  {:name name}
-                  {:id id})
-      (rph/with-meta (rph/wrap)
-        {::audit/props {:team-id (:team-id project)
-                        :prev-name (:name project)}}))))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id name] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [project (db/get-by-id conn :project id ::sql/for-update true)]
+    (db/update! conn :project
+                {:name name}
+                {:id id})
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:team-id (:team-id project)
+                      :prev-name (:name project)}})))
 
 ;; --- MUTATION: Delete Project
 
 (defn- delete-project
-  [conn project-id]
-  (let [project (db/update! conn :project
-                            {:deleted-at (dt/now)}
+  [conn team project-id]
+  (let [delay   (ldel/get-deletion-delay team)
+        project (db/update! conn :project
+                            {:deleted-at (dt/in-future delay)}
                             {:id project-id}
                             {::db/return-keys true})]
 
@@ -272,7 +274,6 @@
 
     project))
 
-
 (def ^:private schema:delete-project
   [:map {:title "delete-project"}
    [:id ::sm/uuid]])
@@ -280,13 +281,16 @@
 (sv/defmethod ::delete-project
   {::doc/added "1.18"
    ::sm/params schema:delete-project
-   ::webhooks/event? true}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (let [project (delete-project conn id)]
-      (rph/with-meta (rph/wrap)
-        {::audit/props {:team-id (:team-id project)
-                        :name (:name project)
-                        :created-at (:created-at project)
-                        :modified-at (:modified-at project)}}))))
+   ::webhooks/event? true
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
+  (check-edition-permissions! conn profile-id id)
+  (let [team    (teams/get-team conn
+                                :profile-id profile-id
+                                :project-id id)
+        project (delete-project conn team id)]
+    (rph/with-meta (rph/wrap)
+      {::audit/props {:team-id (:team-id project)
+                      :name (:name project)
+                      :created-at (:created-at project)
+                      :modified-at (:modified-at project)}})))

backend/src/app/rpc/commands/teams.clj

@@ -17,6 +17,7 @@
    [app.db :as db]
    [app.db.sql :as sql]
    [app.email :as eml]
+   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.media :as media]
@@ -76,9 +77,10 @@
   (perms/make-check-fn has-read-permissions?))
 
 (defn decode-row
-  [{:keys [features] :as row}]
+  [{:keys [features subscription] :as row}]
   (cond-> row
-    (some? features) (assoc :features (db/decode-pgarray features #{}))))
+    (some? features) (assoc :features (db/decode-pgarray features #{}))
+    (some? subscription) (assoc :subscription (db/decode-transit-pgobject subscription))))
 
 ;; FIXME: move
 
@@ -113,29 +115,42 @@
 
 ;; --- Query: Teams
 
-(declare get-teams)
-
-(def ^:private schema:get-teams
-  [:map {:title "get-teams"}])
-
-(sv/defmethod ::get-teams
-  {::doc/added "1.17"
-   ::sm/params schema:get-teams}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (dm/with-open [conn (db/open pool)]
-    (get-teams conn profile-id)))
-
 (def sql:get-teams-with-permissions
-  "select t.*,
+  "SELECT t.*,
           tp.is_owner,
           tp.is_admin,
           tp.can_edit,
-          (t.id = ?) as is_default
-     from team_profile_rel as tp
-     join team as t on (t.id = tp.team_id)
-    where t.deleted_at is null
-      and tp.profile_id = ?
-    order by tp.created_at asc")
+          (t.id = ?) AS is_default
+     FROM team_profile_rel AS tp
+     JOIN team AS t ON (t.id = tp.team_id)
+    WHERE t.deleted_at IS null
+      AND tp.profile_id = ?
+    ORDER BY tp.created_at ASC")
+
+(def sql:get-teams-with-permissions-and-subscription
+  "SELECT t.*,
+          tp.is_owner,
+          tp.is_admin,
+          tp.can_edit,
+          (t.id = ?) AS is_default,
+
+          jsonb_build_object(
+            '~:type', COALESCE(p.props->'~:subscription'->>'~:type', 'professional'),
+            '~:status', CASE COALESCE(p.props->'~:subscription'->>'~:type', 'professional')
+                          WHEN 'professional' THEN 'active'
+                          ELSE COALESCE(p.props->'~:subscription'->>'~:status', 'incomplete')
+                       END,
+            '~:seats', p.props->'~:subscription'->'~:quantity'
+          ) AS subscription
+     FROM team_profile_rel AS tp
+     JOIN team AS t ON (t.id = tp.team_id)
+     JOIN team_profile_rel AS tpr
+       ON (tpr.team_id = t.id AND tpr.is_owner IS true)
+     JOIN profile AS p
+       ON (tpr.profile_id = p.id)
+    WHERE t.deleted_at IS null
+      AND tp.profile_id = ?
+    ORDER BY tp.created_at ASC")
 
 (defn process-permissions
   [team]
@@ -150,13 +165,53 @@
         (dissoc :is-owner :is-admin :can-edit)
         (assoc :permissions permissions))))
 
+(def ^:private
+  xform:process-teams
+  (comp
+   (map decode-row)
+   (map process-permissions)))
+
 (defn get-teams
   [conn profile-id]
-  (let [profile (profile/get-profile conn profile-id)]
-    (->> (db/exec! conn [sql:get-teams-with-permissions (:default-team-id profile) profile-id])
-         (map decode-row)
-         (map process-permissions)
-         (vec))))
+  (let [profile (profile/get-profile conn profile-id)
+        sql     (if (contains? cf/flags :subscriptions)
+                  sql:get-teams-with-permissions-and-subscription
+                  sql:get-teams-with-permissions)]
+
+    (->> (db/exec! conn [sql (:default-team-id profile) profile-id])
+         (into [] xform:process-teams))))
+
+(def ^:private schema:get-teams
+  [:map {:title "get-teams"}])
+
+(sv/defmethod ::get-teams
+  {::doc/added "1.17"
+   ::sm/params schema:get-teams}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (get-teams conn profile-id)))
+
+(def ^:private sql:get-owned-teams
+  "SELECT t.id, t.name,
+          (SELECT count(*) FROM team_profile_rel WHERE team_id=t.id) AS total_members,
+          (SELECT count(*) FROM team_profile_rel WHERE team_id=t.id AND can_edit=true) AS total_editors
+     FROM team AS t
+     JOIN team_profile_rel AS tpr ON (tpr.team_id = t.id)
+    WHERE t.is_default IS false
+      AND tpr.is_owner IS true
+      AND tpr.profile_id = ?
+      AND t.deleted_at IS NULL")
+
+(defn- get-owned-teams
+  [cfg profile-id]
+  (->> (db/exec! cfg [sql:get-owned-teams profile-id])
+       (into [] (map decode-row))))
+
+(sv/defmethod ::get-owned-teams
+  {::doc/added "2.8.0"
+   ::sm/params schema:get-teams}
+  [cfg {:keys [::rpc/profile-id]}]
+  (get-owned-teams cfg profile-id))
 
 ;; --- Query: Team (by ID)
 
@@ -181,39 +236,43 @@
 (defn get-team
   [conn & {:keys [profile-id team-id project-id file-id] :as params}]
 
-  (dm/assert!
-   "connection or pool is mandatory"
-   (or (db/connection? conn)
-       (db/pool? conn)))
+  (assert (uuid? profile-id) "profile-id is mandatory")
+  (assert (or (db/connection? conn)
+              (db/pool? conn))
+          "connection or pool is mandatory")
 
-  (dm/assert!
-   "profile-id is mandatory"
-   (uuid? profile-id))
+  (let [{:keys [default-team-id] :as profile}
+        (profile/get-profile conn profile-id)
 
-  (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
-        result (cond
-                 (some? team-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
-                                ") SELECT * FROM teams WHERE id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id team-id]))
+        sql
+        (if (contains? cf/flags :subscriptions)
+          sql:get-teams-with-permissions-and-subscription
+          sql:get-teams-with-permissions)
 
-                 (some? project-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                " WHERE p.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id project-id]))
+        result
+        (cond
+          (some? team-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT * FROM teams WHERE id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id team-id]))
 
-                 (some? file-id)
-                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
-                                "SELECT t.* FROM teams AS t "
-                                "  JOIN project AS p ON (p.team_id = t.id) "
-                                "  JOIN file AS f ON (f.project_id = p.id) "
-                                " WHERE f.id=?")]
-                   (db/exec-one! conn [sql default-team-id profile-id file-id]))
+          (some? project-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT t.* FROM teams AS t "
+                         "  JOIN project AS p ON (p.team_id = t.id) "
+                         " WHERE p.id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id project-id]))
 
-                 :else
-                 (throw (IllegalArgumentException. "invalid arguments")))]
+          (some? file-id)
+          (let [sql (str "WITH teams AS (" sql ") "
+                         "SELECT t.* FROM teams AS t "
+                         "  JOIN project AS p ON (p.team_id = t.id) "
+                         "  JOIN file AS f ON (f.project_id = p.id) "
+                         " WHERE f.id=?")]
+            (db/exec-one! conn [sql default-team-id profile-id file-id]))
+
+          :else
+          (throw (IllegalArgumentException. "invalid arguments")))]
 
     (when-not result
       (ex/raise :type :not-found
@@ -527,14 +586,14 @@
 
 (sv/defmethod ::update-team
   {::doc/added "1.17"
-   ::sm/params schema:update-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id name] :as params}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id id)
-    (db/update! conn :team
-                {:name name}
-                {:id id})
-    nil))
+   ::sm/params schema:update-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id name]}]
+  (check-edition-permissions! conn profile-id id)
+  (db/update! conn :team
+              {:name name}
+              {:id id})
+  nil)
 
 
 ;; --- Mutation: Leave Team
@@ -592,22 +651,22 @@
 
 (sv/defmethod ::leave-team
   {::doc/added "1.17"
-   ::sm/params schema:leave-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (leave-team conn (assoc params :profile-id profile-id))))
+   ::sm/params schema:leave-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (leave-team conn (assoc params :profile-id profile-id)))
 
 ;; --- Mutation: Delete Team
 
 (defn- delete-team
   "Mark a team for deletion"
-  [conn team-id]
+  [conn {:keys [id] :as team}]
 
-  (let [deleted-at (dt/now)
-        team       (db/update! conn :team
-                               {:deleted-at deleted-at}
-                               {:id team-id}
-                               {::db/return-keys true})]
+  (let [delay (ldel/get-deletion-delay team)
+        team  (db/update! conn :team
+                          {:deleted-at (dt/in-future delay)}
+                          {:id id}
+                          {::db/return-keys true})]
 
     (when (:is-default team)
       (ex/raise :type :validation
@@ -617,8 +676,8 @@
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :team
-                                :deleted-at deleted-at
-                                :id team-id}})
+                                :deleted-at (:deleted-at team)
+                                :id id}})
     team))
 
 (def ^:private schema:delete-team
@@ -627,16 +686,18 @@
 
 (sv/defmethod ::delete-team
   {::doc/added "1.17"
-   ::sm/params schema:delete-team}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (get-permissions conn profile-id id)]
-      (when-not (:is-owner perms)
-        (ex/raise :type :validation
-                  :code :only-owner-can-delete-team))
+   ::sm/params schema:delete-team
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
+  (let [team  (get-team conn :profile-id profile-id :team-id id)
+        perms (get team :permissions)]
 
-      (delete-team conn id)
-      nil)))
+    (when-not (:is-owner perms)
+      (ex/raise :type :validation
+                :code :only-owner-can-delete-team))
+
+    (delete-team conn team)
+    nil))
 
 ;; --- Mutation: Team Update Role
 
@@ -714,31 +775,30 @@
 
 (sv/defmethod ::delete-team-member
   {::doc/added "1.17"
-   ::sm/params schema:delete-team-member}
-  [{:keys [::db/pool ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id team-id member-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [team  (get-team pool :profile-id profile-id :team-id team-id)
-          perms (get-permissions conn profile-id team-id)]
-      (when-not (or (:is-owner perms)
-                    (:is-admin perms))
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+   ::sm/params schema:delete-team-member
+   ::db/transaction true}
+  [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id team-id member-id] :as params}]
+  (let [team  (get-team conn :profile-id profile-id :team-id team-id)
+        perms (get-permissions conn profile-id team-id)]
+    (when-not (or (:is-owner perms)
+                  (:is-admin perms))
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (when (= member-id profile-id)
-        (ex/raise :type :validation
-                  :code :cant-remove-yourself))
+    (when (= member-id profile-id)
+      (ex/raise :type :validation
+                :code :cant-remove-yourself))
 
-      (db/delete! conn :team-profile-rel {:profile-id member-id
-                                          :team-id team-id})
+    (db/delete! conn :team-profile-rel {:profile-id member-id
+                                        :team-id team-id})
+    (mbus/pub! msgbus
+               :topic member-id
+               :message {:type :team-membership-change
+                         :change :removed
+                         :team-id team-id
+                         :team-name (:name team)})
 
-      (mbus/pub! msgbus
-                 :topic member-id
-                 :message {:type :team-membership-change
-                           :change :removed
-                           :team-id team-id
-                           :team-name (:name team)})
-
-      nil)))
+    nil))
 
 ;; --- Mutation: Update Team Photo
 
@@ -764,16 +824,16 @@
   (let [team  (get-team pool :profile-id profile-id :team-id team-id)
         photo (profile/upload-photo cfg params)]
 
-    (db/with-atomic [conn pool]
-      (check-admin-permissions! conn profile-id team-id)
-      ;; Mark object as touched for make it ellegible for tentative
-      ;; garbage collection.
-      (when-let [id (:photo-id team)]
-        (sto/touch-object! storage id))
+    (check-admin-permissions! pool profile-id team-id)
 
-      ;; Save new photo
-      (db/update! pool :team
-                  {:photo-id (:id photo)}
-                  {:id team-id})
+    ;; Mark object as touched for make it ellegible for tentative
+    ;; garbage collection.
+    (when-let [id (:photo-id team)]
+      (sto/touch-object! storage id))
 
-      (assoc team :photo-id (:id photo)))))
+    ;; Save new photo
+    (db/update! pool :team
+                {:photo-id (:id photo)}
+                {:id team-id})
+
+    (assoc team :photo-id (:id photo))))

backend/src/app/rpc/commands/teams_invitations.clj

@@ -408,20 +408,20 @@
 (sv/defmethod ::update-team-invitation-role
   {::doc/added "1.17"
    ::doc/module :teams
-   ::sm/params schema:update-team-invitation-role}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id email role] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (teams/get-permissions conn profile-id team-id)]
+   ::sm/params schema:update-team-invitation-role
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email role] :as params}]
+  (let [perms (teams/get-permissions conn profile-id team-id)]
 
-      (when-not (:is-admin perms)
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+    (when-not (:is-admin perms)
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (db/update! conn :team-invitation
-                  {:role (name role) :updated-at (dt/now)}
-                  {:team-id team-id :email-to (profile/clean-email email)})
+    (db/update! conn :team-invitation
+                {:role (name role) :updated-at (dt/now)}
+                {:team-id team-id :email-to (profile/clean-email email)})
 
-      nil)))
+    nil))
 
 ;; --- Mutation: Delete invitation
 
@@ -432,20 +432,20 @@
 
 (sv/defmethod ::delete-team-invitation
   {::doc/added "1.17"
-   ::sm/params schema:delete-team-invition}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id email] :as params}]
-  (db/with-atomic [conn pool]
-    (let [perms (teams/get-permissions conn profile-id team-id)]
+   ::sm/params schema:delete-team-invition
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id email] :as params}]
+  (let [perms (teams/get-permissions conn profile-id team-id)]
 
-      (when-not (:is-admin perms)
-        (ex/raise :type :validation
-                  :code :insufficient-permissions))
+    (when-not (:is-admin perms)
+      (ex/raise :type :validation
+                :code :insufficient-permissions))
 
-      (let [invitation (db/delete! conn :team-invitation
-                                   {:team-id team-id
-                                    :email-to (profile/clean-email email)}
-                                   {::db/return-keys true})]
-        (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}})))))
+    (let [invitation (db/delete! conn :team-invitation
+                                 {:team-id team-id
+                                  :email-to (profile/clean-email email)}
+                                 {::db/return-keys true})]
+      (rph/wrap nil {::audit/props {:invitation-id (:id invitation)}}))))
 
 
 ;; --- Mutation: Request Team Invitation

backend/src/app/rpc/commands/viewer.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.viewer
   (:require
+   [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
@@ -78,7 +79,10 @@
                   :always
                   (update :data select-keys [:id :options :pages :pages-index :components]))
 
-        libs    (files/get-file-libraries conn file-id)
+        libs    (->> (bfc/get-file-libraries conn file-id)
+                     (mapv (fn [{:keys [id] :as lib}]
+                             (merge lib (files/get-file cfg id)))))
+
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]
                              (-> row

backend/src/app/rpc/commands/webhooks.clj

@@ -144,20 +144,20 @@
 
 (sv/defmethod ::delete-webhook
   {::doc/added "1.17"
-   ::sm/params schema:delete-webhook}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
-  (db/with-atomic [conn pool]
-    (let [whook (-> (db/get conn :webhook {:id id}) decode-row)]
-      (check-webhook-edition-permissions! conn profile-id (:team-id whook) (:profile-id whook))
-      (db/delete! conn :webhook {:id id})
-      nil)))
+   ::sm/params schema:delete-webhook
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (let [whook (-> (db/get conn :webhook {:id id}) decode-row)]
+    (check-webhook-edition-permissions! conn profile-id (:team-id whook) (:profile-id whook))
+    (db/delete! conn :webhook {:id id})
+    nil))
 
 ;; --- Query: Webhooks
 
 (def sql:get-webhooks
-  "SELECT id, uri, mtype, is_active, error_code, error_count, profile_id 
-     FROM webhook 
-    WHERE team_id = ? 
+  "SELECT id, uri, mtype, is_active, error_code, error_count, profile_id
+     FROM webhook
+    WHERE team_id = ?
     ORDER BY uri")
 
 (def ^:private schema:get-webhooks

backend/src/app/rpc/doc.clj

@@ -9,6 +9,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.json :as json]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as smdj]
@@ -19,7 +20,6 @@
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
-   [app.util.json :as json]
    [app.util.services :as sv]
    [app.util.template :as tmpl]
    [clojure.java.io :as io]
@@ -86,7 +86,7 @@
     (fn [request]
       (let [params  (:query-params request)
             pstyle  (:type params "js")
-            context (assoc context :param-style pstyle)]
+            context (assoc @context :param-style pstyle)]
 
         {::yres/status 200
          ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
@@ -178,8 +178,7 @@
     (fn [_]
       {::yres/status 200
        ::yres/headers {"content-type" "application/json; charset=utf-8"}
-       ::yres/body (json/encode context)})
-
+       ::yres/body (json/encode @context)})
     (fn [_]
       {::yres/status 404})))
 
@@ -209,7 +208,7 @@
 
 (defmethod ig/init-key ::routes
   [_ {:keys [::rpc/methods] :as cfg}]
-  [(let [context (prepare-doc-context methods)]
+  [(let [context (delay (prepare-doc-context methods))]
      [["/_doc"
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]
@@ -217,7 +216,7 @@
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]])
 
-   (let [context (prepare-openapi-context methods)]
+   (let [context (delay (prepare-openapi-context methods))]
      [["/openapi"
        {:handler (openapi-handler)
         :allowed-methods #{:get}}]

backend/src/app/setup.clj

@@ -78,19 +78,19 @@
 
 (defmethod ig/init-key ::props
   [_ {:keys [::db/pool ::key] :as cfg}]
-  (db/with-atomic [conn pool]
-    (db/xact-lock! conn 0)
-    (when-not key
-      (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
-                         "all sessions on each restart, it is highly recommended setting up the "
-                         "PENPOT_SECRET_KEY environment variable")))
 
-    (let [secret (or key (generate-random-key))]
-      (-> (get-all-props conn)
-          (assoc :secret-key secret)
-          (assoc :tokens-key (keys/derive secret :salt "tokens"))
-          (update :instance-id handle-instance-id conn (db/read-only? pool))))))
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (db/xact-lock! conn 0)
+                    (when-not key
+                      (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
+                                         "all sessions on each restart, it is highly recommended setting up the "
+                                         "PENPOT_SECRET_KEY environment variable")))
 
+                    (let [secret (or key (generate-random-key))]
+                      (-> (get-all-props conn)
+                          (assoc :secret-key secret)
+                          (assoc :tokens-key (keys/derive secret :salt "tokens"))
+                          (update :instance-id handle-instance-id conn (db/read-only? pool)))))))
 
 ;; FIXME
 (sm/register! ::props :any)

backend/src/app/srepl.clj

@@ -6,13 +6,17 @@
 
 (ns app.srepl
   "Server Repl."
+  (:refer-clojure :exclude [read-line])
   (:require
+   [app.common.exceptions :as ex]
+   [app.common.json :as json]
    [app.common.logging :as l]
    [app.config :as cf]
-   [app.srepl.cli]
+   [app.srepl.cli :as cli]
    [app.srepl.main]
-   [app.util.json :as json]
    [app.util.locks :as locks]
+   [app.util.time :as dt]
+   [clojure.core :as c]
    [clojure.core.server :as ccs]
    [clojure.main :as cm]
    [integrant.core :as ig]))
@@ -28,17 +32,80 @@
    :init repl-init
    :read ccs/repl-read))
 
+(defn- ex->data
+  [cause phase]
+  (let [data (ex-data cause)
+        explain (ex/explain data)]
+    (cond-> {:phase phase
+             :code (get data :code :unknown)
+             :type (get data :type :unknown)
+             :hint (or (get data :hint) (ex-message cause))}
+      (some? explain)
+      (assoc :explain explain))))
+
+(defn read-line
+  []
+  (if-let [line (c/read-line)]
+    (try
+      (l/dbg :hint "decode" :data line)
+      (json/decode line :key-fn json/read-kebab-key)
+      (catch Throwable _cause
+        (l/warn :hint "unable to decode data" :data line)
+        nil))
+    ::eof))
+
 (defn json-repl
   []
-  (let [out  *out*
-        lock (locks/create)]
-    (ccs/prepl *in*
-               (fn [m]
-                 (binding [*out* out,
-                           *flush-on-newline* true,
-                           *print-readably* true]
-                   (locks/locking lock
-                     (println (json/encode-str m))))))))
+  (let [lock (locks/create)
+        out  *out*
+
+        out-fn
+        (fn [m]
+          (locks/locking lock
+            (binding [*out* out]
+              (l/warn :hint "write" :data m)
+              (println (json/encode m :key-fn json/write-camel-key)))))
+
+        tapfn
+        (fn [val]
+          (out-fn {:tag :tap :val val}))]
+
+    (binding [*out* (PrintWriter-on #(out-fn {:tag :out :val %1}) nil true)
+              *err* (PrintWriter-on #(out-fn {:tag :err :val %1}) nil true)]
+      (try
+        (add-tap tapfn)
+        (loop []
+          (when (try
+                  (let [data   (read-line)
+                        tpoint (dt/tpoint)]
+
+                    (l/dbg :hint "received" :data (if (= data ::eof) "EOF" data))
+
+                    (try
+                      (when-not (= data ::eof)
+                        (when-not (nil? data)
+                          (let [result  (cli/exec data)
+                                elapsed (tpoint)]
+                            (l/warn :hint "result" :data result)
+                            (out-fn {:tag :ret
+                                     :val (if (instance? Throwable result)
+                                            (Throwable->map result)
+                                            result)
+                                     :elapsed (inst-ms elapsed)})))
+                        true)
+                      (catch Throwable cause
+                        (let [elapsed (tpoint)]
+                          (out-fn {:tag :ret
+                                   :err (ex->data cause :eval)
+                                   :elapsed (inst-ms elapsed)})
+                          true))))
+                  (catch Throwable cause
+                    (out-fn {:tag :ret
+                             :err (ex->data cause :read)})
+                    true))
+            (recur)))
+        (finally
+          (remove-tap tapfn))))))
 
 ;; --- State initialization
 

backend/src/app/srepl/cli.clj

@@ -9,14 +9,23 @@
   (:require
    [app.auth :as auth]
    [app.common.exceptions :as ex]
+   [app.common.schema :as sm]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.rpc.commands.auth :as cmd.auth]
    [app.rpc.commands.profile :as cmd.profile]
-   [app.util.json :as json]
+   [app.setup :as-alias setup]
+   [app.tokens :as tokens]
    [app.util.time :as dt]
    [cuerdas.core :as str]))
 
+(defn coercer
+  [schema & {:as opts}]
+  (let [decode-fn (sm/decoder schema sm/json-transformer)
+        check-fn (sm/check-fn schema opts)]
+    (fn [data]
+      (-> data decode-fn check-fn))))
+
 (defn- get-current-system
   []
   (or (deref (requiring-resolve 'app.main/system))
@@ -24,88 +33,218 @@
 
 (defmulti ^:private exec-command ::cmd)
 
+(defmethod exec-command :default
+  [{:keys [::cmd]}]
+  (ex/raise :type :internal
+            :code :not-implemented
+            :hint (str/ffmt "command '%' not implemented" cmd)))
+
 (defn exec
   "Entry point with external tools integrations that uses PREPL
   interface for interacting with running penpot backend."
   [data]
-  (let [data (json/decode data)]
-    (-> {::cmd (keyword (:cmd data "default"))}
-        (merge (:params data))
-        (exec-command))))
+  (-> {::cmd (get data :cmd)}
+      (merge (:params data))
+      (exec-command)))
 
-(defmethod exec-command :create-profile
+(defmethod exec-command "create-profile"
   [{:keys [fullname email password is-active]
     :or {is-active true}}]
-  (when-let [system (get-current-system)]
-    (db/with-atomic [conn (:app.db/pool system)]
-      (let [password (cmd.profile/derive-password system password)
-            params   {:id (uuid/next)
-                      :email email
-                      :fullname fullname
-                      :is-active is-active
-                      :password password
-                      :props {}}]
-        (->> (cmd.auth/create-profile! conn params)
-             (cmd.auth/create-profile-rels! conn))))))
+  (some-> (get-current-system)
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (let [password (cmd.profile/derive-password system password)
+                   params   {:id (uuid/next)
+                             :email email
+                             :fullname fullname
+                             :is-active is-active
+                             :password password
+                             :props {}}]
+               (->> (cmd.auth/create-profile! conn params)
+                    (cmd.auth/create-profile-rels! conn)))))))
 
-(defmethod exec-command :update-profile
+(defmethod exec-command "update-profile"
   [{:keys [fullname email password is-active]}]
-  (when-let [system (get-current-system)]
-    (db/with-atomic [conn (:app.db/pool system)]
-      (let [params (cond-> {}
-                     (some? fullname)
-                     (assoc :fullname fullname)
+  (some-> (get-current-system)
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (let [params (cond-> {}
+                            (some? fullname)
+                            (assoc :fullname fullname)
 
-                     (some? password)
-                     (assoc :password (auth/derive-password password))
+                            (some? password)
+                            (assoc :password (auth/derive-password password))
 
-                     (some? is-active)
-                     (assoc :is-active is-active))]
-        (when (seq params)
-          (let [res (db/update! conn :profile
-                                params
-                                {:email email
-                                 :deleted-at nil})]
-            (pos? (db/get-update-count res))))))))
+                            (some? is-active)
+                            (assoc :is-active is-active))]
+               (when (seq params)
+                 (let [res (db/update! conn :profile
+                                       params
+                                       {:email email
+                                        :deleted-at nil})]
+                   (pos? (db/get-update-count res)))))))))
 
-(defmethod exec-command :delete-profile
+(defmethod exec-command "echo"
+  [params]
+  params)
+
+
+(defmethod exec-command "delete-profile"
   [{:keys [email soft]}]
   (when-not email
     (ex/raise :type :assertion
               :code :invalid-arguments
               :hint "email should be provided"))
 
-  (when-let [system (get-current-system)]
-    (db/with-atomic [conn (:app.db/pool system)]
+  (some-> (get-current-system)
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (let [res (if soft
+                         (db/update! conn :profile
+                                     {:deleted-at (dt/now)}
+                                     {:email email :deleted-at nil})
+                         (db/delete! conn :profile
+                                     {:email email}))]
+               (pos? (db/get-update-count res)))))))
 
-      (let [res (if soft
-                  (db/update! conn :profile
-                              {:deleted-at (dt/now)}
-                              {:email email :deleted-at nil})
-                  (db/delete! conn :profile
-                              {:email email}))]
-        (pos? (db/get-update-count res))))))
-
-(defmethod exec-command :search-profile
+(defmethod exec-command "search-profile"
   [{:keys [email]}]
   (when-not email
     (ex/raise :type :assertion
               :code :invalid-arguments
               :hint "email should be provided"))
 
-  (when-let [system (get-current-system)]
-    (db/with-atomic [conn (:app.db/pool system)]
+  (some-> (get-current-system)
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (let [sql (str "select email, fullname, created_at, deleted_at from profile "
+                            " where email similar to ? order by created_at desc limit 100")]
+               (db/exec! conn [sql email]))))))
 
-      (let [sql (str "select email, fullname, created_at, deleted_at from profile "
-                     " where email similar to ? order by created_at desc limit 100")]
-        (db/exec! conn [sql email])))))
-
-(defmethod exec-command :derive-password
+(defmethod exec-command "derive-password"
   [{:keys [password]}]
   (auth/derive-password password))
 
-(defmethod exec-command :default
-  [{:keys [::cmd]}]
-  (ex/raise :type :internal
-            :code :not-implemented
-            :hint (str/ffmt "command '%' not implemented" (name cmd))))
+(defmethod exec-command "authenticate"
+  [{:keys [token]}]
+  (when-let [system (get-current-system)]
+    (let [props  (get system ::setup/props)]
+      (tokens/verify props {:token token :iss "authentication"}))))
+
+(def ^:private schema:get-customer
+  [:map [:id ::sm/uuid]])
+
+(def coerce-get-customer-params
+  (coercer schema:get-customer
+           :type :validation
+           :hint "invalid data provided for `get-customer` rpc call"))
+
+(def sql:get-customer-slots
+  "WITH teams AS (
+    SELECT tpr.team_id AS id,
+           tpr.profile_id AS profile_id
+      FROM team_profile_rel AS tpr
+     WHERE tpr.is_owner IS true
+       AND tpr.profile_id = ?
+  ), teams_with_slots AS (
+    SELECT tpr.team_id AS id,
+           count(*) AS total
+      FROM team_profile_rel AS tpr
+     WHERE tpr.team_id IN (SELECT id FROM teams)
+       AND tpr.can_edit IS true
+     GROUP BY 1
+     ORDER BY 2
+  )
+  SELECT max(total) AS total FROM teams_with_slots;")
+
+(defn- get-customer-slots
+  [system profile-id]
+  (let [result (db/exec-one! system [sql:get-customer-slots profile-id])]
+    (:total result)))
+
+(defmethod exec-command "get-customer"
+  [params]
+  (when-let [system (get-current-system)]
+    (let [{:keys [id] :as params} (coerce-get-customer-params params)
+          {:keys [props] :as profile} (cmd.profile/get-profile system id)]
+      {:id (get profile :id)
+       :name (get profile :fullname)
+       :email (get profile :email)
+       :num-editors (get-customer-slots system id)
+       :subscription (get props :subscription)})))
+
+(def ^:private schema:customer-subscription
+  [:map {:title "CustomerSubscription"}
+   [:id ::sm/text]
+   [:customer-id ::sm/text]
+   [:type [:enum
+           "unlimited"
+           "professional"
+           "enterprise"]]
+   [:status [:enum
+             "active"
+             "canceled"
+             "incomplete"
+             "incomplete_expired"
+             "pass_due"
+             "paused"
+             "trialing"
+             "unpaid"]]
+
+   [:billing-period [:enum
+                     "month"
+                     "day"
+                     "week"
+                     "year"]]
+   [:quantity :int]
+   [:description [:maybe ::sm/text]]
+   [:created-at ::sm/timestamp]
+   [:start-date [:maybe ::sm/timestamp]]
+   [:ended-at [:maybe ::sm/timestamp]]
+   [:trial-end [:maybe ::sm/timestamp]]
+   [:trial-start [:maybe ::sm/timestamp]]
+   [:cancel-at [:maybe ::sm/timestamp]]
+   [:canceled-at [:maybe ::sm/timestamp]]
+
+   [:current-period-end ::sm/timestamp]
+   [:current-period-start ::sm/timestamp]
+   [:cancel-at-period-end :boolean]
+
+   [:cancellation-details
+    [:map {:title "CancellationDetails"}
+     [:comment [:maybe ::sm/text]]
+     [:reason [:maybe ::sm/text]]
+     [:feedback [:maybe
+                 [:enum
+                  "customer_service"
+                  "low_quality"
+                  "missing_feature"
+                  "other"
+                  "switched_service"
+                  "too_complex"
+                  "too_expensive"
+                  "unused"]]]]]])
+
+(def ^:private schema:update-customer-subscription
+  [:map
+   [:id ::sm/uuid]
+   [:subscription [:maybe schema:customer-subscription]]])
+
+(def coerce-update-customer-subscription-params
+  (coercer schema:update-customer-subscription
+           :type :validation
+           :hint "invalid data provided for `update-customer-subscription` rpc call"))
+
+(defmethod exec-command "update-customer-subscription"
+  [params]
+  (when-let [system (get-current-system)]
+    (let [{:keys [id subscription]} (coerce-update-customer-subscription-params params)
+          ;; FIXME: locking
+          {:keys [props] :as profile} (cmd.profile/get-profile system id)
+          props (assoc props :subscription subscription)]
+
+      (db/update! system :profile
+                  {:props (db/tjson props)}
+                  {:id id}
+                  {::db/return-keys false})
+      true)))

backend/src/app/srepl/components_v2.clj

@@ -1,306 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.srepl.components-v2
-  (:require
-   [app.common.fressian :as fres]
-   [app.common.logging :as l]
-   [app.db :as db]
-   [app.features.components-v2 :as feat]
-   [app.main :as main]
-   [app.srepl.helpers :as h]
-   [app.util.events :as events]
-   [app.util.time :as dt]
-   [app.worker :as-alias wrk]
-   [datoteka.fs :as fs]
-   [datoteka.io :as io]
-   [promesa.exec :as px]
-   [promesa.exec.semaphore :as ps]
-   [promesa.util :as pu]))
-
-(def ^:dynamic *scope* nil)
-(def ^:dynamic *semaphore* nil)
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; PRIVATE HELPERS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(def ^:private sql:get-files-by-created-at
-  "SELECT id, features,
-          row_number() OVER (ORDER BY created_at DESC) AS rown
-     FROM file
-    WHERE deleted_at IS NULL
-    ORDER BY created_at DESC")
-
-(defn- get-files
-  [conn]
-  (->> (db/cursor conn [sql:get-files-by-created-at] {:chunk-size 500})
-       (map feat/decode-row)
-       (remove (fn [{:keys [features]}]
-                 (contains? features "components/v2")))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; PUBLIC API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn migrate-file!
-  [file-id & {:keys [rollback? validate? label cache skip-on-graphic-error?]
-              :or {rollback? true
-                   validate? false
-                   skip-on-graphic-error? true}}]
-  (l/dbg :hint "migrate:start" :rollback rollback?)
-  (let [tpoint  (dt/tpoint)
-        file-id (h/parse-uuid file-id)]
-
-    (binding [feat/*stats* (atom {})
-              feat/*cache* cache]
-      (try
-        (-> (assoc main/system ::db/rollback rollback?)
-            (feat/migrate-file! file-id
-                                :validate? validate?
-                                :skip-on-graphic-error? skip-on-graphic-error?
-                                :label label))
-
-        (-> (deref feat/*stats*)
-            (assoc :elapsed (dt/format-duration (tpoint))))
-
-        (catch Throwable cause
-          (l/wrn :hint "migrate:error" :cause cause))
-
-        (finally
-          (let [elapsed (dt/format-duration (tpoint))]
-            (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
-
-(defn migrate-team!
-  [team-id & {:keys [rollback? skip-on-graphic-error? validate? label cache]
-              :or {rollback? true
-                   validate? true
-                   skip-on-graphic-error? true}}]
-
-  (l/dbg :hint "migrate:start" :rollback rollback?)
-
-  (let [team-id (h/parse-uuid team-id)
-        stats   (atom {})
-        tpoint  (dt/tpoint)]
-
-    (binding [feat/*stats* stats
-              feat/*cache* cache]
-      (try
-        (-> (assoc main/system ::db/rollback rollback?)
-            (feat/migrate-team! team-id
-                                :label label
-                                :validate? validate?
-                                :skip-on-graphics-error? skip-on-graphic-error?))
-
-        (-> (deref feat/*stats*)
-            (assoc :elapsed (dt/format-duration (tpoint))))
-
-        (catch Throwable cause
-          (l/dbg :hint "migrate:error" :cause cause))
-
-        (finally
-          (let [elapsed (dt/format-duration (tpoint))]
-            (l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
-
-(defn migrate-files!
-  "A REPL helper for migrate all files.
-
-  This function starts multiple concurrent file migration processes
-  until thw maximum number of jobs is reached which by default has the
-  value of `1`. This is controled with the `:max-jobs` option.
-
-  If you want to run this on multiple machines you will need to specify
-  the total number of partitions and the current partition.
-
-  In order to get the report table populated, you will need to provide
-  a correct `:label`. That label is also used for persist a file
-  snaphot before continue with the migration."
-  [& {:keys [max-jobs max-items rollback? validate?
-             cache skip-on-graphic-error?
-             label partitions current-partition]
-      :or {validate? false
-           rollback? true
-           max-jobs 1
-           current-partition 1
-           skip-on-graphic-error? true
-           max-items Long/MAX_VALUE}}]
-
-  (when (int? partitions)
-    (when-not (int? current-partition)
-      (throw (IllegalArgumentException. "missing `current-partition` parameter")))
-    (when-not (<= 0 current-partition partitions)
-      (throw (IllegalArgumentException. "invalid value on `current-partition` parameter"))))
-
-  (let [stats     (atom {})
-        tpoint    (dt/tpoint)
-        factory   (px/thread-factory :virtual false :prefix "penpot/migration/")
-        executor  (px/cached-executor :factory factory)
-
-        sjobs     (ps/create :permits max-jobs)
-
-        migrate-file
-        (fn [file-id rown]
-          (try
-            (db/tx-run! (assoc main/system ::db/rollback rollback?)
-                        (fn [system]
-                          (db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-                          (feat/migrate-file! system file-id
-                                              :rown rown
-                                              :label label
-                                              :validate? validate?
-                                              :skip-on-graphic-error? skip-on-graphic-error?)))
-
-            (catch Throwable cause
-              (l/wrn :hint "unexpected error on processing file (skiping)"
-                     :file-id (str file-id))
-
-              (events/tap :error
-                          (ex-info "unexpected error on processing file (skiping)"
-                                   {:file-id file-id}
-                                   cause))
-
-              (swap! stats update :errors (fnil inc 0)))
-
-            (finally
-              (ps/release! sjobs))))
-
-        process-file
-        (fn [{:keys [id rown]}]
-          (ps/acquire! sjobs)
-          (px/run! executor (partial migrate-file id rown)))]
-
-    (l/dbg :hint "migrate:start"
-           :label label
-           :rollback rollback?
-           :max-jobs max-jobs
-           :max-items max-items)
-
-    (binding [feat/*stats* stats
-              feat/*cache* cache]
-      (try
-        (db/tx-run! main/system
-                    (fn [{:keys [::db/conn] :as system}]
-                      (db/exec! conn ["SET LOCAL statement_timeout = 0"])
-                      (db/exec! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
-
-                      (run! process-file
-                            (->> (get-files conn)
-                                 (filter (fn [{:keys [rown] :as row}]
-                                           (if (int? partitions)
-                                             (= current-partition (inc (mod rown partitions)))
-                                             true)))
-                                 (take max-items)))
-
-                      ;; Close and await tasks
-                      (pu/close! executor)))
-
-        (-> (deref stats)
-            (assoc :elapsed (dt/format-duration (tpoint))))
-
-        (catch Throwable cause
-          (l/dbg :hint "migrate:error" :cause cause)
-          (events/tap :error cause))
-
-        (finally
-          (let [elapsed (dt/format-duration (tpoint))]
-            (l/dbg :hint "migrate:end"
-                   :rollback rollback?
-                   :elapsed elapsed)))))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; CACHE POPULATE
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(def sql:sobjects-for-cache
-  "SELECT id,
-          row_number() OVER (ORDER BY created_at) AS index
-     FROM storage_object
-    WHERE (metadata->>'~:bucket' = 'file-media-object' OR
-           metadata->>'~:bucket' IS NULL)
-      AND metadata->>'~:content-type' = 'image/svg+xml'
-      AND deleted_at IS NULL
-      AND size < 1135899
-    ORDER BY created_at ASC")
-
-(defn populate-cache!
-  "A REPL helper for migrate all files.
-
-  This function starts multiple concurrent file migration processes
-  until thw maximum number of jobs is reached which by default has the
-  value of `1`. This is controled with the `:max-jobs` option.
-
-  If you want to run this on multiple machines you will need to specify
-  the total number of partitions and the current partition.
-
-  In order to get the report table populated, you will need to provide
-  a correct `:label`. That label is also used for persist a file
-  snaphot before continue with the migration."
-  [& {:keys [max-jobs] :or {max-jobs 1}}]
-
-  (let [tpoint    (dt/tpoint)
-
-        factory   (px/thread-factory :virtual false :prefix "penpot/cache/")
-        executor  (px/cached-executor :factory factory)
-
-        sjobs     (ps/create :permits max-jobs)
-
-        retrieve-sobject
-        (fn [id index]
-          (let [path   (feat/get-sobject-cache-path id)
-                parent (fs/parent path)]
-
-            (try
-              (when-not (fs/exists? parent)
-                (fs/create-dir parent))
-
-              (if (fs/exists? path)
-                (l/inf :hint "create cache entry" :status "exists" :index index :id (str id) :path (str path))
-                (let [svg-data (feat/get-optimized-svg id)]
-                  (with-open [^java.lang.AutoCloseable stream (io/output-stream path)]
-                    (let [writer (fres/writer stream)]
-                      (fres/write! writer svg-data)))
-
-                  (l/inf :hint "create cache entry" :status "created"
-                         :index index
-                         :id (str id)
-                         :path (str path))))
-
-              (catch Throwable cause
-                (l/wrn :hint "create cache entry"
-                       :status "error"
-                       :index index
-                       :id (str id)
-                       :path (str path)
-                       :cause cause))
-
-              (finally
-                (ps/release! sjobs)))))
-
-        process-sobject
-        (fn [{:keys [id index]}]
-          (ps/acquire! sjobs)
-          (px/run! executor (partial retrieve-sobject id index)))]
-
-    (l/dbg :hint "migrate:start"
-           :max-jobs max-jobs)
-
-    (try
-      (binding [feat/*system* main/system]
-        (run! process-sobject
-              (db/exec! main/system [sql:sobjects-for-cache]))
-
-        ;; Close and await tasks
-        (pu/close! executor))
-
-      {:elapsed (dt/format-duration (tpoint))}
-
-      (catch Throwable cause
-        (l/dbg :hint "populate:error" :cause cause))
-
-      (finally
-        (let [elapsed (dt/format-duration (tpoint))]
-          (l/dbg :hint "populate:end"
-                 :elapsed elapsed))))))

backend/src/app/srepl/fixes.clj

@@ -179,7 +179,7 @@
                       component-child (first component-children)]
                   (if (or (nil? child) (nil? component-child))
                     container
-                    (let [container (if (and (not (ctk/is-main-of? component-child child true))
+                    (let [container (if (and (not (ctk/is-main-of? component-child child))
                                              (nil? (ctk/get-swap-slot child))
                                              (ctk/instance-head? child))
                                       (let [slot (guess-swap-slot component-child component-container)]

backend/src/app/srepl/fixes/lost_colors.clj

@@ -0,0 +1,88 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.srepl.fixes.lost-colors
+  "A collection of adhoc fixes scripts."
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.logging :as l]
+   [app.common.types.color :as types.color]
+   [app.db :as db]
+   [app.srepl.helpers :as h]))
+
+(def sql:get-affected-files
+  "SELECT fm.file_id AS id FROM file_migration AS fm WHERE fm.name = '0008-fix-library-colors-v2'")
+
+(def sql:get-matching-snapshot
+  "SELECT * FROM file_change
+    WHERE file_id = ?
+      AND created_at <= ?
+      AND label IS NOT NULL
+      AND data IS NOT NULL
+    ORDER BY created_at DESC
+    LIMIT 2")
+
+(defn get-affected-migration
+  [conn file-id]
+  (db/get* conn :file-migration
+           {:name "0008-fix-library-colors-v2"
+            :file-id file-id}))
+
+(defn get-last-valid-snapshot
+  [conn migration]
+  (let [[snapshot] (db/exec! conn [sql:get-matching-snapshot
+                                   (:file-id migration)
+                                   (:created-at migration)])]
+    (when snapshot
+      (let [snapshot (assoc snapshot :id (:file-id snapshot))]
+        (bfc/decode-file h/*system* snapshot)))))
+
+(defn restore-color
+  [{:keys [data] :as snapshot} color]
+  (when-let [scolor (get-in data [:colors (:id color)])]
+    (-> (select-keys scolor types.color/library-color-attrs)
+        (types.color/check-library-color))))
+
+(defn restore-missing-colors
+  [{:keys [id] :as file} & _opts]
+  (l/inf :hint "process file" :file-id (str id) :name (:name file) :has-colors (-> file :data :colors not-empty boolean))
+  (if-let [colors (-> file :data :colors not-empty)]
+    (let [migration (get-affected-migration h/*system* id)]
+      (if-let [snapshot (get-last-valid-snapshot h/*system* migration)]
+        (do
+          (l/inf :hint "using snapshot" :snapshot (:label snapshot))
+          (let [colors (reduce-kv (fn [colors color-id color]
+                                    (if-let [result (restore-color snapshot color)]
+                                      (do
+                                        (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id))
+                                        (assoc colors color-id result))
+                                      (do
+                                        (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color))
+                                        colors)))
+                                  colors
+                                  colors)
+                file   (-> file
+                           (update :data assoc :colors colors)
+                           (update :migrations disj "0008-fix-library-colors-v2"))]
+
+            (db/delete! h/*system* :file-migration
+                        {:name "0008-fix-library-colors-v2"
+                         :file-id (:id file)})
+            file))
+
+        (do
+          (db/delete! h/*system* :file-migration
+                      {:name "0008-fix-library-colors-v2"
+                       :file-id (:id file)})
+          nil)))
+
+    (do
+      (db/delete! h/*system* :file-migration
+                  {:name "0008-fix-library-colors-v2"
+                   :file-id (:id file)})
+      nil)))
+
+

backend/src/app/srepl/helpers.clj

@@ -10,9 +10,9 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.data :as d]
+   [app.common.files.migrations :as fmg]
    [app.common.files.validate :as cfv]
    [app.db :as db]
-   [app.features.components-v2 :as feat.comp-v2]
    [app.main :as main]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.files-snapshot :as fsnap]
@@ -61,6 +61,27 @@
                 {:id id})
     team))
 
+(def ^:private sql:get-and-lock-team-files
+  "SELECT f.id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+    WHERE p.team_id = ?
+      AND p.deleted_at IS NULL
+      AND f.deleted_at IS NULL
+      FOR UPDATE")
+
+(defn get-team
+  [conn team-id]
+  (-> (db/get conn :team {:id team-id}
+              {::db/remove-deleted false
+               ::db/check-deleted false})
+      (update :features db/decode-pgarray #{})))
+
+(defn get-and-lock-team-files
+  [conn team-id]
+  (transduce (map :id) conj []
+             (db/plan conn [sql:get-and-lock-team-files team-id])))
+
 (defn reset-file-data!
   "Hardcode replace of the data of one file."
   [system id data]
@@ -95,7 +116,7 @@
 (defn take-team-snapshot!
   [system team-id label]
   (let [conn (db/get-connection system)]
-    (->> (feat.comp-v2/get-and-lock-team-files conn team-id)
+    (->> (get-and-lock-team-files conn team-id)
          (reduce (fn [result file-id]
                    (let [file (fsnap/get-file-snapshots system file-id)]
                      (fsnap/create-file-snapshot! system file
@@ -107,19 +128,16 @@
 (defn restore-team-snapshot!
   [system team-id label]
   (let [conn (db/get-connection system)
-        ids  (->> (feat.comp-v2/get-and-lock-team-files conn team-id)
+        ids  (->> (get-and-lock-team-files conn team-id)
                   (into #{}))
 
         snap (search-file-snapshots conn ids label)
 
-        ids' (into #{} (map :file-id) snap)
-        team (-> (feat.comp-v2/get-team conn team-id)
-                 (update :features disj "components/v2"))]
+        ids' (into #{} (map :file-id) snap)]
 
     (when (not= ids ids')
       (throw (RuntimeException. "no uniform snapshot available")))
 
-    (feat.comp-v2/update-team! conn team)
     (reduce (fn [result {:keys [file-id id]}]
               (fsnap/restore-file-snapshot! system file-id id)
               (inc result))
@@ -128,13 +146,9 @@
 
 (defn process-file!
   [system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}]
-  (let [conn  (db/get-connection system)
-        file  (bfc/get-file system file-id ::db/for-update true)
+  (let [file  (bfc/get-file system file-id ::db/for-update true)
         libs  (when with-libraries?
-                (->> (files/get-file-libraries conn file-id)
-                     (into [file] (map (fn [{:keys [id]}]
-                                         (bfc/get-file system id))))
-                     (d/index-by :id)))
+                (bfc/get-resolved-file-libraries system file))
 
         file' (when file
                 (if with-libraries?
@@ -142,7 +156,9 @@
                   (update-fn file opts)))]
 
     (when (and (some? file')
-               (not (identical? file file')))
+               (or (fmg/migrated? file)
+                   (not (identical? file file'))))
+
       (when validate?
         (cfv/validate-file-schema! file'))
 

backend/src/app/srepl/main.clj

@@ -17,12 +17,12 @@
    [app.common.files.validate :as cfv]
    [app.common.logging :as l]
    [app.common.pprint :as p]
+   [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.components-v2 :as feat.comp-v2]
    [app.features.fdata :as feat.fdata]
    [app.loggers.audit :as audit]
    [app.main :as main]
@@ -101,38 +101,46 @@
   "Mark the profile blocked and removes all the http sessiones
   associated with the profile-id."
   [email]
-  (db/with-atomic [conn (:app.db/pool main/system)]
-    (when-let [profile (db/get* conn :profile
-                                {:email (str/lower email)}
-                                {:columns [:id :email]})]
-      (when-not (:is-blocked profile)
-        (db/update! conn :profile {:is-active true} {:id (:id profile)})
-        :activated))))
+  (some-> main/system
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (when-let [profile (db/get* conn :profile
+                                         {:email (str/lower email)}
+                                         {:columns [:id :email]})]
+               (when-not (:is-blocked profile)
+                 (db/update! conn :profile {:is-active true} {:id (:id profile)})
+                 :activated))))))
 
 (defn mark-profile-as-blocked!
   "Mark the profile blocked and removes all the http sessiones
   associated with the profile-id."
   [email]
-  (db/with-atomic [conn (:app.db/pool main/system)]
-    (when-let [profile (db/get* conn :profile
-                                {:email (str/lower email)}
-                                {:columns [:id :email]})]
-      (when-not (:is-blocked profile)
-        (db/update! conn :profile {:is-blocked true} {:id (:id profile)})
-        (db/delete! conn :http-session {:profile-id (:id profile)})
-        :blocked))))
+  (some-> main/system
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (when-let [profile (db/get* conn :profile
+                                         {:email (str/lower email)}
+                                         {:columns [:id :email]})]
+               (when-not (:is-blocked profile)
+                 (db/update! conn :profile {:is-blocked true} {:id (:id profile)})
+                 (db/delete! conn :http-session {:profile-id (:id profile)})
+                 :blocked))))))
 
 (defn reset-password!
   "Reset a password to a specific one for a concrete user or all users
   if email is `:all` keyword."
   [& {:keys [email password] :or {password "123123"} :as params}]
-  (us/verify! (contains? params :email) "`email` parameter is mandatory")
-  (db/with-atomic [conn (:app.db/pool main/system)]
-    (let [password (derive-password password)]
-      (if (= email :all)
-        (db/exec! conn ["update profile set password=?" password])
-        (let [email (str/lower email)]
-          (db/exec! conn ["update profile set password=? where email=?" password email]))))))
+  (when-not email
+    (throw (IllegalArgumentException. "email is mandatory")))
+
+  (some-> main/system
+          (db/tx-run!
+           (fn [{:keys [::db/conn] :as system}]
+             (let [password (derive-password password)]
+               (if (= email :all)
+                 (db/exec! conn ["update profile set password=?" password])
+                 (let [email (str/lower email)]
+                   (db/exec! conn ["update profile set password=? where email=?" password email]))))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FEATURES
@@ -148,14 +156,18 @@
   [file-id & {:as opts}]
   (process-file! file-id feat.fdata/enable-pointer-map opts))
 
+(defn enable-path-data-feature-on-file!
+  [file-id & {:as opts}]
+  (process-file! file-id feat.fdata/enable-path-data opts))
+
 (defn enable-storage-features-on-file!
   [file-id & {:as opts}]
   (enable-objects-map-feature-on-file! file-id opts)
   (enable-pointer-map-feature-on-file! file-id opts))
 
 (defn enable-team-feature!
-  [team-id feature]
-  (when-not (contains? cfeat/supported-features feature)
+  [team-id feature & {:keys [skip-check] :or {skip-check false}}]
+  (when (and (not skip-check) (not (contains? cfeat/supported-features feature)))
     (ex/raise :type :assertion
               :code :feature-not-supported
               :hint (str "feature '" feature "' not supported")))
@@ -173,9 +185,8 @@
                       :enabled))))))
 
 (defn disable-team-feature!
-  [team-id feature]
-
-  (when-not (contains? cfeat/supported-features feature)
+  [team-id feature & {:keys [skip-check] :or {skip-check false}}]
+  (when (and (not skip-check) (not (contains? cfeat/supported-features feature)))
     (ex/raise :type :assertion
               :code :feature-not-supported
               :hint (str "feature '" feature "' not supported")))
@@ -202,100 +213,116 @@
   This method allows send flash notifications to specified target destinations.
   The message can be a free text or a preconfigured one.
 
-  The destination can be: all, profile-id, team-id, or a coll of them."
-  [{:keys [::mbus/msgbus ::db/pool]} & {:keys [dest code message level]
-                                        :or {code :generic level :info}
-                                        :as params}]
+  The destination can be: all, profile-id, team-id, or a coll of them.
+  It also can be:
+
+  {:email \"some@example.com\"}
+  [[:email \"some@example.com\"], ...]
+
+  Command examples:
+
+  (notify! :dest :all :code :maintenance)
+  (notify! :dest :all :code :upgrade-version)
+  "
+  [& {:keys [dest code message level]
+      :or {code :generic level :info}
+      :as params}]
 
   (when-not (contains? #{:success :error :info :warning} level)
     (ex/raise :type :assertion
               :code :incorrect-level
               :hint (str "level '" level "' not supported")))
 
-  (letfn [(send [dest]
-            (l/inf :hint "sending notification" :dest (str dest))
-            (let [message {:type :notification
-                           :code code
-                           :level level
-                           :version (:full cf/version)
-                           :subs-id dest
-                           :message message}
-                  message (->> (dissoc params :dest :code :message :level)
-                               (merge message))]
-              (mbus/pub! msgbus
-                         :topic (str dest)
-                         :message message)))
+  (let [{:keys [::mbus/msgbus ::db/pool]} main/system
 
-          (resolve-profile [email]
-            (some-> (db/get* pool :profile {:email (str/lower email)} {:columns [:id]}) :id vector))
+        send
+        (fn [dest]
+          (l/inf :hint "sending notification" :dest (str dest))
+          (let [message {:type :notification
+                         :code code
+                         :level level
+                         :version (:full cf/version)
+                         :subs-id dest
+                         :message message}
+                message (->> (dissoc params :dest :code :message :level)
+                             (merge message))]
+            (mbus/pub! msgbus
+                       :topic dest
+                       :message message)))
 
-          (resolve-team [team-id]
-            (->> (db/query pool :team-profile-rel
-                           {:team-id team-id}
-                           {:columns [:profile-id]})
-                 (map :profile-id)))
+        resolve-profile
+        (fn [email]
+          (some-> (db/get* pool :profile {:email (str/lower email)} {:columns [:id]}) :id vector))
 
-          (resolve-dest [dest]
-            (cond
-              (= :all dest)
-              [uuid/zero]
+        resolve-team
+        (fn [team-id]
+          (->> (db/query pool :team-profile-rel
+                         {:team-id team-id}
+                         {:columns [:profile-id]})
+               (map :profile-id)))
 
-              (uuid? dest)
-              [dest]
+        resolve-dest
+        (fn resolve-dest [dest]
+          (cond
+            (= :all dest)
+            [uuid/zero]
 
-              (string? dest)
-              (some-> dest h/parse-uuid resolve-dest)
+            (uuid? dest)
+            [dest]
 
-              (nil? dest)
-              (resolve-dest uuid/zero)
+            (string? dest)
+            (some-> dest h/parse-uuid resolve-dest)
 
-              (map? dest)
-              (sequence (comp
-                         (map vec)
-                         (mapcat resolve-dest))
-                        dest)
+            (nil? dest)
+            [uuid/zero]
 
-              (and (vector? dest)
-                   (every? vector? dest))
-              (sequence (comp
-                         (map vec)
-                         (mapcat resolve-dest))
-                        dest)
+            (map? dest)
+            (sequence (comp
+                       (map vec)
+                       (mapcat resolve-dest))
+                      dest)
 
-              (and (vector? dest)
-                   (keyword? (first dest)))
-              (let [[op param] dest]
+            (and (vector? dest)
+                 (every? vector? dest))
+            (sequence (comp
+                       (map vec)
+                       (mapcat resolve-dest))
+                      dest)
+
+            (and (vector? dest)
+                 (keyword? (first dest)))
+            (let [[op param] dest]
+              (cond
+                (= op :email)
                 (cond
-                  (= op :email)
-                  (cond
-                    (and (coll? param)
-                         (every? string? param))
-                    (sequence (comp
-                               (keep resolve-profile)
-                               (mapcat identity))
-                              param)
+                  (and (coll? param)
+                       (every? string? param))
+                  (sequence (comp
+                             (keep resolve-profile)
+                             (mapcat identity))
+                            param)
 
-                    (string? param)
-                    (resolve-profile param))
+                  (string? param)
+                  (resolve-profile param))
 
-                  (= op :team-id)
-                  (cond
-                    (coll? param)
-                    (sequence (comp
-                               (mapcat resolve-team)
-                               (keep h/parse-uuid))
-                              param)
+                (= op :team-id)
+                (cond
+                  (coll? param)
+                  (sequence (comp
+                             (mapcat resolve-team)
+                             (keep h/parse-uuid))
+                            param)
 
-                    (uuid? param)
-                    (resolve-team param)
+                  (uuid? param)
+                  (resolve-team param)
 
-                    (string? param)
-                    (some-> param h/parse-uuid resolve-team))
+                  (string? param)
+                  (some-> param h/parse-uuid resolve-team))
 
-                  (= op :profile-id)
-                  (if (coll? param)
-                    (sequence (keep h/parse-uuid) param)
-                    (resolve-dest param))))))]
+                (= op :profile-id)
+                (if (coll? param)
+                  (sequence (keep h/parse-uuid) param)
+                  (resolve-dest param))))))]
 
     (->> (resolve-dest dest)
          (filter some?)
@@ -314,14 +341,23 @@
     (db/tx-run! main/system fsnap/create-file-snapshot! {:file-id file-id :label label})))
 
 (defn restore-file-snapshot!
-  [file-id label]
-  (let [file-id (h/parse-uuid file-id)]
+  [file-id & {:keys [label id]}]
+  (let [file-id     (h/parse-uuid file-id)
+        snapshot-id (some-> id h/parse-uuid)]
     (db/tx-run! main/system
                 (fn [{:keys [::db/conn] :as system}]
-                  (when-let [snapshot (->> (h/search-file-snapshots conn #{file-id} label)
-                                           (map :id)
-                                           (first))]
-                    (fsnap/restore-file-snapshot! system file-id (:id snapshot)))))))
+                  (cond
+                    (uuid? snapshot-id)
+                    (fsnap/restore-file-snapshot! system file-id snapshot-id)
+
+                    (string? label)
+                    (->> (h/search-file-snapshots conn #{file-id} label)
+                         (map :id)
+                         (first)
+                         (fsnap/restore-file-snapshot! system file-id))
+
+                    :else
+                    (throw (ex-info "snapshot id or label should be provided" {})))))))
 
 (defn list-file-snapshots!
   [file-id & {:as _}]
@@ -355,14 +391,27 @@
   [file-id]
   (let [file-id (h/parse-uuid file-id)]
     (db/tx-run! (assoc main/system ::db/rollback true)
-                (fn [{:keys [::db/conn] :as system}]
-                  (let [file (h/get-file system file-id)
-                        libs (->> (files/get-file-libraries conn file-id)
-                                  (into [file] (map (fn [{:keys [id]}]
-                                                      (h/get-file system id))))
-                                  (d/index-by :id))]
+                (fn [system]
+                  (let [file (bfc/get-file system file-id)
+                        libs (bfc/get-resolved-file-libraries system file)]
                     (cfv/validate-file file libs))))))
 
+(defn validate-file-schema
+  "Validate structure, referencial integrity and semantic coherence of
+  all contents of a file. Returns a list of errors."
+  [file-id]
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! (assoc main/system ::db/rollback true)
+                (fn [system]
+                  (try
+                    (let [file (bfc/get-file system file-id)]
+                      (cfv/validate-file-schema! file)
+                      (println "OK"))
+                    (catch Exception cause
+                      (if-let [explain (-> cause ex-data ::sm/explain)]
+                        (println (sm/humanize-explain explain))
+                        (ex/print-throwable cause))))))))
+
 (defn repair-file!
   "Repair the list of errors detected by validation."
   [file-id & {:keys [rollback?] :or {rollback? true} :as opts}]
@@ -384,10 +433,12 @@
   "Apply a function to the file. Optionally save the changes or not.
   The function receives the decoded and migrated file data."
   [file-id update-fn & {:keys [rollback?] :or {rollback? true} :as opts}]
-  (db/tx-run! (assoc main/system ::db/rollback rollback?)
-              (fn [system]
-                (binding [h/*system* system]
-                  (h/process-file! system file-id update-fn opts)))))
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! (assoc main/system ::db/rollback rollback?)
+                (fn [system]
+                  (binding [h/*system* system
+                            db/*conn* (db/get-connection system)]
+                    (h/process-file! system file-id update-fn opts))))))
 
 (defn process-team-files!
   "Apply a function to each file of the specified team."
@@ -399,8 +450,9 @@
                   (when (string? label)
                     (h/take-team-snapshot! system team-id label))
 
-                  (binding [h/*system* system]
-                    (->> (feat.comp-v2/get-and-lock-team-files conn team-id)
+                  (binding [h/*system* system
+                            db/*conn* (db/get-connection system)]
+                    (->> (h/get-and-lock-team-files conn team-id)
                          (reduce (fn [result file-id]
                                    (if (h/process-file! system file-id update-fn opts)
                                      (inc result)
@@ -439,7 +491,8 @@
                      :index idx)
               (let [system (assoc main/system ::db/rollback rollback?)]
                 (db/tx-run! system (fn [system]
-                                     (binding [h/*system* system]
+                                     (binding [h/*system* system
+                                               db/*conn* (db/get-connection system)]
                                        (h/process-file! system file-id update-fn opts)))))
 
               (catch Throwable cause

backend/src/app/tasks/file_gc.clj

@@ -10,6 +10,7 @@
   file is eligible to be garbage collected after some period of
   inactivity (the default threshold is 72h)."
   (:require
+   [app.binfile.cleaner :as bfl]
    [app.binfile.common :as bfc]
    [app.common.files.helpers :as cfh]
    [app.common.files.validate :as cfv]
@@ -258,6 +259,7 @@
   (if-let [file (get-file cfg file-id)]
     (let [file (->> file
                     (bfc/decode-file cfg)
+                    (bfl/clean-file)
                     (clean-media! cfg)
                     (clean-fragments! cfg))
           file (assoc file :has-media-trimmed true)]

backend/src/app/tasks/objects_gc.clj

@@ -9,7 +9,6 @@
   of deleted or unreachable objects."
   (:require
    [app.common.logging :as l]
-   [app.config :as cf]
    [app.db :as db]
    [app.storage :as sto]
    [app.util.time :as dt]
@@ -18,15 +17,15 @@
 (def ^:private sql:get-profiles
   "SELECT id, photo_id FROM profile
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-profiles!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-profiles min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id]}]
                  (l/trc :hint "permanently delete" :rel "profile" :id (str id))
 
@@ -41,15 +40,15 @@
 (def ^:private sql:get-teams
   "SELECT deleted_at, id, photo_id FROM team
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-teams!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-teams min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "team"
@@ -69,15 +68,15 @@
   "SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
      FROM team_font_variant
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-fonts!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-fonts min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at] :as font}]
                  (l/trc :hint "permanently delete"
                         :rel "team-font-variant"
@@ -101,15 +100,15 @@
   "SELECT id, deleted_at, team_id
      FROM project
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-projects!
-  [{:keys [::db/conn ::min-age ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-projects min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  (->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "project"
@@ -127,15 +126,15 @@
   "SELECT id, deleted_at, project_id, data_backend, data_ref_id
      FROM file
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-files!
-  [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-files min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
+  (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id deleted-at project-id] :as file}]
                  (l/trc :hint "permanently delete"
                         :rel "file"
@@ -156,15 +155,15 @@
   "SELECT file_id, revn, media_id, deleted_at
      FROM file_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-thumbnails!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-thumbnails min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-thumbnail"
@@ -185,15 +184,15 @@
   "SELECT file_id, object_id, media_id, deleted_at
      FROM file_tagged_object_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-object-thumbnails!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-object-thumbnails min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-tagged-object-thumbnail"
@@ -214,15 +213,15 @@
   "SELECT file_id, id, deleted_at, data_ref_id
      FROM file_data_fragment
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-data-fragments!
-  [{:keys [::db/conn ::sto/storage ::min-age ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-file-data-fragments min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
+  (->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
                  (l/trc :hint "permanently delete"
                         :rel "file-data-fragment"
@@ -240,15 +239,15 @@
   "SELECT id, file_id, media_id, thumbnail_id, deleted_at
      FROM file_media_object
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-media-objects!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-media-objects min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
                  (l/trc :hint "permanently delete"
                         :rel "file-media-object"
@@ -269,15 +268,15 @@
   "SELECT id, file_id, deleted_at, data_backend, data_ref_id
      FROM file_change
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() - ?::interval
+      AND deleted_at < now() + ?::interval
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-changes!
-  [{:keys [::db/conn ::min-age ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-change min-age chunk-size] {:fetch-size 5})
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
                  (l/trc :hint "permanently delete"
                         :rel "file-change"
@@ -324,16 +323,13 @@
 
 (defmethod ig/expand-key ::handler
   [k v]
-  {k (assoc v
-            ::min-age (cf/get-deletion-delay)
-            ::chunk-size 100)})
+  {k (assoc v ::chunk-size 100)})
 
 (defmethod ig/init-key ::handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (let [min-age (dt/duration (or (:min-age props) (::min-age cfg)))
-          cfg     (assoc cfg ::min-age (db/interval min-age))]
-
+    (let [threshold (dt/duration (get props :deletion-threshold 0))
+          cfg       (assoc cfg ::deletion-threshold (db/interval threshold))]
       (loop [procs (map deref deletion-proc-vars)
              total 0]
         (if-let [proc-fn (first procs)]

backend/src/app/tasks/tasks_gc.clj

@@ -26,18 +26,14 @@
   {k (assoc v ::min-age (cf/get-deletion-delay))})
 
 (defmethod ig/init-key ::handler
-  [_ {:keys [::db/pool ::min-age] :as cfg}]
+  [_ {:keys [::min-age] :as cfg}]
   (fn [{:keys [props] :as task}]
     (let [min-age (or (:min-age props) min-age)]
-      (db/with-atomic [conn pool]
-        (let [interval (db/interval min-age)
-              result   (db/exec-one! conn [sql:delete-completed-tasks interval])
-              result   (db/get-update-count result)]
-
-          (l/debug :hint "task finished" :total result)
-
-          (when (:rollback? props)
-            (db/rollback! conn))
-
-          result)))))
-
+      (-> cfg
+          (assoc ::db/rollback (:rollback? props))
+          (db/tx-run! (fn [{:keys [::db/conn]}]
+                        (let [interval (db/interval min-age)
+                              result   (db/exec-one! conn [sql:delete-completed-tasks interval])
+                              result   (db/get-update-count result)]
+                          (l/debug :hint "task finished" :total result)
+                          result)))))))

backend/src/app/util/events.clj

@@ -10,7 +10,6 @@
   to them. Mainly used in http.sse for progress reporting."
   (:refer-clojure :exclude [tap run!])
   (:require
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [promesa.exec :as px]
@@ -18,33 +17,30 @@
 
 (def ^:dynamic *channel* nil)
 
-(defn channel
-  []
-  (sp/chan :buf 32))
-
 (defn tap
-  [type data]
-  (when-let [channel *channel*]
-    (sp/put! channel [type data])
-    nil))
+  ([type data]
+   (when-let [channel *channel*]
+     (sp/put! channel [type data])
+     nil))
+  ([channel type data]
+   (when channel
+     (sp/put! channel [type data])
+     nil)))
 
 (defn start-listener
-  [on-event on-close]
-
-  (dm/assert!
-   "expected active events channel"
-   (sp/chan? *channel*))
+  [channel on-event on-close]
+  (assert (sp/chan? channel) "expected active events channel")
 
   (px/thread
     {:virtual true}
     (try
       (loop []
-        (when-let [event (sp/take! *channel*)]
+        (when-let [event (sp/take! channel)]
           (let [result (ex/try! (on-event event))]
             (if (ex/exception? result)
               (do
                 (l/wrn :hint "unexpected exception" :cause result)
-                (sp/close! *channel*))
+                (sp/close! channel))
               (recur)))))
       (finally
         (on-close)))))
@@ -55,7 +51,7 @@
   [f on-event]
 
   (binding [*channel* (sp/chan :buf 32)]
-    (let [listener (start-listener on-event (constantly nil))]
+    (let [listener (start-listener *channel* on-event (constantly nil))]
       (try
         (f)
         (finally

backend/src/app/worker/dispatcher.clj

@@ -71,11 +71,12 @@
 
           (run-batch! [rconn]
             (try
-              (db/with-atomic [conn pool]
-                (if-let [tasks (get-tasks conn)]
-                  (->> (group-by :queue tasks)
-                       (run! (partial push-tasks! conn rconn)))
-                  (px/sleep (::wait-duration cfg))))
+              (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                                (if-let [tasks (get-tasks conn)]
+                                  (->> (group-by :queue tasks)
+                                       (run! (partial push-tasks! conn rconn)))
+                                  ;; FIXME: this sleep should be outside the transaction
+                                  (px/sleep (::wait-duration cfg)))))
               (catch InterruptedException cause
                 (throw cause))
               (catch Exception cause

backend/src/app/worker/runner.clj

@@ -24,6 +24,33 @@
 
 (set! *warn-on-reflection* true)
 
+(def schema:task
+  [:map {:title "Task"}
+   [:id ::sm/uuid]
+   [:queue :string]
+   [:name :string]
+   [:created-at ::sm/inst]
+   [:modified-at ::sm/inst]
+   [:scheduled-at {:optional true} ::sm/inst]
+   [:completed-at {:optional true} ::sm/inst]
+   [:error {:optional true} :string]
+   [:max-retries :int]
+   [:retry-num :int]
+   [:priority :int]
+   [:status [:enum "scheduled" "completed" "new" "retry" "failed"]]
+   [:label {:optional true} :string]
+   [:props :map]])
+
+(def schema:result
+  [:map {:title "TaskResult"}
+   [:status [:enum "retry" "failed" "completed"]]
+   [:error {:optional true} [:fn ex/exception?]]
+   [:inc-by {:optional true} :int]
+   [:delay {:optional true} :int]])
+
+(def valid-task-result?
+  (sm/validator schema:result))
+
 (defn- decode-task-row
   [{:keys [props] :as row}]
   (cond-> row
@@ -51,10 +78,11 @@
            :retry (:retry-num task))
     (let [tpoint  (dt/tpoint)
           task-fn (wrk/get-task registry (:name task))
-          result  (if task-fn
-                    (task-fn task)
-                    {:status :completed :task task})
-          elapsed (dt/format-duration (tpoint))]
+          result  (when task-fn (task-fn task))
+          elapsed (dt/format-duration (tpoint))
+          result  (if (valid-task-result? result)
+                    result
+                    {:status "completed"})]
 
       (when-not task-fn
         (l/wrn :hint "no task handler found" :name (:name task)))
@@ -76,7 +104,7 @@
         (if (and (< (:retry-num task)
                     (:max-retries task))
                  (= ::retry (:type edata)))
-          (cond-> {:status :retry :task task :error cause}
+          (cond-> {:status "retry" :error cause}
             (dt/duration? (:delay edata))
             (assoc :delay (:delay edata))
 
@@ -87,8 +115,8 @@
                    ::l/context (get-error-context cause task)
                    :cause cause)
             (if (>= (:retry-num task) (:max-retries task))
-              {:status :failed :task task :error cause}
-              {:status :retry :task task :error cause})))))))
+              {:status "failed" :error cause}
+              {:status "retry" :error cause})))))))
 
 (defn- run-task!
   [{:keys [::id ::timeout] :as cfg} task-id]
@@ -116,12 +144,17 @@
              :task-id task-id)
 
       :else
-      (run-task cfg task))))
+      (let [result (run-task cfg task)]
+        (with-meta result
+          {::task task})))))
 
 (defn- run-worker-loop!
   [{:keys [::db/pool ::rds/rconn ::timeout ::queue] :as cfg}]
-  (letfn [(handle-task-retry [{:keys [task error inc-by delay] :or {inc-by 1 delay 1000}}]
-            (let [explain (ex-message error)
+  (letfn [(handle-task-retry [{:keys [error inc-by delay] :or {inc-by 1 delay 1000} :as result}]
+            (let [explain (if (ex/exception? error)
+                            (ex-message error)
+                            (str error))
+                  task    (-> result meta ::task)
                   nretry  (+ (:retry-num task) inc-by)
                   now     (dt/now)
                   delay   (->> (iterate #(* % 2) delay) (take nretry) (last))]
@@ -134,8 +167,9 @@
                           {:id (:id task)})
               nil))
 
-          (handle-task-failure [{:keys [task error]}]
-            (let [explain (ex-message error)]
+          (handle-task-failure [{:keys [error] :as result}]
+            (let [task    (-> result meta ::task)
+                  explain (ex-message error)]
               (db/update! pool :task
                           {:error explain
                            :modified-at (dt/now)
@@ -143,8 +177,9 @@
                           {:id (:id task)})
               nil))
 
-          (handle-task-completion [{:keys [task]}]
-            (let [now (dt/now)]
+          (handle-task-completion [result]
+            (let [task (-> result meta ::task)
+                  now  (dt/now)]
               (db/update! pool :task
                           {:completed-at now
                            :modified-at now
@@ -168,10 +203,11 @@
           (process-result [{:keys [status] :as result}]
             (ex/try!
              (case status
-               :retry (handle-task-retry result)
-               :failed (handle-task-failure result)
-               :completed (handle-task-completion result)
-               nil)))
+               "retry"     (handle-task-retry result)
+               "failed"    (handle-task-failure result)
+               "completed" (handle-task-completion result)
+               (throw (IllegalArgumentException.
+                       (str "invalid status received: " status))))))
 
           (run-task-loop [task-id]
             (loop [result (run-task! cfg task-id)]

backend/test/backend_tests/helpers.clj

@@ -138,14 +138,13 @@
                  "  FROM information_schema.tables "
                  " WHERE table_schema = 'public' "
                  "   AND table_name != 'migrations';")]
-    (db/with-atomic [conn *pool*]
-      (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-      (db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
-      (let [result (->> (db/exec! conn [sql])
-                        (map :table-name))]
-        (doseq [table result]
-          (db/exec! conn [(str "delete from " table ";")]))))
-
+    (db/transact! *pool* (fn [conn]
+                           (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
+                           (db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
+                           (let [result (->> (db/exec! conn [sql])
+                                             (map :table-name))]
+                             (doseq [table result]
+                               (db/exec! conn [(str "delete from " table ";")])))))
     (next)))
 
 (defn clean-storage
@@ -223,7 +222,7 @@
   ([params]
    (mark-file-deleted* *system* params))
   ([conn {:keys [id] :as params}]
-   (#'files/mark-file-deleted conn id)))
+   (#'files/mark-file-deleted conn {} id)))
 
 (defn create-team*
   ([i params] (create-team* *system* i params))

backend/test/backend_tests/http_middleware_access_token_test.clj

@@ -20,45 +20,33 @@
 (t/use-fixtures :each th/database-reset)
 
 (t/deftest soft-auth-middleware
-  (db/with-atomic [conn (::db/pool th/*system*)]
-    (let [profile (th/create-profile* 1)
-          system  (-> th/*system*
-                      (assoc ::db/conn conn)
-                      (assoc ::main/props (:app.setup/props th/*system*)))
+  (let [profile (th/create-profile* 1)
+        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
 
-          token   (app.rpc.commands.access-token/create-access-token
-                   system (:id profile) "test" nil)
+        request (volatile! nil)
+        handler (#'app.http.access-token/wrap-soft-auth
+                 (fn [req] (vreset! request req))
+                 th/*system*)]
 
-          request (volatile! nil)
-          handler (#'app.http.access-token/wrap-soft-auth
-                   (fn [req] (vreset! request req))
-                   system)]
+    (with-mocks [m1 {:target 'app.http.access-token/get-token
+                     :return nil}]
+      (handler {})
+      (t/is (= {} @request)))
 
-      (with-mocks [m1 {:target 'app.http.access-token/get-token
-                       :return nil}]
-        (handler {})
-        (t/is (= {} @request)))
+    (with-mocks [m1 {:target 'app.http.access-token/get-token
+                     :return (:token token)}]
+      (handler {})
 
-      (with-mocks [m1 {:target 'app.http.access-token/get-token
-                       :return (:token token)}]
-        (handler {})
-
-        (let [token-id (get @request :app.http.access-token/id)]
-          (t/is (= token-id (:id token))))))))
+      (let [token-id (get @request :app.http.access-token/id)]
+        (t/is (= token-id (:id token)))))))
 
 (t/deftest authz-middleware
   (let [profile (th/create-profile* 1)
-        system  (assoc th/*system* ::main/props (:app.setup/props th/*system*))
-
-        token   (db/with-atomic [conn (::db/pool th/*system*)]
-                  (let [system (assoc system ::db/conn conn)]
-                    (app.rpc.commands.access-token/create-access-token
-                     system (:id profile) "test" nil)))
-
+        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
         request (volatile! {})
         handler (#'app.http.access-token/wrap-authz
                  (fn [req] (vreset! request req))
-                 system)]
+                 th/*system*)]
 
     (handler nil)
     (t/is (nil? @request))

backend/test/backend_tests/rpc_file_test.clj

@@ -8,10 +8,10 @@
   (:require
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
-   [app.common.pprint :as pp]
    [app.common.thumbnails :as thc]
    [app.common.types.shape :as cts]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.http :as http]
@@ -123,8 +123,27 @@
                   :components-v2 true}
             out (th/command! data)]
 
-        ;; (th/print-result! out)
 
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+
+        (let [result (:result out)]
+          (t/is (some? (:deleted-at result)))
+          (t/is (= file-id (:id result)))
+          (t/is (= "new name" (:name result)))
+          (t/is (= 1 (count (get-in result [:data :pages]))))
+          (t/is (nil? (:users result))))))
+
+    (th/db-update! :file
+                   {:deleted-at (dt/now)}
+                   {:id file-id})
+
+    (t/testing "query single file after delete and wait"
+      (let [data {::th/type :get-file
+                  ::rpc/profile-id (:id prof)
+                  :id file-id
+                  :components-v2 true}
+            out (th/command! data)]
         (let [error      (:error out)
               error-data (ex-data error)]
           (t/is (th/ex-info? error))
@@ -195,7 +214,7 @@
         (t/is (= 5 (count rows))))
 
       ;; The objects-gc should remove unused fragments
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 3 (:processed res))))
 
       ;; Check the number of fragments
@@ -230,7 +249,7 @@
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
       ;; The objects-gc should remove unused fragments
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 3 (:processed res))))
 
       ;; Check the number of fragments;
@@ -254,7 +273,7 @@
         (t/is (= 4 (count rows)))
         (t/is (= 2 (count (remove (comp some? :deleted-at) rows)))))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       (let [rows (th/db-query :file-data-fragment {:file-id (:id file)})]
@@ -324,8 +343,9 @@
                 :name "image"
                 :frame-id uuid/zero
                 :parent-id uuid/zero
-                :type :image
-                :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}})}])
+                :type :rect
+                :fills [{:fill-opacity 1
+                         :fill-image {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
 
       ;; Check that reference storage objects on filemediaobjects
       ;; are the same because of deduplication feature.
@@ -355,7 +375,7 @@
         (t/is (= 2 (count rows)))
         (t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 3 (:processed res))))
 
       ;; check file media objects
@@ -386,7 +406,7 @@
 
       ;; This only clears fragments, the file media objects still referenced because
       ;; snapshots are preserved
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       ;; Mark all snapshots to be a non-snapshot file change
@@ -395,7 +415,7 @@
 
       ;; Rerun the file-gc and objects-gc
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       ;; Now that file-gc have deleted the file-media-object usage,
@@ -443,7 +463,8 @@
           fmo3    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
           fmo4    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
           fmo5    (add-file-media-object :profile-id (:id profile) :file-id (:id file))
-          s-shid  (uuid/random)
+          s1-shid  (uuid/random)
+          s2-shid  (uuid/random)
           t-shid  (uuid/random)
 
           page-id (first (get-in file [:data :pages]))]
@@ -462,19 +483,31 @@
        :changes
        [{:type :add-obj
          :page-id page-id
-         :id s-shid
+         :id s1-shid
          :parent-id uuid/zero
          :frame-id uuid/zero
          :components-v2 true
          :obj (cts/setup-shape
-               {:id s-shid
+               {:id s1-shid
                 :name "image"
                 :frame-id uuid/zero
                 :parent-id uuid/zero
-                :type :image
-                :metadata {:id (:id fmo1) :width 100 :height 100 :mtype "image/jpeg"}
-                :fills [{:opacity 1 :fill-image {:id (:id fmo2) :width 100 :height 100 :mtype "image/jpeg"}}]
-                :strokes [{:opacity 1 :stroke-image {:id (:id fmo3) :width 100 :height 100 :mtype "image/jpeg"}}]})}
+                :type :rect
+                :fills [{:fill-opacity 1 :fill-image {:id (:id fmo2) :width 101 :height 100 :mtype "image/jpeg"}}]
+                :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo3) :width 102 :height 100 :mtype "image/jpeg"}}]})}
+        {:type :add-obj
+         :page-id page-id
+         :id s2-shid
+         :parent-id uuid/zero
+         :frame-id uuid/zero
+         :components-v2 true
+         :obj (cts/setup-shape
+               {:id s2-shid
+                :name "image"
+                :frame-id uuid/zero
+                :parent-id uuid/zero
+                :type :rect
+                :fills [{:fill-opacity 1 :fill-image {:id (:id fmo1) :width 103 :height 100 :mtype "image/jpeg"}}]})}
         {:type :add-obj
          :page-id page-id
          :id t-shid
@@ -500,7 +533,8 @@
                                                              {:fills [{:fill-opacity 1
                                                                        :fill-color "#000000"}]
                                                               :text "bye"}]}]}]}
-                :strokes [{:opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
+                :strokes [{:stroke-opacity 1 :stroke-image {:id (:id fmo5) :width 100 :height 100 :mtype "image/jpeg"}}]})}])
+
 
       ;; run the file-gc task immediately without forced min-age
       (t/is (false? (th/run-task! :file-gc {:file-id (:id file)})))
@@ -508,7 +542,7 @@
       ;; run the task again
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       (let [rows (th/db-query :file-data-fragment {:file-id (:id file)
@@ -538,10 +572,13 @@
        :vern 0
        :changes [{:type :del-obj
                   :page-id (first (get-in file [:data :pages]))
-                  :id s-shid}
+                  :id s1-shid}
                  {:type :del-obj
                   :page-id (first (get-in file [:data :pages]))
-                  :id t-shid}])
+                  :id t-shid}
+                 {:type :del-obj
+                  :page-id (first (get-in file [:data :pages]))
+                  :id s2-shid}])
 
       ;; Now, we have deleted the usage of pointers to the
       ;; file-media-objects, if we paste file-gc, they should be marked
@@ -550,7 +587,7 @@
 
       ;; This only removes unused fragments, file media are still
       ;; referenced on snapshots.
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       ;; Mark all snapshots to be a non-snapshot file change
@@ -560,7 +597,7 @@
       ;; Rerun file-gc and objects-gc task for the same file once all snapshots are
       ;; "expired/deleted"
       (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 6 (:processed res))))
 
       (let [rows (th/db-query :file-data-fragment {:file-id (:id file)
@@ -712,7 +749,7 @@
       ;; Now that file-gc have marked for deletion the object
       ;; thumbnail lets execute the objects-gc task which remove
       ;; the rows and mark as touched the storage object rows
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 5 (:processed res))))
 
       ;; Now that objects-gc have deleted the object thumbnail lets
@@ -741,7 +778,7 @@
         (t/is (= 1 (count rows)))
         (t/is (= 0 (count (remove (comp some? :deleted-at) rows)))))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         ;; (pp/pprint res)
         (t/is (= 3 (:processed res))))
 
@@ -876,7 +913,7 @@
                                      :profile-id (:id profile1)})]
     ;; file is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of files
@@ -907,7 +944,7 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion (should be noop)
-    (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of file libraries of a after hard deletion
@@ -921,7 +958,7 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 1 (:processed result))))
 
     ;; query the list of file libraries of a after hard deletion
@@ -1176,7 +1213,7 @@
         (t/is (= 2 (count rows)))
         (t/is (= 1 (count (remove :deleted-at rows)))))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 4 (:processed res))))
 
       (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1232,7 +1269,7 @@
         (t/is (= 2 (count rows)))
         (t/is (= 1 (count (remove (comp some? :deleted-at) rows)))))
 
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed res))))
 
       (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
@@ -1251,7 +1288,7 @@
     (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
       ;; Preventive objects-gc
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 1 (:processed result))))
 
       ;; Check the number of fragments before adding the page
@@ -1272,7 +1309,7 @@
       (th/run-pending-tasks!))
 
       ;; Clean objects after file-gc
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 1 (:processed result))))
 
       ;; Check the number of fragments before adding the page
@@ -1324,7 +1361,7 @@
     (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
 
       ;; The objects-gc should remove unused fragments
-    (let [res (th/run-task! :objects-gc {:min-age 0})]
+    (let [res (th/run-task! :objects-gc {})]
       (t/is (= 2 (:processed res))))
 
       ;; Check the number of fragments before adding the page
@@ -1712,6 +1749,7 @@
         [{:fill-image
           {:id (:id fmedia)
            :name "test"
+           :mtype "image/jpeg"
            :width 200
            :height 200}}]]
 
@@ -1820,8 +1858,7 @@
       (t/is (= (:id file-2) (:file-id (get rows 1))))
       (t/is (nil? (:deleted-at (get rows 1)))))
 
-    (th/run-task! :objects-gc
-                  {:min-age 0})
+    (th/run-task! :objects-gc {})
 
     (let [rows (th/db-exec! ["SELECT * FROM file_media_object ORDER BY created_at ASC"])]
       (t/is (= 1 (count rows)))

backend/test/backend_tests/rpc_font_test.clj

@@ -7,6 +7,7 @@
 (ns backend-tests.rpc-font-test
   (:require
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
@@ -144,7 +145,7 @@
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:min-age 0})]
+    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 2 (:processed res))))
 
     (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
@@ -204,7 +205,7 @@
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:min-age 0})]
+    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 1 (:processed res))))
 
     (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
@@ -263,7 +264,7 @@
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:min-age 0})]
+    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 1 (:processed res))))
 
     (let [res (th/run-task! :storage-gc-touched {:min-age 0})]

backend/test/backend_tests/rpc_media_test.clj

@@ -48,7 +48,7 @@
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 122785 (:size mobj1)))
-        (t/is (= 3302 (:size mobj2)))))))
+        (t/is (= 3297 (:size mobj2)))))))
 
 (t/deftest media-object-upload
   (let [prof   (th/create-profile* 1)
@@ -85,7 +85,7 @@
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 312043 (:size mobj1)))
-        (t/is (= 3887   (:size mobj2)))))))
+        (t/is (= 3890   (:size mobj2)))))))
 
 
 (t/deftest media-object-upload-idempotency
@@ -163,7 +163,7 @@
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 122785 (:size mobj1)))
-        (t/is (= 3302 (:size mobj2)))))))
+        (t/is (= 3297 (:size mobj2)))))))
 
 (t/deftest media-object-upload-command
   (let [prof   (th/create-profile* 1)
@@ -200,7 +200,7 @@
         (t/is (sto/object? mobj1))
         (t/is (sto/object? mobj2))
         (t/is (= 312043 (:size mobj1)))
-        (t/is (= 3887   (:size mobj2)))))))
+        (t/is (= 3890   (:size mobj2)))))))
 
 
 (t/deftest media-object-upload-idempotency-command

backend/test/backend_tests/rpc_profile_test.clj

@@ -209,16 +209,16 @@
                     ::rpc/profile-id (:id prof1)
                     :id (:id team1)}
             out    (th/command! params)]
-      ;; (th/print-result! out)
+        ;; (th/print-result! out)
 
         (let [team (th/db-get :team {:id (:id team1)} {::db/remove-deleted false})]
           (t/is (dt/instant? (:deleted-at team)))))
 
-    ;; Request profile to be deleted
+      ;; Request profile to be deleted
       (let [params {::th/type :delete-profile
                     ::rpc/profile-id (:id prof1)}
             out    (th/command! params)]
-      ;; (th/print-result! out)
+        ;; (th/print-result! out)
         (t/is (nil? (:result out)))
         (t/is (nil? (:error out)))))))
 
@@ -379,15 +379,14 @@
 (t/deftest prepare-register-and-register-profile-1
   (let [data  {::th/type :prepare-register-profile
                :email "user@example.com"
+               :fullname "foobar"
                :password "foobar"}
         out   (th/command! data)
         token (get-in out [:result :token])]
     (t/is (string? token))
 
     ;; try register without token
-    (let [data  {::th/type :register-profile
-                 :fullname "foobar"
-                 :accept-terms-and-privacy true}
+    (let [data  {::th/type :register-profile}
           out   (th/command! data)]
       ;; (th/print-result! out)
       (let [error (:error out)]
@@ -398,11 +397,8 @@
     ;; try correct register
     (let [data  {::th/type :register-profile
                  :token token
-                 :fullname "foobar"
                  :utm_campaign "utma"
-                 :mtm_campaign "mtma"
-                 :accept-terms-and-privacy true
-                 :accept-newsletter-subscription true}]
+                 :mtm_campaign "mtma"}]
       (let [{:keys [result error]} (th/command! data)]
         (t/is (nil? error))))
 
@@ -424,6 +420,7 @@
       ;; PREPARE REGISTER
       (let [data  {::th/type :prepare-register-profile
                    :email "hello@example.com"
+                   :fullname "foobar"
                    :password "foobar"}
             out   (th/command! data)
             token (get-in out [:result :token])]
@@ -432,10 +429,7 @@
 
       ;; DO REGISTRATION
       (let [data  {::th/type :register-profile
-                   :token @current-token
-                   :fullname "foobar"
-                   :accept-terms-and-privacy true
-                   :accept-newsletter-subscription true}
+                   :token @current-token}
             out   (th/command! data)]
         (t/is (nil? (:error out)))
         (t/is (= 1 (:call-count @mock))))
@@ -445,6 +439,7 @@
       ;; PREPARE REGISTER: second attempt
       (let [data  {::th/type :prepare-register-profile
                    :email "hello@example.com"
+                   :fullname "foobar"
                    :password "foobar"}
             out   (th/command! data)
             token (get-in out [:result :token])]
@@ -479,6 +474,7 @@
       ;; PREPARE REGISTER
       (let [data  {::th/type :prepare-register-profile
                    :email "hello@example.com"
+                   :fullname "foobar"
                    :password "foobar"}
             out   (th/command! data)
             token (get-in out [:result :token])]
@@ -487,10 +483,7 @@
 
       ;; DO REGISTRATION
       (let [data  {::th/type :register-profile
-                   :token @current-token
-                   :fullname "foobar"
-                   :accept-terms-and-privacy true
-                   :accept-newsletter-subscription true}
+                   :token @current-token}
             out   (th/command! data)]
         (t/is (nil? (:error out)))
         (t/is (= 1 (:call-count @mock))))
@@ -504,6 +497,7 @@
       ;; PREPARE REGISTER: second attempt
       (let [data  {::th/type :prepare-register-profile
                    :email "hello@example.com"
+                   :fullname "foobar"
                    :password "foobar"}
             out   (th/command! data)
             token (get-in out [:result :token])]
@@ -514,10 +508,7 @@
                       :return true}]
         ;; DO REGISTRATION: second attempt
         (let [data  {::th/type :register-profile
-                     :token @current-token
-                     :fullname "foobar"
-                     :accept-terms-and-privacy true
-                     :accept-newsletter-subscription true}
+                     :token @current-token}
               out   (th/command! data)]
           (t/is (nil? (:error out)))
           (t/is (= 0 (:call-count @mock))))))))
@@ -532,6 +523,7 @@
                                  :member-email "user@example.com"})
         data  {::th/type :prepare-register-profile
                :invitation-token itoken
+               :fullname "foobar"
                :email "user@example.com"
                :password "foobar"}
 
@@ -542,8 +534,7 @@
 
     (let [rtoken (:token result)
           data   {::th/type :register-profile
-                  :token rtoken
-                  :fullname "foobar"}
+                  :token rtoken}
 
           {:keys [result error] :as out} (th/command! data)]
         ;; (th/print-result! out)
@@ -563,6 +554,7 @@
         data  {::th/type :prepare-register-profile
                :invitation-token itoken
                :email "user@example.com"
+               :fullname "foobar"
                :password "foobar"}
         out   (th/command! data)]
 
@@ -582,6 +574,7 @@
                                    :member-email "user@example.com"})
           data  {::th/type :prepare-register-profile
                  :invitation-token itoken
+                 :fullname "foobar"
                  :email "user@example.com"
                  :password "foobar"}
           out (th/command! data)]
@@ -604,6 +597,7 @@
           data  {::th/type :prepare-register-profile
                  :invitation-token itoken
                  :email "user@example.com"
+                 :fullname "foobar"
                  :password "foobar"}
           out   (th/command! data)]
 
@@ -624,6 +618,7 @@
 
           data  {::th/type :prepare-register-profile
                  :invitation-token itoken
+                 :fullname "foobar"
                  :email "user@example.com"
                  :password "foobar"}
           out   (th/command! data)]
@@ -636,6 +631,7 @@
 (t/deftest prepare-register-with-registration-disabled
   (with-redefs [app.config/flags #{}]
     (let [data  {::th/type :prepare-register-profile
+                 :fullname "foobar"
                  :email "user@example.com"
                  :password "foobar"}
           out  (th/command! data)]
@@ -648,6 +644,7 @@
 (t/deftest prepare-register-with-existing-user
   (let [profile (th/create-profile* 1)
         data    {::th/type :prepare-register-profile
+                 :fullname "foobar"
                  :email (:email profile)
                  :password "foobar"}
         out     (th/command! data)]
@@ -660,6 +657,7 @@
 
   (let [pool  (:app.db/pool th/*system*)
         data  {::th/type :prepare-register-profile
+               :fullname "foobar"
                :email "user@example.com"
                :password "foobar"}]
 
@@ -674,6 +672,7 @@
 (t/deftest register-profile-with-complained-email
   (let [pool  (:app.db/pool th/*system*)
         data  {::th/type :prepare-register-profile
+               :fullname "foobar"
                :email "user@example.com"
                :password "foobar"}]
 
@@ -688,6 +687,7 @@
 
 (t/deftest register-profile-with-email-as-password
   (let [data {::th/type :prepare-register-profile
+              :fullname "foobar"
               :email "user@example.com"
               :password "USER@example.com"}
         out  (th/command! data)]

backend/test/backend_tests/rpc_project_test.clj

@@ -7,6 +7,7 @@
 (ns backend-tests.rpc-project-test
   (:require
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
@@ -178,7 +179,7 @@
 
     ;; project is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of projects
@@ -210,7 +211,7 @@
         (t/is (= 1 (count result)))))
 
     ;; run permanent deletion (should be noop)
-    (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of files of a after soft deletion
@@ -224,7 +225,7 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 1 (:processed result))))
 
     ;; query the list of files of a after hard deletion

backend/test/backend_tests/rpc_team_test.clj

@@ -8,6 +8,7 @@
   (:require
    [app.common.logging :as l]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
@@ -449,6 +450,26 @@
       (t/is (nil? res)))))
 
 
+(t/deftest get-owned-teams
+  (let [profile1 (th/create-profile* 1 {:is-active true})
+        profile2 (th/create-profile* 2 {:is-active true})
+        team1    (th/create-team* 1 {:profile-id (:id profile1)})
+        team2    (th/create-team* 2 {:profile-id (:id profile2)})
+
+        params   {::th/type :get-owned-teams
+                  ::rpc/profile-id (:id profile1)}
+        out      (th/command! params)]
+
+    ;; (th/print-result! out)
+    (t/is (th/success? out))
+    (let [[item1 :as result] (:result out)]
+      (t/is (= 1 (count result)))
+      (t/is (= (:id team1) (:id item1)))
+      (t/is (= 1 (:total-members item1)))
+      (t/is (= 1 (:total-editors item1)))
+      (t/is (not= (:default-team-id profile1) (:id item1))))))
+
+
 (t/deftest team-deletion-1
   (let [profile1 (th/create-profile* 1 {:is-active true})
         team     (th/create-team* 1 {:profile-id (:id profile1)})
@@ -459,7 +480,7 @@
 
     ;; team is not deleted because it does not meet all
     ;; conditions to be deleted.
-    (let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of teams
@@ -493,7 +514,7 @@
     (th/run-pending-tasks!)
 
     ;; run permanent deletion (should be noop)
-    (let [result (th/run-task! :objects-gc {:min-age (dt/duration {:minutes 1})})]
+    (let [result (th/run-task! :objects-gc {})]
       (t/is (= 0 (:processed result))))
 
     ;; query the list of projects after hard deletion
@@ -507,7 +528,7 @@
         (t/is (= :not-found (:type edata)))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:min-age (dt/duration 0)})]
+    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 2 (:processed result))))
 
     ;; query the list of projects of a after hard deletion
@@ -521,7 +542,6 @@
       (let [edata (-> out :error ex-data)]
         (t/is (= :not-found (:type edata)))))))
 
-
 (t/deftest team-deletion-2
   (let [storage (-> (:app.storage/storage th/*system*)
                     (assoc ::sto/backend :assets-fs))
@@ -564,7 +584,7 @@
       (t/is (= 1 (count rows)))
       (t/is (dt/instant? (:deleted-at (first rows)))))
 
-    (let [result (th/run-task! :objects-gc {:min-age 0})]
+    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
       (t/is (= 5 (:processed result))))))
 
 (t/deftest create-team-access-request

backend/yarn.lock

@@ -19,25 +19,34 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@npmcli/agent@npm:^2.0.0":
-  version: 2.2.2
-  resolution: "@npmcli/agent@npm:2.2.2"
+"@isaacs/fs-minipass@npm:^4.0.0":
+  version: 4.0.1
+  resolution: "@isaacs/fs-minipass@npm:4.0.1"
+  dependencies:
+    minipass: "npm:^7.0.4"
+  checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2
+  languageName: node
+  linkType: hard
+
+"@npmcli/agent@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "@npmcli/agent@npm:3.0.0"
   dependencies:
     agent-base: "npm:^7.1.0"
     http-proxy-agent: "npm:^7.0.0"
     https-proxy-agent: "npm:^7.0.1"
     lru-cache: "npm:^10.0.1"
     socks-proxy-agent: "npm:^8.0.3"
-  checksum: 10c0/325e0db7b287d4154ecd164c0815c08007abfb07653cc57bceded17bb7fd240998a3cbdbe87d700e30bef494885eccc725ab73b668020811d56623d145b524ae
+  checksum: 10c0/efe37b982f30740ee77696a80c196912c274ecd2cb243bc6ae7053a50c733ce0f6c09fda085145f33ecf453be19654acca74b69e81eaad4c90f00ccffe2f9271
   languageName: node
   linkType: hard
 
-"@npmcli/fs@npm:^3.1.0":
-  version: 3.1.1
-  resolution: "@npmcli/fs@npm:3.1.1"
+"@npmcli/fs@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "@npmcli/fs@npm:4.0.0"
   dependencies:
     semver: "npm:^7.3.5"
-  checksum: 10c0/c37a5b4842bfdece3d14dfdb054f73fe15ed2d3da61b34ff76629fb5b1731647c49166fd2a8bf8b56fcfa51200382385ea8909a3cbecdad612310c114d3f6c99
+  checksum: 10c0/c90935d5ce670c87b6b14fab04a965a3b8137e585f8b2a6257263bd7f97756dd736cb165bb470e5156a9e718ecd99413dccc54b1138c1a46d6ec7cf325982fe5
   languageName: node
   linkType: hard
 
@@ -48,29 +57,17 @@ __metadata:
   languageName: node
   linkType: hard
 
-"abbrev@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "abbrev@npm:2.0.0"
-  checksum: 10c0/f742a5a107473946f426c691c08daba61a1d15942616f300b5d32fd735be88fef5cba24201757b6c407fd564555fb48c751cfa33519b2605c8a7aadd22baf372
+"abbrev@npm:^3.0.0":
+  version: 3.0.1
+  resolution: "abbrev@npm:3.0.1"
+  checksum: 10c0/21ba8f574ea57a3106d6d35623f2c4a9111d9ee3e9a5be47baed46ec2457d2eac46e07a5c4a60186f88cb98abbe3e24f2d4cca70bc2b12f1692523e2209a9ccf
   languageName: node
   linkType: hard
 
-"agent-base@npm:^7.0.2, agent-base@npm:^7.1.0, agent-base@npm:^7.1.1":
-  version: 7.1.1
-  resolution: "agent-base@npm:7.1.1"
-  dependencies:
-    debug: "npm:^4.3.4"
-  checksum: 10c0/e59ce7bed9c63bf071a30cc471f2933862044c97fd9958967bfe22521d7a0f601ce4ed5a8c011799d0c726ca70312142ae193bbebb60f576b52be19d4a363b50
-  languageName: node
-  linkType: hard
-
-"aggregate-error@npm:^3.0.0":
-  version: 3.1.0
-  resolution: "aggregate-error@npm:3.1.0"
-  dependencies:
-    clean-stack: "npm:^2.0.0"
-    indent-string: "npm:^4.0.0"
-  checksum: 10c0/a42f67faa79e3e6687a4923050e7c9807db3848a037076f791d10e092677d65c1d2d863b7848560699f40fc0502c19f40963fb1cd1fb3d338a7423df8e45e039
+"agent-base@npm:^7.1.0, agent-base@npm:^7.1.2":
+  version: 7.1.3
+  resolution: "agent-base@npm:7.1.3"
+  checksum: 10c0/6192b580c5b1d8fb399b9c62bf8343d76654c2dd62afcb9a52b2cf44a8b6ace1e3b704d3fe3547d91555c857d3df02603341ff2cb961b9cfe2b12f9f3c38ee11
   languageName: node
   linkType: hard
 
@@ -82,9 +79,9 @@ __metadata:
   linkType: hard
 
 "ansi-regex@npm:^6.0.1":
-  version: 6.0.1
-  resolution: "ansi-regex@npm:6.0.1"
-  checksum: 10c0/cbe16dbd2c6b2735d1df7976a7070dd277326434f0212f43abf6d87674095d247968209babdaad31bb00882fa68807256ba9be340eec2f1004de14ca75f52a08
+  version: 6.1.0
+  resolution: "ansi-regex@npm:6.1.0"
+  checksum: 10c0/a91daeddd54746338478eef88af3439a7edf30f8e23196e2d6ed182da9add559c601266dbef01c2efa46a958ad6f1f8b176799657616c702b5b02e799e7fd8dc
   languageName: node
   linkType: hard
 
@@ -141,21 +138,21 @@ __metadata:
   linkType: hard
 
 "brace-expansion@npm:^1.1.7":
-  version: 1.1.11
-  resolution: "brace-expansion@npm:1.1.11"
+  version: 1.1.12
+  resolution: "brace-expansion@npm:1.1.12"
   dependencies:
     balanced-match: "npm:^1.0.0"
     concat-map: "npm:0.0.1"
-  checksum: 10c0/695a56cd058096a7cb71fb09d9d6a7070113c7be516699ed361317aca2ec169f618e28b8af352e02ab4233fb54eb0168460a40dc320bab0034b36ab59aaad668
+  checksum: 10c0/975fecac2bb7758c062c20d0b3b6288c7cc895219ee25f0a64a9de662dbac981ff0b6e89909c3897c1f84fa353113a721923afdec5f8b2350255b097f12b1f73
   languageName: node
   linkType: hard
 
 "brace-expansion@npm:^2.0.1":
-  version: 2.0.1
-  resolution: "brace-expansion@npm:2.0.1"
+  version: 2.0.2
+  resolution: "brace-expansion@npm:2.0.2"
   dependencies:
     balanced-match: "npm:^1.0.0"
-  checksum: 10c0/b358f2fe060e2d7a87aa015979ecea07f3c37d4018f8d6deb5bd4c229ad3a0384fe6029bb76cd8be63c81e516ee52d1a0673edbe2023d53a5191732ae3c3e49f
+  checksum: 10c0/6d117a4c793488af86b83172deb6af143e94c17bc53b0b3cec259733923b4ca84679d506ac261f4ba3c7ed37c46018e2ff442f9ce453af8643ecd64f4a54e6cf
   languageName: node
   linkType: hard
 
@@ -175,11 +172,11 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cacache@npm:^18.0.0":
-  version: 18.0.3
-  resolution: "cacache@npm:18.0.3"
+"cacache@npm:^19.0.1":
+  version: 19.0.1
+  resolution: "cacache@npm:19.0.1"
   dependencies:
-    "@npmcli/fs": "npm:^3.1.0"
+    "@npmcli/fs": "npm:^4.0.0"
     fs-minipass: "npm:^3.0.0"
     glob: "npm:^10.2.2"
     lru-cache: "npm:^10.0.1"
@@ -187,11 +184,11 @@ __metadata:
     minipass-collect: "npm:^2.0.1"
     minipass-flush: "npm:^1.0.5"
     minipass-pipeline: "npm:^1.2.4"
-    p-map: "npm:^4.0.0"
-    ssri: "npm:^10.0.0"
-    tar: "npm:^6.1.11"
-    unique-filename: "npm:^3.0.0"
-  checksum: 10c0/dfda92840bb371fb66b88c087c61a74544363b37a265023223a99965b16a16bbb87661fe4948718d79df6e0cc04e85e62784fbcf1832b2a5e54ff4c46fbb45b7
+    p-map: "npm:^7.0.2"
+    ssri: "npm:^12.0.0"
+    tar: "npm:^7.4.3"
+    unique-filename: "npm:^4.0.0"
+  checksum: 10c0/01f2134e1bd7d3ab68be851df96c8d63b492b1853b67f2eecb2c37bb682d37cb70bb858a16f2f0554d3c0071be6dfe21456a1ff6fa4b7eed996570d6a25ffe9c
   languageName: node
   linkType: hard
 
@@ -214,17 +211,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"chownr@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "chownr@npm:2.0.0"
-  checksum: 10c0/594754e1303672171cc04e50f6c398ae16128eb134a88f801bf5354fd96f205320f23536a045d9abd8b51024a149696e51231565891d4efdab8846021ecf88e6
-  languageName: node
-  linkType: hard
-
-"clean-stack@npm:^2.0.0":
-  version: 2.2.0
-  resolution: "clean-stack@npm:2.2.0"
-  checksum: 10c0/1f90262d5f6230a17e27d0c190b09d47ebe7efdd76a03b5a1127863f7b3c9aec4c3e6c8bb3a7bbf81d553d56a1fd35728f5a8ef4c63f867ac8d690109742a8c1
+"chownr@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "chownr@npm:3.0.0"
+  checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10
   languageName: node
   linkType: hard
 
@@ -251,26 +241,26 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cross-spawn@npm:^7.0.0":
-  version: 7.0.3
-  resolution: "cross-spawn@npm:7.0.3"
+"cross-spawn@npm:^7.0.6":
+  version: 7.0.6
+  resolution: "cross-spawn@npm:7.0.6"
   dependencies:
     path-key: "npm:^3.1.0"
     shebang-command: "npm:^2.0.0"
     which: "npm:^2.0.1"
-  checksum: 10c0/5738c312387081c98d69c98e105b6327b069197f864a60593245d64c8089c8a0a744e16349281210d56835bb9274130d825a78b2ad6853ca13cfbeffc0c31750
+  checksum: 10c0/053ea8b2135caff68a9e81470e845613e374e7309a47731e81639de3eaeb90c3d01af0e0b44d2ab9d50b43467223b88567dfeb3262db942dc063b9976718ffc1
   languageName: node
   linkType: hard
 
 "debug@npm:4, debug@npm:^4, debug@npm:^4.3.4":
-  version: 4.3.4
-  resolution: "debug@npm:4.3.4"
+  version: 4.4.1
+  resolution: "debug@npm:4.4.1"
   dependencies:
-    ms: "npm:2.1.2"
+    ms: "npm:^2.1.3"
   peerDependenciesMeta:
     supports-color:
       optional: true
-  checksum: 10c0/cedbec45298dd5c501d01b92b119cd3faebe5438c3917ff11ae1bff86a6c722930ac9c8659792824013168ba6db7c4668225d845c633fbdafbbf902a6389f736
+  checksum: 10c0/d2b44bc1afd912b49bb7ebb0d50a860dc93a4dd7d946e8de94abc957bb63726b7dd5aa48c18c2386c379ec024c46692e15ed3ed97d481729f929201e671fcd55
   languageName: node
   linkType: hard
 
@@ -319,9 +309,21 @@ __metadata:
   linkType: hard
 
 "exponential-backoff@npm:^3.1.1":
-  version: 3.1.1
-  resolution: "exponential-backoff@npm:3.1.1"
-  checksum: 10c0/160456d2d647e6019640bd07111634d8c353038d9fa40176afb7cd49b0548bdae83b56d05e907c2cce2300b81cae35d800ef92fefb9d0208e190fa3b7d6bb579
+  version: 3.1.2
+  resolution: "exponential-backoff@npm:3.1.2"
+  checksum: 10c0/d9d3e1eafa21b78464297df91f1776f7fbaa3d5e3f7f0995648ca5b89c069d17055033817348d9f4a43d1c20b0eab84f75af6991751e839df53e4dfd6f22e844
+  languageName: node
+  linkType: hard
+
+"fdir@npm:^6.4.4":
+  version: 6.4.6
+  resolution: "fdir@npm:6.4.6"
+  peerDependencies:
+    picomatch: ^3 || ^4
+  peerDependenciesMeta:
+    picomatch:
+      optional: true
+  checksum: 10c0/45b559cff889934ebb8bc498351e5acba40750ada7e7d6bde197768d2fa67c149be8ae7f8ff34d03f4e1eb20f2764116e56440aaa2f6689e9a4aa7ef06acafe9
   languageName: node
   linkType: hard
 
@@ -335,21 +337,12 @@ __metadata:
   linkType: hard
 
 "foreground-child@npm:^3.1.0":
-  version: 3.1.1
-  resolution: "foreground-child@npm:3.1.1"
+  version: 3.3.1
+  resolution: "foreground-child@npm:3.3.1"
   dependencies:
-    cross-spawn: "npm:^7.0.0"
+    cross-spawn: "npm:^7.0.6"
     signal-exit: "npm:^4.0.1"
-  checksum: 10c0/9700a0285628abaeb37007c9a4d92bd49f67210f09067638774338e146c8e9c825c5c877f072b2f75f41dc6a2d0be8664f79ffc03f6576649f54a84fb9b47de0
-  languageName: node
-  linkType: hard
-
-"fs-minipass@npm:^2.0.0":
-  version: 2.1.0
-  resolution: "fs-minipass@npm:2.1.0"
-  dependencies:
-    minipass: "npm:^3.0.0"
-  checksum: 10c0/703d16522b8282d7299337539c3ed6edddd1afe82435e4f5b76e34a79cd74e488a8a0e26a636afc2440e1a23b03878e2122e3a2cfe375a5cf63c37d92b86a004
+  checksum: 10c0/8986e4af2430896e65bc2788d6679067294d6aee9545daefc84923a0a4b399ad9c7a3ea7bd8c0b2b80fdf4a92de4c69df3f628233ff3224260e9c1541a9e9ed3
   languageName: node
   linkType: hard
 
@@ -390,18 +383,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"glob@npm:^10.2.2, glob@npm:^10.3.10":
-  version: 10.3.16
-  resolution: "glob@npm:10.3.16"
+"glob@npm:^10.2.2":
+  version: 10.4.5
+  resolution: "glob@npm:10.4.5"
   dependencies:
     foreground-child: "npm:^3.1.0"
     jackspeak: "npm:^3.1.2"
-    minimatch: "npm:^9.0.1"
-    minipass: "npm:^7.0.4"
-    path-scurry: "npm:^1.11.0"
+    minimatch: "npm:^9.0.4"
+    minipass: "npm:^7.1.2"
+    package-json-from-dist: "npm:^1.0.0"
+    path-scurry: "npm:^1.11.1"
   bin:
     glob: dist/esm/bin.mjs
-  checksum: 10c0/f7eb4c3e66f221f0be3967c02527047167967549bdf8ed1bd5f6277d43a35191af4e2bb8c89f07a79664958bae088fd06659e69a0f1de462972f1eab52a715e8
+  checksum: 10c0/19a9759ea77b8e3ca0a43c2f07ecddc2ad46216b786bb8f993c445aee80d345925a21e5280c7b7c6c59e860a0154b84e4b2b60321fea92cd3c56b4a7489f160e
   languageName: node
   linkType: hard
 
@@ -420,9 +414,9 @@ __metadata:
   linkType: hard
 
 "http-cache-semantics@npm:^4.1.1":
-  version: 4.1.1
-  resolution: "http-cache-semantics@npm:4.1.1"
-  checksum: 10c0/ce1319b8a382eb3cbb4a37c19f6bfe14e5bb5be3d09079e885e8c513ab2d3cd9214902f8a31c9dc4e37022633ceabfc2d697405deeaf1b8f3552bb4ed996fdfc
+  version: 4.2.0
+  resolution: "http-cache-semantics@npm:4.2.0"
+  checksum: 10c0/45b66a945cf13ec2d1f29432277201313babf4a01d9e52f44b31ca923434083afeca03f18417f599c9ab3d0e7b618ceb21257542338b57c54b710463b4a53e37
   languageName: node
   linkType: hard
 
@@ -437,12 +431,12 @@ __metadata:
   linkType: hard
 
 "https-proxy-agent@npm:^7.0.1":
-  version: 7.0.4
-  resolution: "https-proxy-agent@npm:7.0.4"
+  version: 7.0.6
+  resolution: "https-proxy-agent@npm:7.0.6"
   dependencies:
-    agent-base: "npm:^7.0.2"
+    agent-base: "npm:^7.1.2"
     debug: "npm:4"
-  checksum: 10c0/bc4f7c38da32a5fc622450b6cb49a24ff596f9bd48dcedb52d2da3fa1c1a80e100fb506bd59b326c012f21c863c69b275c23de1a01d0b84db396822fdf25e52b
+  checksum: 10c0/f729219bc735edb621fa30e6e84e60ee5d00802b8247aac0d7b79b0bd6d4b3294737a337b93b86a0bd9e68099d031858a39260c976dc14cdbba238ba1f8779ac
   languageName: node
   linkType: hard
 
@@ -469,13 +463,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"indent-string@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "indent-string@npm:4.0.0"
-  checksum: 10c0/1e1904ddb0cb3d6cce7cd09e27a90184908b7a5d5c21b92e232c93579d314f0b83c246ffb035493d0504b1e9147ba2c9b21df0030f48673fba0496ecd698161f
-  languageName: node
-  linkType: hard
-
 "ip-address@npm:^9.0.5":
   version: 9.0.5
   resolution: "ip-address@npm:9.0.5"
@@ -518,13 +505,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"is-lambda@npm:^1.0.1":
-  version: 1.0.1
-  resolution: "is-lambda@npm:1.0.1"
-  checksum: 10c0/85fee098ae62ba6f1e24cf22678805473c7afd0fb3978a3aa260e354cb7bcb3a5806cf0a98403188465efedec41ab4348e8e4e79305d409601323855b3839d4d
-  languageName: node
-  linkType: hard
-
 "is-number@npm:^7.0.0":
   version: 7.0.0
   resolution: "is-number@npm:7.0.0"
@@ -547,15 +527,15 @@ __metadata:
   linkType: hard
 
 "jackspeak@npm:^3.1.2":
-  version: 3.1.2
-  resolution: "jackspeak@npm:3.1.2"
+  version: 3.4.3
+  resolution: "jackspeak@npm:3.4.3"
   dependencies:
     "@isaacs/cliui": "npm:^8.0.2"
     "@pkgjs/parseargs": "npm:^0.11.0"
   dependenciesMeta:
     "@pkgjs/parseargs":
       optional: true
-  checksum: 10c0/5f1922a1ca0f19869e23f0dc4374c60d36e922f7926c76fecf8080cc6f7f798d6a9caac1b9428327d14c67731fd551bb3454cb270a5e13a0718f3b3660ec3d5d
+  checksum: 10c0/6acc10d139eaefdbe04d2f679e6191b3abf073f111edf10b1de5302c97ec93fffeb2fdd8681ed17f16268aa9dd4f8c588ed9d1d3bffbbfa6e8bf897cbb3149b9
   languageName: node
   linkType: hard
 
@@ -567,36 +547,35 @@ __metadata:
   linkType: hard
 
 "lru-cache@npm:^10.0.1, lru-cache@npm:^10.2.0":
-  version: 10.2.2
-  resolution: "lru-cache@npm:10.2.2"
-  checksum: 10c0/402d31094335851220d0b00985084288136136992979d0e015f0f1697e15d1c86052d7d53ae86b614e5b058425606efffc6969a31a091085d7a2b80a8a1e26d6
+  version: 10.4.3
+  resolution: "lru-cache@npm:10.4.3"
+  checksum: 10c0/ebd04fbca961e6c1d6c0af3799adcc966a1babe798f685bb84e6599266599cd95d94630b10262f5424539bc4640107e8a33aa28585374abf561d30d16f4b39fb
   languageName: node
   linkType: hard
 
 "luxon@npm:^3.4.4":
-  version: 3.4.4
-  resolution: "luxon@npm:3.4.4"
-  checksum: 10c0/02e26a0b039c11fd5b75e1d734c8f0332c95510f6a514a9a0991023e43fb233884da02d7f966823ffb230632a733fc86d4a4b1e63c3fbe00058b8ee0f8c728af
+  version: 3.6.1
+  resolution: "luxon@npm:3.6.1"
+  checksum: 10c0/906d57a9dc4d1de9383f2e9223e378c298607c1b4d17b6657b836a3cd120feb1c1de3b5d06d846a3417e1ca764de8476e8c23b3cd4083b5cdb870adcb06a99d5
   languageName: node
   linkType: hard
 
-"make-fetch-happen@npm:^13.0.0":
-  version: 13.0.1
-  resolution: "make-fetch-happen@npm:13.0.1"
+"make-fetch-happen@npm:^14.0.3":
+  version: 14.0.3
+  resolution: "make-fetch-happen@npm:14.0.3"
   dependencies:
-    "@npmcli/agent": "npm:^2.0.0"
-    cacache: "npm:^18.0.0"
+    "@npmcli/agent": "npm:^3.0.0"
+    cacache: "npm:^19.0.1"
     http-cache-semantics: "npm:^4.1.1"
-    is-lambda: "npm:^1.0.1"
     minipass: "npm:^7.0.2"
-    minipass-fetch: "npm:^3.0.0"
+    minipass-fetch: "npm:^4.0.0"
     minipass-flush: "npm:^1.0.5"
     minipass-pipeline: "npm:^1.2.4"
-    negotiator: "npm:^0.6.3"
-    proc-log: "npm:^4.2.0"
+    negotiator: "npm:^1.0.0"
+    proc-log: "npm:^5.0.0"
     promise-retry: "npm:^2.0.1"
-    ssri: "npm:^10.0.0"
-  checksum: 10c0/df5f4dbb6d98153b751bccf4dc4cc500de85a96a9331db9805596c46aa9f99d9555983954e6c1266d9f981ae37a9e4647f42b9a4bb5466f867f4012e582c9e7e
+    ssri: "npm:^12.0.0"
+  checksum: 10c0/c40efb5e5296e7feb8e37155bde8eb70bc57d731b1f7d90e35a092fde403d7697c56fb49334d92d330d6f1ca29a98142036d6480a12681133a0a1453164cb2f0
   languageName: node
   linkType: hard
 
@@ -609,12 +588,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"minimatch@npm:^9.0.1":
-  version: 9.0.4
-  resolution: "minimatch@npm:9.0.4"
+"minimatch@npm:^9.0.4":
+  version: 9.0.5
+  resolution: "minimatch@npm:9.0.5"
   dependencies:
     brace-expansion: "npm:^2.0.1"
-  checksum: 10c0/2c16f21f50e64922864e560ff97c587d15fd491f65d92a677a344e970fe62aafdbeafe648965fa96d33c061b4d0eabfe0213466203dd793367e7f28658cf6414
+  checksum: 10c0/de96cf5e35bdf0eab3e2c853522f98ffbe9a36c37797778d2665231ec1f20a9447a7e567cb640901f89e4daaa95ae5d70c65a9e8aa2bb0019b6facbc3c0575ed
   languageName: node
   linkType: hard
 
@@ -627,18 +606,18 @@ __metadata:
   languageName: node
   linkType: hard
 
-"minipass-fetch@npm:^3.0.0":
-  version: 3.0.5
-  resolution: "minipass-fetch@npm:3.0.5"
+"minipass-fetch@npm:^4.0.0":
+  version: 4.0.1
+  resolution: "minipass-fetch@npm:4.0.1"
   dependencies:
     encoding: "npm:^0.1.13"
     minipass: "npm:^7.0.3"
     minipass-sized: "npm:^1.0.3"
-    minizlib: "npm:^2.1.2"
+    minizlib: "npm:^3.0.1"
   dependenciesMeta:
     encoding:
       optional: true
-  checksum: 10c0/9d702d57f556274286fdd97e406fc38a2f5c8d15e158b498d7393b1105974b21249289ec571fa2b51e038a4872bfc82710111cf75fae98c662f3d6f95e72152b
+  checksum: 10c0/a3147b2efe8e078c9bf9d024a0059339c5a09c5b1dded6900a219c218cc8b1b78510b62dae556b507304af226b18c3f1aeb1d48660283602d5b6586c399eed5c
   languageName: node
   linkType: hard
 
@@ -678,76 +657,68 @@ __metadata:
   languageName: node
   linkType: hard
 
-"minipass@npm:^5.0.0":
-  version: 5.0.0
-  resolution: "minipass@npm:5.0.0"
-  checksum: 10c0/a91d8043f691796a8ac88df039da19933ef0f633e3d7f0d35dcd5373af49131cf2399bfc355f41515dc495e3990369c3858cd319e5c2722b4753c90bf3152462
+"minipass@npm:^5.0.0 || ^6.0.2 || ^7.0.0, minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.0.4, minipass@npm:^7.1.2":
+  version: 7.1.2
+  resolution: "minipass@npm:7.1.2"
+  checksum: 10c0/b0fd20bb9fb56e5fa9a8bfac539e8915ae07430a619e4b86ff71f5fc757ef3924b23b2c4230393af1eda647ed3d75739e4e0acb250a6b1eb277cf7f8fe449557
   languageName: node
   linkType: hard
 
-"minipass@npm:^5.0.0 || ^6.0.2 || ^7.0.0, minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.0.4":
-  version: 7.1.1
-  resolution: "minipass@npm:7.1.1"
-  checksum: 10c0/fdccc2f99c31083f45f881fd1e6971d798e333e078ab3c8988fb818c470fbd5e935388ad9adb286397eba50baebf46ef8ff487c8d3f455a69c6f3efc327bdff9
-  languageName: node
-  linkType: hard
-
-"minizlib@npm:^2.1.1, minizlib@npm:^2.1.2":
-  version: 2.1.2
-  resolution: "minizlib@npm:2.1.2"
+"minizlib@npm:^3.0.1":
+  version: 3.0.2
+  resolution: "minizlib@npm:3.0.2"
   dependencies:
-    minipass: "npm:^3.0.0"
-    yallist: "npm:^4.0.0"
-  checksum: 10c0/64fae024e1a7d0346a1102bb670085b17b7f95bf6cfdf5b128772ec8faf9ea211464ea4add406a3a6384a7d87a0cd1a96263692134323477b4fb43659a6cab78
+    minipass: "npm:^7.1.2"
+  checksum: 10c0/9f3bd35e41d40d02469cb30470c55ccc21cae0db40e08d1d0b1dff01cc8cc89a6f78e9c5d2b7c844e485ec0a8abc2238111213fdc5b2038e6d1012eacf316f78
   languageName: node
   linkType: hard
 
-"mkdirp@npm:^1.0.3":
-  version: 1.0.4
-  resolution: "mkdirp@npm:1.0.4"
+"mkdirp@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "mkdirp@npm:3.0.1"
   bin:
-    mkdirp: bin/cmd.js
-  checksum: 10c0/46ea0f3ffa8bc6a5bc0c7081ffc3907777f0ed6516888d40a518c5111f8366d97d2678911ad1a6882bf592fa9de6c784fea32e1687bb94e1f4944170af48a5cf
+    mkdirp: dist/cjs/src/bin.js
+  checksum: 10c0/9f2b975e9246351f5e3a40dcfac99fcd0baa31fbfab615fe059fb11e51f10e4803c63de1f384c54d656e4db31d000e4767e9ef076a22e12a641357602e31d57d
   languageName: node
   linkType: hard
 
-"ms@npm:2.1.2":
-  version: 2.1.2
-  resolution: "ms@npm:2.1.2"
-  checksum: 10c0/a437714e2f90dbf881b5191d35a6db792efbca5badf112f87b9e1c712aace4b4b9b742dd6537f3edf90fd6f684de897cec230abde57e87883766712ddda297cc
+"ms@npm:^2.1.3":
+  version: 2.1.3
+  resolution: "ms@npm:2.1.3"
+  checksum: 10c0/d924b57e7312b3b63ad21fc5b3dc0af5e78d61a1fc7cfb5457edaf26326bf62be5307cc87ffb6862ef1c2b33b0233cdb5d4f01c4c958cc0d660948b65a287a48
   languageName: node
   linkType: hard
 
-"negotiator@npm:^0.6.3":
-  version: 0.6.3
-  resolution: "negotiator@npm:0.6.3"
-  checksum: 10c0/3ec9fd413e7bf071c937ae60d572bc67155262068ed522cf4b3be5edbe6ddf67d095ec03a3a14ebf8fc8e95f8e1d61be4869db0dbb0de696f6b837358bd43fc2
+"negotiator@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "negotiator@npm:1.0.0"
+  checksum: 10c0/4c559dd52669ea48e1914f9d634227c561221dd54734070791f999c52ed0ff36e437b2e07d5c1f6e32909fc625fe46491c16e4a8f0572567d4dd15c3a4fda04b
   languageName: node
   linkType: hard
 
 "node-gyp@npm:latest":
-  version: 10.1.0
-  resolution: "node-gyp@npm:10.1.0"
+  version: 11.2.0
+  resolution: "node-gyp@npm:11.2.0"
   dependencies:
     env-paths: "npm:^2.2.0"
     exponential-backoff: "npm:^3.1.1"
-    glob: "npm:^10.3.10"
     graceful-fs: "npm:^4.2.6"
-    make-fetch-happen: "npm:^13.0.0"
-    nopt: "npm:^7.0.0"
-    proc-log: "npm:^3.0.0"
+    make-fetch-happen: "npm:^14.0.3"
+    nopt: "npm:^8.0.0"
+    proc-log: "npm:^5.0.0"
     semver: "npm:^7.3.5"
-    tar: "npm:^6.1.2"
-    which: "npm:^4.0.0"
+    tar: "npm:^7.4.3"
+    tinyglobby: "npm:^0.2.12"
+    which: "npm:^5.0.0"
   bin:
     node-gyp: bin/node-gyp.js
-  checksum: 10c0/9cc821111ca244a01fb7f054db7523ab0a0cd837f665267eb962eb87695d71fb1e681f9e21464cc2fd7c05530dc4c81b810bca1a88f7d7186909b74477491a3c
+  checksum: 10c0/bd8d8c76b06be761239b0c8680f655f6a6e90b48e44d43415b11c16f7e8c15be346fba0cbf71588c7cdfb52c419d928a7d3db353afc1d952d19756237d8f10b9
   languageName: node
   linkType: hard
 
 "nodemon@npm:^3.1.2":
-  version: 3.1.2
-  resolution: "nodemon@npm:3.1.2"
+  version: 3.1.10
+  resolution: "nodemon@npm:3.1.10"
   dependencies:
     chokidar: "npm:^3.5.2"
     debug: "npm:^4"
@@ -761,18 +732,18 @@ __metadata:
     undefsafe: "npm:^2.0.5"
   bin:
     nodemon: bin/nodemon.js
-  checksum: 10c0/7a091067d766768fb6660b796194b01748bba5dc3f1e3ed3dd5f804bfa305e207d24635755078ee5e7cc53848cea35204901e0a6e51ac64483bb8e9ecb237c95
+  checksum: 10c0/95b64d647f2c22e85e375b250517b0a4b32c2d2392ad898444e331f70d6b1ab43b17f53a8a1d68d5879ab8401fc6cd6e26f0d2a8736240984f6b5a8435b407c0
   languageName: node
   linkType: hard
 
-"nopt@npm:^7.0.0":
-  version: 7.2.1
-  resolution: "nopt@npm:7.2.1"
+"nopt@npm:^8.0.0":
+  version: 8.1.0
+  resolution: "nopt@npm:8.1.0"
   dependencies:
-    abbrev: "npm:^2.0.0"
+    abbrev: "npm:^3.0.0"
   bin:
     nopt: bin/nopt.js
-  checksum: 10c0/a069c7c736767121242037a22a788863accfa932ab285a1eb569eb8cd534b09d17206f68c37f096ae785647435e0c5a5a0a67b42ec743e481a455e5ae6a6df81
+  checksum: 10c0/62e9ea70c7a3eb91d162d2c706b6606c041e4e7b547cbbb48f8b3695af457dd6479904d7ace600856bf923dd8d1ed0696f06195c8c20f02ac87c1da0e1d315ef
   languageName: node
   linkType: hard
 
@@ -783,12 +754,17 @@ __metadata:
   languageName: node
   linkType: hard
 
-"p-map@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "p-map@npm:4.0.0"
-  dependencies:
-    aggregate-error: "npm:^3.0.0"
-  checksum: 10c0/592c05bd6262c466ce269ff172bb8de7c6975afca9b50c975135b974e9bdaafbfe80e61aaaf5be6d1200ba08b30ead04b88cfa7e25ff1e3b93ab28c9f62a2c75
+"p-map@npm:^7.0.2":
+  version: 7.0.3
+  resolution: "p-map@npm:7.0.3"
+  checksum: 10c0/46091610da2b38ce47bcd1d8b4835a6fa4e832848a6682cf1652bc93915770f4617afc844c10a77d1b3e56d2472bb2d5622353fa3ead01a7f42b04fc8e744a5c
+  languageName: node
+  linkType: hard
+
+"package-json-from-dist@npm:^1.0.0":
+  version: 1.0.1
+  resolution: "package-json-from-dist@npm:1.0.1"
+  checksum: 10c0/62ba2785eb655fec084a257af34dbe24292ab74516d6aecef97ef72d4897310bc6898f6c85b5cd22770eaa1ce60d55a0230e150fb6a966e3ecd6c511e23d164b
   languageName: node
   linkType: hard
 
@@ -799,7 +775,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"path-scurry@npm:^1.11.0":
+"path-scurry@npm:^1.11.1":
   version: 1.11.1
   resolution: "path-scurry@npm:1.11.1"
   dependencies:
@@ -816,17 +792,17 @@ __metadata:
   languageName: node
   linkType: hard
 
-"proc-log@npm:^3.0.0":
-  version: 3.0.0
-  resolution: "proc-log@npm:3.0.0"
-  checksum: 10c0/f66430e4ff947dbb996058f6fd22de2c66612ae1a89b097744e17fb18a4e8e7a86db99eda52ccf15e53f00b63f4ec0b0911581ff2aac0355b625c8eac509b0dc
+"picomatch@npm:^4.0.2":
+  version: 4.0.2
+  resolution: "picomatch@npm:4.0.2"
+  checksum: 10c0/7c51f3ad2bb42c776f49ebf964c644958158be30d0a510efd5a395e8d49cb5acfed5b82c0c5b365523ce18e6ab85013c9ebe574f60305892ec3fa8eee8304ccc
   languageName: node
   linkType: hard
 
-"proc-log@npm:^4.2.0":
-  version: 4.2.0
-  resolution: "proc-log@npm:4.2.0"
-  checksum: 10c0/17db4757c2a5c44c1e545170e6c70a26f7de58feb985091fb1763f5081cab3d01b181fb2dd240c9f4a4255a1d9227d163d5771b7e69c9e49a561692db865efb9
+"proc-log@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "proc-log@npm:5.0.0"
+  checksum: 10c0/bbe5edb944b0ad63387a1d5b1911ae93e05ce8d0f60de1035b218cdcceedfe39dbd2c697853355b70f1a090f8f58fe90da487c85216bf9671f9499d1a897e9e3
   languageName: node
   linkType: hard
 
@@ -878,11 +854,11 @@ __metadata:
   linkType: hard
 
 "semver@npm:^7.3.5, semver@npm:^7.5.3":
-  version: 7.6.2
-  resolution: "semver@npm:7.6.2"
+  version: 7.7.2
+  resolution: "semver@npm:7.7.2"
   bin:
     semver: bin/semver.js
-  checksum: 10c0/97d3441e97ace8be4b1976433d1c32658f6afaff09f143e52c593bae7eef33de19e3e369c88bd985ce1042c6f441c80c6803078d1de2a9988080b66684cbb30c
+  checksum: 10c0/aca305edfbf2383c22571cb7714f48cadc7ac95371b4b52362fb8eeffdfbc0de0669368b82b2b15978f8848f01d7114da65697e56cd8c37b0dab8c58e543f9ea
   languageName: node
   linkType: hard
 
@@ -926,23 +902,23 @@ __metadata:
   linkType: hard
 
 "socks-proxy-agent@npm:^8.0.3":
-  version: 8.0.3
-  resolution: "socks-proxy-agent@npm:8.0.3"
+  version: 8.0.5
+  resolution: "socks-proxy-agent@npm:8.0.5"
   dependencies:
-    agent-base: "npm:^7.1.1"
+    agent-base: "npm:^7.1.2"
     debug: "npm:^4.3.4"
-    socks: "npm:^2.7.1"
-  checksum: 10c0/4950529affd8ccd6951575e21c1b7be8531b24d924aa4df3ee32df506af34b618c4e50d261f4cc603f1bfd8d426915b7d629966c8ce45b05fb5ad8c8b9a6459d
+    socks: "npm:^2.8.3"
+  checksum: 10c0/5d2c6cecba6821389aabf18728325730504bf9bb1d9e342e7987a5d13badd7a98838cc9a55b8ed3cb866ad37cc23e1086f09c4d72d93105ce9dfe76330e9d2a6
   languageName: node
   linkType: hard
 
-"socks@npm:^2.7.1":
-  version: 2.8.3
-  resolution: "socks@npm:2.8.3"
+"socks@npm:^2.8.3":
+  version: 2.8.5
+  resolution: "socks@npm:2.8.5"
   dependencies:
     ip-address: "npm:^9.0.5"
     smart-buffer: "npm:^4.2.0"
-  checksum: 10c0/d54a52bf9325165770b674a67241143a3d8b4e4c8884560c4e0e078aace2a728dffc7f70150660f51b85797c4e1a3b82f9b7aa25e0a0ceae1a243365da5c51a7
+  checksum: 10c0/e427d0eb0451cfd04e20b9156ea8c0e9b5e38a8d70f21e55c30fbe4214eda37cfc25d782c63f9adc5fbdad6d062a0f127ef2cefc9a44b6fee2b9ea5d1ed10827
   languageName: node
   linkType: hard
 
@@ -970,12 +946,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ssri@npm:^10.0.0":
-  version: 10.0.6
-  resolution: "ssri@npm:10.0.6"
+"ssri@npm:^12.0.0":
+  version: 12.0.0
+  resolution: "ssri@npm:12.0.0"
   dependencies:
     minipass: "npm:^7.0.3"
-  checksum: 10c0/e5a1e23a4057a86a97971465418f22ea89bd439ac36ade88812dd920e4e61873e8abd6a9b72a03a67ef50faa00a2daf1ab745c5a15b46d03e0544a0296354227
+  checksum: 10c0/caddd5f544b2006e88fa6b0124d8d7b28208b83c72d7672d5ade44d794525d23b540f3396108c4eb9280dcb7c01f0bef50682f5b4b2c34291f7c5e211fd1417d
   languageName: node
   linkType: hard
 
@@ -1028,17 +1004,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tar@npm:^6.1.11, tar@npm:^6.1.2":
-  version: 6.2.1
-  resolution: "tar@npm:6.2.1"
+"tar@npm:^7.4.3":
+  version: 7.4.3
+  resolution: "tar@npm:7.4.3"
   dependencies:
-    chownr: "npm:^2.0.0"
-    fs-minipass: "npm:^2.0.0"
-    minipass: "npm:^5.0.0"
-    minizlib: "npm:^2.1.1"
-    mkdirp: "npm:^1.0.3"
-    yallist: "npm:^4.0.0"
-  checksum: 10c0/a5eca3eb50bc11552d453488344e6507156b9193efd7635e98e867fab275d527af53d8866e2370cd09dfe74378a18111622ace35af6a608e5223a7d27fe99537
+    "@isaacs/fs-minipass": "npm:^4.0.0"
+    chownr: "npm:^3.0.0"
+    minipass: "npm:^7.1.2"
+    minizlib: "npm:^3.0.1"
+    mkdirp: "npm:^3.0.1"
+    yallist: "npm:^5.0.0"
+  checksum: 10c0/d4679609bb2a9b48eeaf84632b6d844128d2412b95b6de07d53d8ee8baf4ca0857c9331dfa510390a0727b550fd543d4d1a10995ad86cdf078423fbb8d99831d
+  languageName: node
+  linkType: hard
+
+"tinyglobby@npm:^0.2.12":
+  version: 0.2.14
+  resolution: "tinyglobby@npm:0.2.14"
+  dependencies:
+    fdir: "npm:^6.4.4"
+    picomatch: "npm:^4.0.2"
+  checksum: 10c0/f789ed6c924287a9b7d3612056ed0cda67306cd2c80c249fd280cf1504742b12583a2089b61f4abbd24605f390809017240e250241f09938054c9b363e51c0a6
   languageName: node
   linkType: hard
 
@@ -1067,21 +1053,21 @@ __metadata:
   languageName: node
   linkType: hard
 
-"unique-filename@npm:^3.0.0":
-  version: 3.0.0
-  resolution: "unique-filename@npm:3.0.0"
+"unique-filename@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "unique-filename@npm:4.0.0"
   dependencies:
-    unique-slug: "npm:^4.0.0"
-  checksum: 10c0/6363e40b2fa758eb5ec5e21b3c7fb83e5da8dcfbd866cc0c199d5534c42f03b9ea9ab069769cc388e1d7ab93b4eeef28ef506ab5f18d910ef29617715101884f
+    unique-slug: "npm:^5.0.0"
+  checksum: 10c0/38ae681cceb1408ea0587b6b01e29b00eee3c84baee1e41fd5c16b9ed443b80fba90c40e0ba69627e30855570a34ba8b06702d4a35035d4b5e198bf5a64c9ddc
   languageName: node
   linkType: hard
 
-"unique-slug@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "unique-slug@npm:4.0.0"
+"unique-slug@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "unique-slug@npm:5.0.0"
   dependencies:
     imurmurhash: "npm:^0.1.4"
-  checksum: 10c0/cb811d9d54eb5821b81b18205750be84cb015c20a4a44280794e915f5a0a70223ce39066781a354e872df3572e8155c228f43ff0cce94c7cbf4da2cc7cbdd635
+  checksum: 10c0/d324c5a44887bd7e105ce800fcf7533d43f29c48757ac410afd42975de82cc38ea2035c0483f4de82d186691bf3208ef35c644f73aa2b1b20b8e651be5afd293
   languageName: node
   linkType: hard
 
@@ -1096,14 +1082,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"which@npm:^4.0.0":
-  version: 4.0.0
-  resolution: "which@npm:4.0.0"
+"which@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "which@npm:5.0.0"
   dependencies:
     isexe: "npm:^3.1.1"
   bin:
     node-which: bin/which.js
-  checksum: 10c0/449fa5c44ed120ccecfe18c433296a4978a7583bf2391c50abce13f76878d2476defde04d0f79db8165bdf432853c1f8389d0485ca6e8ebce3bbcded513d5e6a
+  checksum: 10c0/e556e4cd8b7dbf5df52408c9a9dd5ac6518c8c5267c8953f5b0564073c66ed5bf9503b14d876d0e9c7844d4db9725fb0dcf45d6e911e17e26ab363dc3965ae7b
   languageName: node
   linkType: hard
 
@@ -1130,8 +1116,8 @@ __metadata:
   linkType: hard
 
 "ws@npm:^8.17.0":
-  version: 8.17.0
-  resolution: "ws@npm:8.17.0"
+  version: 8.18.2
+  resolution: "ws@npm:8.18.2"
   peerDependencies:
     bufferutil: ^4.0.1
     utf-8-validate: ">=5.0.2"
@@ -1140,7 +1126,7 @@ __metadata:
       optional: true
     utf-8-validate:
       optional: true
-  checksum: 10c0/55241ec93a66fdfc4bf4f8bc66c8eb038fda2c7a4ee8f6f157f2ca7dc7aa76aea0c0da0bf3adb2af390074a70a0e45456a2eaf80e581e630b75df10a64b0a990
+  checksum: 10c0/4b50f67931b8c6943c893f59c524f0e4905bbd183016cfb0f2b8653aa7f28dad4e456b9d99d285bbb67cca4fedd9ce90dfdfaa82b898a11414ebd66ee99141e4
   languageName: node
   linkType: hard
 
@@ -1150,3 +1136,10 @@ __metadata:
   checksum: 10c0/2286b5e8dbfe22204ab66e2ef5cc9bbb1e55dfc873bbe0d568aa943eb255d131890dfd5bf243637273d31119b870f49c18fcde2c6ffbb7a7a092b870dc90625a
   languageName: node
   linkType: hard
+
+"yallist@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "yallist@npm:5.0.0"
+  checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416
+  languageName: node
+  linkType: hard

common/deps.edn

@@ -1,10 +1,10 @@
 {:deps
- {org.clojure/clojure {:mvn/version "1.12.0"}
+ {org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/data.json {:mvn/version "2.5.1"}
   org.clojure/tools.cli {:mvn/version "1.1.230"}
-  org.clojure/clojurescript {:mvn/version "1.11.132"}
   org.clojure/test.check {:mvn/version "1.1.1"}
   org.clojure/data.fressian {:mvn/version "1.1.0"}
+  org.clojure/clojurescript {:mvn/version "1.12.42"}
 
   ;; Logging
   org.apache.logging.log4j/log4j-api {:mvn/version "2.24.3"}
@@ -12,14 +12,14 @@
   org.apache.logging.log4j/log4j-web {:mvn/version "2.24.3"}
   org.apache.logging.log4j/log4j-jul {:mvn/version "2.24.3"}
   org.apache.logging.log4j/log4j-slf4j2-impl {:mvn/version "2.24.3"}
-  org.slf4j/slf4j-api {:mvn/version "2.0.16"}
+  org.slf4j/slf4j-api {:mvn/version "2.0.17"}
   pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.32"}
 
-  selmer/selmer {:mvn/version "1.12.61"}
+  selmer/selmer {:mvn/version "1.12.62"}
   criterium/criterium {:mvn/version "0.4.6"}
 
   metosin/jsonista {:mvn/version "0.3.13"}
-  metosin/malli {:mvn/version "0.17.0"}
+  metosin/malli {:mvn/version "0.18.0"}
 
   expound/expound {:mvn/version "0.9.0"}
   com.cognitect/transit-clj {:mvn/version "1.0.333"}
@@ -28,9 +28,9 @@
   integrant/integrant {:mvn/version "0.13.1"}
 
   funcool/tubax {:mvn/version "2021.05.20-0"}
-  funcool/cuerdas {:mvn/version "2023.11.09-407"}
+  funcool/cuerdas {:mvn/version "2025.06.16-414"}
   funcool/promesa
-  {:git/sha "0c5ed6ad033515a2df4b55addea044f60e9653d0"
+  {:git/sha "f52f58cfacf62f59eab717e2637f37729d0cc383"
    :git/url "https://github.com/funcool/promesa"}
 
   funcool/datoteka
@@ -59,7 +59,7 @@
  {:dev
   {:extra-deps
    {org.clojure/tools.namespace {:mvn/version "RELEASE"}
-    thheller/shadow-cljs {:mvn/version "2.28.20"}
+    thheller/shadow-cljs {:mvn/version "3.1.5"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
     com.bhauman/rebel-readline {:mvn/version "RELEASE"}
     criterium/criterium {:mvn/version "RELEASE"}
@@ -68,7 +68,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.6" :git/sha "52cf7d6"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test
@@ -76,9 +76,9 @@
    :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
 
   :shadow-cljs
-  {:main-opts ["-m" "shadow.cljs.devtools.cli"]}
+  {:main-opts ["-m" "shadow.cljs.devtools.cli"]
+   :jvm-opts ["--sun-misc-unsafe-memory-access=allow"]}
 
   :outdated
   {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
    :main-opts ["-m" "antq.core"]}}}
-

common/package.json

@@ -4,22 +4,20 @@
   "license": "MPL-2.0",
   "author": "Kaleidos INC",
   "private": true,
-  "packageManager": "yarn@4.6.0+sha512.5383cc12567a95f1d668fbe762dfe0075c595b4bfff433be478dbbe24e05251a8e8c3eb992a986667c1d53b6c3a9c85b8398c35a960587fbd9fa3a0915406728",
+  "packageManager": "yarn@4.9.2+sha512.1fc009bc09d13cfd0e19efa44cbfc2b9cf6ca61482725eb35bbc5e257e093ebf4130db6dfe15d604ff4b79efd8e1e8e99b25fa7d0a6197c9f9826358d4d65c3c",
   "type": "module",
   "repository": {
     "type": "git",
     "url": "https://github.com/penpot/penpot"
   },
   "dependencies": {
-    "luxon": "^3.4.4",
-    "sax": "^1.4.1"
+    "luxon": "^3.6.1"
   },
   "devDependencies": {
-    "concurrently": "^9.0.1",
-    "nodemon": "^3.1.7",
-    "shadow-cljs": "2.28.20",
+    "concurrently": "^9.1.2",
+    "nodemon": "^3.1.10",
     "source-map-support": "^0.5.21",
-    "ws": "^8.17.0"
+    "ws": "^8.18.2"
   },
   "scripts": {
     "fmt:clj:check": "cljfmt check --parallel=false src/ test/",

common/scripts/repl

@@ -2,16 +2,20 @@
 
 export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS"
 
-export OPTIONS="
-       -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J-XX:+EnableDynamicAgentLoading \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full"
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
+       -Djdk.tracePinnedThreads=full \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow                    \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
+
+export OPTIONS="-A:dev"
 
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"

common/src/app/common/buffer.cljc

@@ -0,0 +1,165 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.common.buffer
+  "A collection of helpers and macros for work with byte buffers"
+  (:refer-clojure :exclude [clone])
+  (:require
+   [app.common.uuid :as uuid])
+  #?(:cljs
+     (:require-macros [app.common.buffer])
+     :clj
+     (:import [java.nio ByteBuffer ByteOrder])))
+
+(defmacro read-byte
+  [target offset]
+  (if (:ns &env)
+    `(.getInt8 ~target ~offset true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(long (.get ~target ~offset)))))
+
+(defmacro read-bool
+  [target offset]
+  (if (:ns &env)
+    `(== 1 (.getInt8 ~target ~offset true))
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(== 1 (.get ~target ~offset)))))
+
+(defmacro read-short
+  [target offset]
+  (if (:ns &env)
+    `(.getInt16 ~target ~offset true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.getShort ~target ~offset))))
+
+(defmacro read-int
+  [target offset]
+  (if (:ns &env)
+    `(.getInt32 ~target ~offset true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(long (.getInt ~target ~offset)))))
+
+(defmacro read-float
+  [target offset]
+  (if (:ns &env)
+    `(.getFloat32 ~target ~offset true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(double (.getFloat ~target ~offset)))))
+
+(defmacro read-uuid
+  [target offset]
+  (if (:ns &env)
+    `(let [a# (.getUint32 ~target (+ ~offset 0) true)
+           b# (.getUint32 ~target (+ ~offset 4) true)
+           c# (.getUint32 ~target (+ ~offset 8) true)
+           d# (.getUint32 ~target (+ ~offset 12) true)]
+       (uuid/from-unsigned-parts a# b# c# d#))
+
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(try
+         (.order ~target ByteOrder/BIG_ENDIAN)
+         (let [msb# (.getLong ~target (+ ~offset 0))
+               lsb# (.getLong ~target (+ ~offset 8))]
+           (java.util.UUID. (long msb#) (long lsb#)))
+         (finally
+           (.order ~target ByteOrder/LITTLE_ENDIAN))))))
+
+(defmacro write-byte
+  [target offset value]
+  (if (:ns &env)
+    `(.setInt8 ~target ~offset ~value true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.put ~target ~offset (unchecked-byte ~value)))))
+
+(defmacro write-short
+  [target offset value]
+  (if (:ns &env)
+    `(.setInt16 ~target ~offset ~value true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.putShort ~target ~offset (unchecked-short ~value)))))
+
+(defmacro write-int
+  [target offset value]
+  (if (:ns &env)
+    `(.setInt32 ~target ~offset ~value true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.putInt ~target ~offset (unchecked-int ~value)))))
+
+(defmacro write-float
+  [target offset value]
+  (if (:ns &env)
+    `(.setFloat32 ~target ~offset ~value true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.putFloat ~target ~offset (unchecked-float ~value)))))
+
+(defmacro write-uuid
+  [target offset value]
+  (if (:ns &env)
+    `(let [barray# (uuid/get-u32 ~value)]
+       (.setUint32 ~target (+ ~offset 0)  (aget barray# 0) true)
+       (.setUint32 ~target (+ ~offset 4)  (aget barray# 1) true)
+       (.setUint32 ~target (+ ~offset 8)  (aget barray# 2) true)
+       (.setUint32 ~target (+ ~offset 12) (aget barray# 3) true))
+
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
+          value  (with-meta value {:tag 'java.util.UUID})]
+      `(try
+         (.order ~target ByteOrder/BIG_ENDIAN)
+         (.putLong ~target (+ ~offset 0) (.getMostSignificantBits ~value))
+         (.putLong ~target (+ ~offset 8) (.getLeastSignificantBits ~value))
+         (finally
+           (.order ~target ByteOrder/LITTLE_ENDIAN))))))
+
+(defn allocate
+  [size]
+  #?(:clj (let [buffer (ByteBuffer/allocate (int size))]
+            (.order buffer ByteOrder/LITTLE_ENDIAN))
+     :cljs (new js/DataView (new js/ArrayBuffer size))))
+
+(defn clone
+  [buffer]
+  #?(:clj
+     (let [src (.array ^ByteBuffer buffer)
+           len (alength ^bytes src)
+           dst (byte-array len)]
+       (System/arraycopy src 0 dst 0 len)
+       (let [buffer (ByteBuffer/wrap dst)]
+         (.order buffer ByteOrder/LITTLE_ENDIAN)))
+     :cljs
+     (let [buffer'  (.-buffer ^js/DataView buffer)
+           src-view (js/Uint32Array. buffer')
+           dst-buff (js/ArrayBuffer. (.-byteLength buffer'))
+           dst-view (js/Uint32Array. dst-buff)]
+       (.set dst-view src-view)
+       (js/DataView. dst-buff))))
+
+(defn equals?
+  [buffer-a buffer-b]
+  #?(:clj
+     (.equals ^ByteBuffer buffer-a
+              ^ByteBuffer buffer-b)
+
+     :cljs
+     (let [buffer-a (.-buffer buffer-a)
+           buffer-b (.-buffer buffer-b)]
+       (if (= (.-byteLength buffer-a)
+              (.-byteLength buffer-b))
+         (let [cb (js/Uint32Array. buffer-a)
+               ob (js/Uint32Array. buffer-b)
+               sz (alength cb)]
+           (loop [i 0]
+             (if (< i sz)
+               (if (== (aget ob i)
+                       (aget cb i))
+                 (recur (inc i))
+                 false)
+               true)))
+         false))))
+
+(defn buffer?
+  [o]
+  #?(:clj (instance? ByteBuffer o)
+     :cljs (instance? js/DataView o)))

common/src/app/common/colors.cljc

@@ -342,13 +342,20 @@
   (-> (hex->hsl data)
       (conj opacity)))
 
-#?(:cljs
-   (defn format-hsla
-     [[h s l a]]
-     (let [precision 2
-           rounded-s (* 100 (parse-double (d/format-precision s precision)))
-           rounded-l (* 100 (parse-double (d/format-precision l precision)))]
-       (str/concat "" h ", " rounded-s "%, " rounded-l "%, " a))))
+(defn format-hsla
+  [[h s l a]]
+  (let [precision 2
+        rounded-h (int h)
+        rounded-s (d/format-number (* 100 s) precision)
+        rounded-l (d/format-number (* 100 l) precision)
+        rounded-a (d/format-number a precision)]
+    (str/concat "" rounded-h ", " rounded-s "%, " rounded-l "%, " rounded-a)))
+
+(defn format-rgba
+  [[r g b a]]
+  (let [precision 2
+        rounded-a (d/format-number a precision)]
+    (str/ffmt "%, %, %, %" r g b rounded-a)))
 
 (defn- hue->rgb
   "Helper for hsl->rgb"

common/src/app/common/data.cljc

@@ -33,6 +33,16 @@
 (def boolean-or-nil?
   (some-fn nil? boolean?))
 
+(defn in-range?
+  [size i]
+  (and (< i size) (>= i 0)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Commonly used transducers
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def xf:map-id (map :id))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Data Structures
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -82,15 +92,22 @@
   "Assoc a k v pair, in the order position just before the other key."
   [o ks k v before-k]
   (let [f (fn [o']
-            (cond-> (reduce
-                     (fn [acc [k' v']]
-                       (cond
-                         (and before-k (= k' before-k)) (assoc acc k v k' v')
-                         (= k k') acc
-                         :else (assoc acc k' v')))
-                     (ordered-map)
-                     o')
-              (not before-k) (assoc k v)))]
+            (let [found  (volatile! false)
+                  result (reduce
+                          (fn [acc [k' v']]
+                            (cond
+                              (and before-k (= k' before-k))
+                              (do
+                                (vreset! found true)
+                                (assoc acc k v k' v'))
+
+                              (= k k') acc
+                              :else (assoc acc k' v')))
+                          (ordered-map)
+                          o')]
+              (if (or (not before-k) (not @found))
+                (assoc result k v)
+                result)))]
     (if (seq ks)
       (oupdate-in o ks f)
       (f o))))
@@ -1007,27 +1024,35 @@
 (def ^:const trail-zeros-regex-1 #"\.0+$")
 (def ^:const trail-zeros-regex-2 #"(\.\d*[^0])0+$")
 
-#?(:cljs
-   (defn format-precision
-     "Creates a number with predetermined precision and then removes the trailing 0.
+(defn format-precision
+  "Creates a number with predetermined precision and then removes the trailing 0.
   Examples:
     12.0123, 0 => 12
     12.0123, 1 => 12
     12.0123, 2 => 12.01"
-     [num precision]
+  [num precision]
 
-     (if (number? num)
-       (try
-         (let [num-str (mth/to-fixed num precision)
+  (if (number? num)
+    (try
+      (let [num-str (mth/to-fixed num precision)
                ;; Remove all trailing zeros after the comma 100.00000
-               num-str (str/replace num-str trail-zeros-regex-1 "")]
+            num-str (str/replace num-str trail-zeros-regex-1 "")]
            ;; Remove trailing zeros after a decimal number: 0.001|00|
-           (if-let [m (re-find trail-zeros-regex-2 num-str)]
-             (str/replace num-str (first m) (second m))
-             num-str))
-         (catch :default _
-           (str num)))
-       (str num))))
+        (if-let [m (re-find trail-zeros-regex-2 num-str)]
+          (str/replace num-str (first m) (second m))
+          num-str))
+      (catch #?(:clj Throwable :cljs :default) _
+        (str num)))
+    (str num)))
+
+(defn format-number
+  ([value]
+   (format-number value nil))
+  ([value {:keys [precision] :or {precision 2}}]
+   (let [value (if (string? value) (parse-double value) value)]
+     (when (num? value)
+       (let [value (format-precision value precision)]
+         (str value))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Util protocols

common/src/app/common/data/macros.cljc

@@ -6,7 +6,7 @@
 
 (ns app.common.data.macros
   "Data retrieval & manipulation specific macros."
-  (:refer-clojure :exclude [get-in select-keys str with-open min max])
+  (:refer-clojure :exclude [get-in select-keys str with-open max])
   #?(:cljs (:require-macros [app.common.data.macros]))
   (:require
    #?(:clj [clojure.core :as c]
@@ -144,3 +144,8 @@
                  (str "expr assert: " (pr-str expr)))]
      (when *assert*
        `(runtime-assert ~hint (fn [] ~expr))))))
+
+(defn truncate
+  "Truncates a string to a certain length"
+  [s max-length]
+  (subs s 0 (min max-length (count s))))

common/src/app/common/data/undo_stack.cljc

@@ -47,7 +47,7 @@
 
 (defn undo
   [stack]
-  (update stack :index dec))
+  (update stack :index #(max 0 (dec %))))
 
 (defn redo
   [{index :index items :items :as stack}]

common/src/app/common/exceptions.cljc

@@ -47,10 +47,26 @@
     `(try ~@exprs (catch Throwable e# nil))))
 
 (defmacro try!
-  [& exprs]
-  (if (:ns &env)
-    `(try ~@exprs (catch :default e# e#))
-    `(try ~@exprs (catch Throwable e# e#))))
+  [expr & {:keys [reraise-with on-exception]}]
+  (let [ex-sym
+        (gensym "exc")
+
+        generate-catch
+        (fn []
+          (cond
+            (map? reraise-with)
+            `(ex/raise ~@(mapcat identity reraise-with) :cause ~ex-sym)
+
+            on-exception
+            `(let [handler# ~on-exception]
+               (handler# ~ex-sym))
+
+            :else
+            ex-sym))]
+
+    (if (:ns &env)
+      `(try ~expr (catch :default ~ex-sym ~(generate-catch)))
+      `(try ~expr (catch Throwable ~ex-sym ~(generate-catch))))))
 
 (defn ex-info?
   [v]

common/src/app/common/features.cljc

@@ -46,21 +46,30 @@
   #{"fdata/objects-map"
     "fdata/pointer-map"
     "fdata/shape-data-type"
+    "fdata/path-data"
     "components/v2"
     "styles/v2"
     "layout/grid"
     "plugins/runtime"
     "design-tokens/v1"
     "text-editor/v2"
-    "render-wasm/v1"})
+    "render-wasm/v1"
+    "variants/v1"})
 
 ;; A set of features enabled by default
 (def default-features
   #{"fdata/shape-data-type"
+    "fdata/path-data"
     "styles/v2"
     "layout/grid"
     "components/v2"
-    "plugins/runtime"})
+    "plugins/runtime"
+    "design-tokens/v1"})
+
+;; A set of features that should not be propagated to team on creating
+;; or modifying a file
+(def no-team-inheritable-features
+  #{"fdata/path-data"})
 
 ;; A set of features which only affects on frontend and can be enabled
 ;; and disabled freely by the user any time. This features does not
@@ -83,12 +92,12 @@
 ;; be applied (per example backend can operate in both modes with or
 ;; without migration applied)
 (def no-migration-features
-  (-> #{"fdata/objects-map"
-        "fdata/pointer-map"
-        "layout/grid"
+  (-> #{"layout/grid"
+        "design-tokens/v1"
         "fdata/shape-data-type"
-        "design-tokens/v1"}
-      (into frontend-only-features)))
+        "fdata/path-data"}
+      (into frontend-only-features)
+      (into backend-only-features)))
 
 (sm/register!
  ^{::sm/type ::features}
@@ -102,15 +111,14 @@
   "Translate a flag to a feature name"
   [flag]
   (case flag
-    :feature-components-v2 "components/v2"
     :feature-styles-v2 "styles/v2"
-    :feature-grid-layout "layout/grid"
     :feature-fdata-objects-map "fdata/objects-map"
     :feature-fdata-pointer-map "fdata/pointer-map"
     :feature-plugins "plugins/runtime"
     :feature-design-tokens "design-tokens/v1"
     :feature-text-editor-v2 "text-editor/v2"
     :feature-render-wasm "render-wasm/v1"
+    :feature-variants "variants/v1"
     nil))
 
 (defn migrate-legacy-features
@@ -155,7 +163,6 @@
         team-features    (into #{} xf-remove-ephimeral (:features team))]
     (-> enabled-features
         (set/intersection no-migration-features)
-        (set/difference frontend-only-features)
         (set/union team-features))))
 
 (defn check-client-features!
@@ -164,6 +171,8 @@
   frontend client"
   [enabled-features client-features]
   (when (set? client-features)
+    ;; Check if client declares support for features enabled on
+    ;; backend side
     (let [not-supported (-> enabled-features
                             (set/difference client-features)
                             (set/difference frontend-only-features)
@@ -173,14 +182,6 @@
                   :code :feature-not-supported
                   :feature (first not-supported)
                   :hint (str/ffmt "client declares no support for '%' features"
-                                  (str/join "," not-supported)))))
-
-    (let [not-supported (set/difference client-features supported-features)]
-      (when (seq not-supported)
-        (ex/raise :type :restriction
-                  :code :feature-not-supported
-                  :feature (first not-supported)
-                  :hint (str/ffmt "backend does not support '%' features requested by client"
                                   (str/join "," not-supported))))))
 
   enabled-features)
@@ -191,57 +192,55 @@
   supported by the current backend"
   [enabled-features]
   (let [not-supported (set/difference enabled-features supported-features)]
-    (when (seq not-supported)
+    (when-let [not-supported (first not-supported)]
       (ex/raise :type :restriction
                 :code :feature-not-supported
-                :feature (first not-supported)
-                :hint (str/ffmt "features '%' not supported"
-                                (str/join "," not-supported)))))
-  enabled-features)
+                :feature not-supported
+                :hint (str/ffmt "feature '%' not supported on this backend" not-supported)))
+    enabled-features))
 
 (defn check-file-features!
   "Function used for check feature compability between currently
   enabled features set on backend with the provided featured set by
   the penpot file"
-  ([enabled-features file-features]
-   (check-file-features! enabled-features file-features #{}))
-  ([enabled-features file-features client-features]
-   (let [file-features   (into #{} xf-remove-ephimeral file-features)
-         ;; We should ignore all features that does not match with the
-         ;; `no-migration-features` set because we can't enable them
-         ;; as-is, because they probably need migrations
-         client-features (set/intersection client-features no-migration-features)]
-     (let [not-supported (-> enabled-features
-                             (set/union client-features)
-                             (set/difference file-features)
-                             ;; NOTE: we don't want to raise a feature-mismatch
-                             ;; exception for features which don't require an
-                             ;; explicit file migration process or has no real
-                             ;; effect on file data structure
-                             (set/difference no-migration-features))]
-       (when (seq not-supported)
-         (ex/raise :type :restriction
-                   :code :file-feature-mismatch
-                   :feature (first not-supported)
-                   :hint (str/ffmt "enabled features '%' not present in file (missing migration)"
-                                   (str/join "," not-supported)))))
+  [enabled-features file-features]
+  (let [file-features (into #{} xf-remove-ephimeral file-features)
+        not-supported (-> enabled-features
+                          (set/difference file-features)
+                          ;; NOTE: we don't want to raise a feature-mismatch
+                          ;; exception for features which don't require an
+                          ;; explicit file migration process or has no real
+                          ;; effect on file data structure
+                          (set/difference no-migration-features))]
 
-     (check-supported-features! file-features)
+    (when-let [not-supported (first not-supported)]
+      (ex/raise :type :restriction
+                :code :file-feature-mismatch
+                :feature not-supported
+                :hint (str/ffmt "enabled feature '%' not present in file (missing migration)"
+                                not-supported)))
 
-     (let [not-supported (-> file-features
-                             (set/difference enabled-features)
-                             (set/difference client-features)
-                             (set/difference backend-only-features)
-                             (set/difference frontend-only-features))]
+    (check-supported-features! file-features)
 
-       (when (seq not-supported)
-         (ex/raise :type :restriction
-                   :code :file-feature-mismatch
-                   :feature (first not-supported)
-                   :hint (str/ffmt "file features '%' not enabled"
-                                   (str/join "," not-supported))))))
+    ;; Components v1 is deprecated
+    (when-not (contains? file-features "components/v2")
+      (ex/raise :type :restriction
+                :code :file-in-components-v1
+                :hint "components v1 is deprecated"))
 
-   enabled-features))
+    (let [not-supported (-> file-features
+                            (set/difference enabled-features)
+                            (set/difference backend-only-features)
+                            (set/difference frontend-only-features))]
+
+      ;; Check if file has a feature but that feature is not enabled
+      (when-let [not-supported (first not-supported)]
+        (ex/raise :type :restriction
+                  :code :file-feature-mismatch
+                  :feature not-supported
+                  :hint (str/ffmt "file feature '%' not enabled" not-supported))))
+
+    enabled-features))
 
 (defn check-teams-compatibility!
   [{source-features :features} {destination-features :features}]

common/src/app/common/files/changes.cljc

@@ -16,7 +16,6 @@
    [app.common.schema.desc-native :as smd]
    [app.common.schema.generators :as sg]
    [app.common.types.color :as ctc]
-   [app.common.types.colors-list :as ctcl]
    [app.common.types.component :as ctk]
    [app.common.types.components-list :as ctkl]
    [app.common.types.container :as ctn]
@@ -24,13 +23,13 @@
    [app.common.types.grid :as ctg]
    [app.common.types.page :as ctp]
    [app.common.types.pages-list :as ctpl]
+   [app.common.types.path :as path]
    [app.common.types.shape :as cts]
    [app.common.types.shape-tree :as ctst]
-   [app.common.types.token :as cto]
-   [app.common.types.token-theme :as ctot]
    [app.common.types.tokens-lib :as ctob]
    [app.common.types.typographies-list :as ctyl]
    [app.common.types.typography :as ctt]
+   [app.common.types.variant :as ctv]
    [app.common.uuid :as uuid]
    [clojure.set :as set]))
 
@@ -48,14 +47,14 @@
      [:type [:= :assign]]
      ;; NOTE: the full decoding is happening on the handler because it
      ;; needs a proper context of the current shape and its type
-     [:value [:map-of :keyword :any]]
+     [:value [:map-of :keyword ::sm/any]]
      [:ignore-touched {:optional true} :boolean]
      [:ignore-geometry {:optional true} :boolean]]]
    [:set
     [:map {:title "SetOperation"}
      [:type [:= :set]]
      [:attr :keyword]
-     [:val :any]
+     [:val ::sm/any]
      [:ignore-touched {:optional true} :boolean]
      [:ignore-geometry {:optional true} :boolean]]]
    [:set-touched
@@ -239,10 +238,10 @@
       [:component-id {:optional true} ::sm/uuid]
       [:ignore-touched {:optional true} :boolean]
       [:parent-id ::sm/uuid]
-      [:shapes :any]
+      [:shapes ::sm/any]
       [:index {:optional true} [:maybe :int]]
-      [:after-shape {:optional true} :any]
-      [:component-swap {:optional true} :boolean]]]
+      [:after-shape {:optional true} ::sm/any]
+      [:allow-altering-copies {:optional true} :boolean]]]
 
     [:reorder-children
      [:map {:title "ReorderChildrenChange"}
@@ -251,14 +250,14 @@
       [:component-id {:optional true} ::sm/uuid]
       [:ignore-touched {:optional true} :boolean]
       [:parent-id ::sm/uuid]
-      [:shapes :any]]]
+      [:shapes ::sm/any]]]
 
     [:add-page
      [:map {:title "AddPageChange"}
       [:type [:= :add-page]]
       [:id {:optional true} ::sm/uuid]
       [:name {:optional true} :string]
-      [:page {:optional true} :any]]]
+      [:page {:optional true} ::sm/any]]]
 
     [:mod-page
      [:map {:title "ModPageChange"}
@@ -266,7 +265,7 @@
       [:id ::sm/uuid]
       ;; All props are optional, background can be nil because is the
       ;; way to remove already set background
-      [:background {:optional true} [:maybe ::ctc/rgb-color]]
+      [:background {:optional true} [:maybe ctc/schema:hex-color]]
       [:name {:optional true} :string]]]
 
     [:set-plugin-data schema:set-plugin-data-change]
@@ -292,12 +291,12 @@
     [:add-color
      [:map {:title "AddColorChange"}
       [:type [:= :add-color]]
-      [:color ::ctc/color]]]
+      [:color ctc/schema:library-color]]]
 
     [:mod-color
      [:map {:title "ModColorChange"}
       [:type [:= :mod-color]]
-      [:color ::ctc/color]]]
+      [:color ctc/schema:library-color]]]
 
     [:del-color
      [:map {:title "DelColorChange"}
@@ -311,12 +310,12 @@
     [:add-media
      [:map {:title "AddMediaChange"}
       [:type [:= :add-media]]
-      [:object ::ctf/media-object]]]
+      [:object ctf/schema:media]]]
 
     [:mod-media
      [:map {:title "ModMediaChange"}
       [:type [:= :mod-media]]
-      [:object ::ctf/media-object]]]
+      [:object ctf/schema:media]]]
 
     [:del-media
      [:map {:title "DelMediaChange"}
@@ -328,21 +327,25 @@
       [:type [:= :add-component]]
       [:id ::sm/uuid]
       [:name :string]
-      [:shapes {:optional true} [:vector {:gen/max 3} :any]]
+      [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
       [:path {:optional true} :string]]]
 
     [:mod-component
      [:map {:title "ModCompoenentChange"}
       [:type [:= :mod-component]]
       [:id ::sm/uuid]
-      [:shapes {:optional true} [:vector {:gen/max 3} :any]]
-      [:name {:optional true} :string]]]
+      [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
+      [:name {:optional true} :string]
+      [:variant-id {:optional true} ::sm/uuid]
+      [:variant-properties {:optional true} [:vector ::ctv/variant-property]]]]
 
     [:del-component
      [:map {:title "DelComponentChange"}
       [:type [:= :del-component]]
       [:id ::sm/uuid]
-      [:main-instance {:optional true} :any]
+      ;; when it is an undo of a cut-paste, we need to undo the movement
+      ;; of the shapes so we need to move them delta
+      [:delta {:optional true} ::gpt/point]
       [:skip-undelete? {:optional true} :boolean]]]
 
     [:restore-component
@@ -371,49 +374,10 @@
       [:type [:= :del-typography]]
       [:id ::sm/uuid]]]
 
-    [:add-temporary-token-theme
-     [:map {:title "AddTemporaryTokenThemeChange"}
-      [:type [:= :add-temporary-token-theme]]
-      [:token-theme ::ctot/token-theme]]]
-
     [:update-active-token-themes
      [:map {:title "UpdateActiveTokenThemes"}
       [:type [:= :update-active-token-themes]]
-      [:theme-ids [:set :string]]]]
-
-    [:delete-temporary-token-theme
-     [:map {:title "DeleteTemporaryTokenThemeChange"}
-      [:type [:= :delete-temporary-token-theme]]
-      [:id ::sm/uuid]
-      [:name :string]]]
-
-    [:add-token-theme
-     [:map {:title "AddTokenThemeChange"}
-      [:type [:= :add-token-theme]]
-      [:token-theme ::ctot/token-theme]]]
-
-    [:mod-token-theme
-     [:map {:title "ModTokenThemeChange"}
-      [:type [:= :mod-token-theme]]
-      [:group :string]
-      [:name :string]
-      [:token-theme ::ctot/token-theme]]]
-
-    [:del-token-theme
-     [:map {:title "DelTokenThemeChange"}
-      [:type [:= :del-token-theme]]
-      [:group :string]
-      [:name :string]]]
-
-    [:add-token-set
-     [:map {:title "AddTokenSetChange"}
-      [:type [:= :add-token-set]]
-      [:token-set ::ctot/token-set]]]
-
-    [:add-token-sets
-     [:map {:title "AddTokenSetsChange"}
-      [:type [:= :add-token-sets]]
-      [:token-sets [:sequential ::ctot/token-set]]]]
+      [:theme-paths [:set :string]]]]
 
     [:rename-token-set-group
      [:map {:title "RenameTokenSetGroup"}
@@ -421,61 +385,52 @@
       [:set-group-path [:vector :string]]
       [:set-group-fname :string]]]
 
-    [:mod-token-set
-     [:map {:title "ModTokenSetChange"}
-      [:type [:= :mod-token-set]]
-      [:name :string]
-      [:token-set ::ctot/token-set]]]
-
-    [:move-token-set-before
-     [:map {:title "MoveTokenSetBefore"}
-      [:type [:= :move-token-set-before]]
+    [:move-token-set
+     [:map {:title "MoveTokenSet"}
+      [:type [:= :move-token-set]]
       [:from-path [:vector :string]]
       [:to-path [:vector :string]]
       [:before-path [:maybe [:vector :string]]]
-      [:before-group? [:maybe :boolean]]]]
+      [:before-group [:maybe :boolean]]]]
 
-    [:move-token-set-group-before
-     [:map {:title "MoveTokenSetGroupBefore"}
-      [:type [:= :move-token-set-group-before]]
+    [:move-token-set-group
+     [:map {:title "MoveTokenSetGroup"}
+      [:type [:= :move-token-set-group]]
       [:from-path [:vector :string]]
       [:to-path [:vector :string]]
       [:before-path [:maybe [:vector :string]]]
-      [:before-group? [:maybe :boolean]]]]
+      [:before-group [:maybe :boolean]]]]
 
-    [:del-token-set
-     [:map {:title "DelTokenSetChange"}
-      [:type [:= :del-token-set]]
-      [:name :string]]]
-
-    [:del-token-set-path
-     [:map {:title "DelTokenSetPathChange"}
-      [:type [:= :del-token-set-path]]
-      [:path :string]]]
+    [:set-token-theme
+     [:map {:title "SetTokenThemeChange"}
+      [:type [:= :set-token-theme]]
+      [:theme-name :string]
+      [:group :string]
+      [:theme [:maybe ctob/schema:token-theme-attrs]]]]
 
     [:set-tokens-lib
      [:map {:title "SetTokensLib"}
       [:type [:= :set-tokens-lib]]
-      [:tokens-lib :any]]]
+      [:tokens-lib ::sm/any]]]
 
-    [:add-token
-     [:map {:title "AddTokenChange"}
-      [:type [:= :add-token]]
+    [:set-token-set
+     [:map {:title "SetTokenSetChange"}
+      [:type [:= :set-token-set]]
       [:set-name :string]
-      [:token ::cto/token]]]
+      [:group? :boolean]
+      [:token-set [:maybe [:fn ctob/token-set?]]]]]
 
-    [:mod-token
-     [:map {:title "ModTokenChange"}
-      [:type [:= :mod-token]]
+    [:set-token
+     [:map {:title "SetTokenChange"}
+      [:type [:= :set-token]]
       [:set-name :string]
-      [:name :string]
-      [:token ::cto/token]]]
+      [:token-name :string]
+      [:token [:maybe ctob/schema:token-attrs]]]]
 
-    [:del-token
-     [:map {:title "DelTokenChange"}
-      [:type [:= :del-token]]
-      [:set-name :string]
-      [:name :string]]]]])
+    [:set-base-font-size
+     [:map {:title "ModBaseFontSize"}
+      [:type [:= :set-base-font-size]]
+      [:base-font-size :string]]]]])
 
 (def schema:changes
   [:sequential {:gen/max 5 :gen/min 1} schema:change])
@@ -532,7 +487,9 @@
                                (cts/shape? shape-new))
                   (ex/raise :type :assertion
                             :code :data-validation
-                            :hint "invalid shape found after applying changes"
+                            :hint (str "invalid shape found after applying changes on file "
+                                       (:id data-new))
+                            :file-id (:id data-new)
                             ::sm/explain (cts/explain-shape shape-new))))))]
 
     (->> (into #{} (map :page-id) items)
@@ -782,27 +739,29 @@
 
           (update-group [group objects]
             (let [lookup   (d/getf objects)
-                  children (->> group :shapes (map lookup))]
+                  children (get group :shapes)]
               (cond
                 ;; If the group is empty we don't make any changes. Will be removed by a later process
                 (empty? children)
                 group
 
                 (= :bool (:type group))
-                (gsh/update-bool-selrect group children objects)
+                (path/update-bool-shape group objects)
 
                 (:masked-group group)
-                (set-mask-selrect group children)
+                (->> (map lookup children)
+                     (set-mask-selrect group))
 
                 :else
-                (gsh/update-group-selrect group children))))]
+                (->> (map lookup children)
+                     (gsh/update-group-selrect group)))))]
 
     (if page-id
       (d/update-in-when data [:pages-index page-id :objects] reg-objects)
       (d/update-in-when data [:components component-id :objects] reg-objects))))
 
 (defmethod process-change :mov-objects
-  [data {:keys [parent-id shapes index page-id component-id ignore-touched after-shape component-swap syncing]}]
+  [data {:keys [parent-id shapes index page-id component-id ignore-touched after-shape allow-altering-copies syncing]}]
   (letfn [(calculate-invalid-targets [objects shape-id]
             (let [reduce-fn #(into %1 (calculate-invalid-targets objects %2))]
               (->> (get-in objects [shape-id :shapes])
@@ -817,7 +776,7 @@
               (and shape
                    (not (invalid-targets parent-id))
                    (not (cfh/components-nesting-loop? objects shape-id parent-id))
-                   (or component-swap ;; On a component swap it's allowed to change the structure of a copy
+                   (or allow-altering-copies ;; In some cases (like a component swap) it's allowed to change the structure of a copy
                        syncing ;; If we are syncing the changes of a main component, it's allowed to change the structure of a copy
                        (and
                         (not (ctk/in-component-copy? (get objects (:parent-id shape)))) ;; We don't want to change the structure of component copies
@@ -968,15 +927,15 @@
 
 (defmethod process-change :add-color
   [data {:keys [color]}]
-  (ctcl/add-color data color))
+  (ctc/add-color data color))
 
 (defmethod process-change :mod-color
   [data {:keys [color]}]
-  (ctcl/set-color data color))
+  (ctc/set-color data color))
 
 (defmethod process-change :del-color
   [data {:keys [id]}]
-  (ctcl/delete-color data id))
+  (ctc/delete-color data id))
 
 ;; DEPRECATED: remove before 2.3
 (defmethod process-change :add-recent-color
@@ -1009,8 +968,8 @@
   (ctkl/mod-component data params))
 
 (defmethod process-change :del-component
-  [data {:keys [id skip-undelete? main-instance]}]
-  (ctf/delete-component data id skip-undelete? main-instance))
+  [data {:keys [id skip-undelete? delta]}]
+  (ctf/delete-component data id skip-undelete? delta))
 
 (defmethod process-change :restore-component
   [data {:keys [id page-id]}]
@@ -1040,79 +999,61 @@
   [data {:keys [tokens-lib]}]
   (assoc data :tokens-lib tokens-lib))
 
-(defmethod process-change :add-token
-  [data {:keys [set-name token]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/add-token-in-set set-name (ctob/make-token token)))))
+(defmethod process-change :set-token
+  [data {:keys [set-name token-name token]}]
+  (update data :tokens-lib
+          (fn [lib]
+            (let [lib' (ctob/ensure-tokens-lib lib)]
+              (cond
+                (not token)
+                (ctob/delete-token-from-set lib' set-name token-name)
 
-(defmethod process-change :mod-token
-  [data {:keys [set-name name token]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/update-token-in-set
-                                 set-name
-                                 name
-                                 (fn [old-token]
-                                   (ctob/make-token (merge old-token token)))))))
+                (not (ctob/get-token-in-set lib' set-name token-name))
+                (ctob/add-token-in-set lib' set-name (ctob/make-token token))
 
-(defmethod process-change :del-token
-  [data {:keys [set-name name]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/delete-token-from-set
-                                 set-name
-                                 name))))
+                :else
+                (ctob/update-token-in-set lib' set-name token-name (fn [prev-token]
+                                                                     (ctob/make-token (merge prev-token token)))))))))
 
-(defmethod process-change :add-temporary-token-theme
-  [data {:keys [token-theme]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/add-theme (ctob/make-token-theme token-theme)))))
+(defmethod process-change :set-token-set
+  [data {:keys [set-name group? token-set]}]
+  (update data :tokens-lib
+          (fn [lib]
+            (let [lib' (ctob/ensure-tokens-lib lib)]
+              (cond
+                (not token-set)
+                (if group?
+                  (ctob/delete-set-group lib' set-name)
+                  (ctob/delete-set lib' set-name))
+
+                (not (ctob/get-set lib' set-name))
+                (ctob/add-set lib' token-set)
+
+                :else
+                (ctob/update-set lib' set-name (fn [_] token-set)))))))
+
+(defmethod process-change :set-token-theme
+  [data {:keys [group theme-name theme]}]
+  (update data :tokens-lib
+          (fn [lib]
+            (let [lib' (ctob/ensure-tokens-lib lib)]
+              (cond
+                (not theme)
+                (ctob/delete-theme lib' group theme-name)
+
+                (not (ctob/get-theme lib' group theme-name))
+                (ctob/add-theme lib' (ctob/make-token-theme theme))
+
+                :else
+                (ctob/update-theme lib'
+                                   group theme-name
+                                   (fn [prev-token-theme]
+                                     (ctob/make-token-theme (merge prev-token-theme theme)))))))))
 
 (defmethod process-change :update-active-token-themes
-  [data {:keys [theme-ids]}]
+  [data {:keys [theme-paths]}]
   (update data :tokens-lib #(-> % (ctob/ensure-tokens-lib)
-                                (ctob/set-active-themes theme-ids))))
-
-(defmethod process-change :delete-temporary-token-theme
-  [data {:keys [group name]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/delete-theme group name))))
-
-(defmethod process-change :add-token-theme
-  [data {:keys [token-theme]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/add-theme (-> token-theme
-                                                    (ctob/make-token-theme))))))
-
-(defmethod process-change :mod-token-theme
-  [data {:keys [name group token-theme]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/update-theme group name
-                                                   (fn [prev-theme]
-                                                     (merge prev-theme token-theme))))))
-
-(defmethod process-change :del-token-theme
-  [data {:keys [group name]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/delete-theme group name))))
-
-(defmethod process-change :add-token-set
-  [data {:keys [token-set]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/add-set (ctob/make-token-set token-set)))))
-
-(defmethod process-change :add-token-sets
-  [data {:keys [token-sets]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/add-sets (map ctob/make-token-set token-sets)))))
+                                (ctob/set-active-themes theme-paths))))
 
 (defmethod process-change :rename-token-set-group
   [data {:keys [set-group-path set-group-fname]}]
@@ -1121,37 +1062,24 @@
                                  (ctob/ensure-tokens-lib)
                                  (ctob/rename-set-group set-group-path set-group-fname)))))
 
-(defmethod process-change :mod-token-set
-  [data {:keys [name token-set]}]
-  (update data :tokens-lib (fn [lib]
-                             (-> lib
-                                 (ctob/ensure-tokens-lib)
-                                 (ctob/update-set name (fn [prev-set]
-                                                         (merge prev-set (dissoc token-set :tokens))))))))
-
-(defmethod process-change :move-token-set-before
-  [data {:keys [from-path to-path before-path before-group?] :as changes}]
+(defmethod process-change :move-token-set
+  [data {:keys [from-path to-path before-path before-group] :as changes}]
   (update data :tokens-lib #(-> %
                                 (ctob/ensure-tokens-lib)
-                                (ctob/move-set from-path to-path before-path before-group?))))
+                                (ctob/move-set from-path to-path before-path before-group))))
 
-(defmethod process-change :move-token-set-group-before
-  [data {:keys [from-path to-path before-path before-group?]}]
+(defmethod process-change :move-token-set-group
+  [data {:keys [from-path to-path before-path before-group]}]
   (update data :tokens-lib #(-> %
                                 (ctob/ensure-tokens-lib)
-                                (ctob/move-set-group from-path to-path before-path before-group?))))
+                                (ctob/move-set-group from-path to-path before-path before-group))))
 
-(defmethod process-change :del-token-set
-  [data {:keys [name]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/delete-set-path name))))
+;; === Base font size
+
+(defmethod process-change :set-base-font-size
+  [data {:keys [base-font-size]}]
+  (ctf/set-base-font-size data base-font-size))
 
-(defmethod process-change :del-token-set-path
-  [data {:keys [path]}]
-  (update data :tokens-lib #(-> %
-                                (ctob/ensure-tokens-lib)
-                                (ctob/delete-set-path path))))
 
 ;; === Operations
 

common/src/app/common/files/changes_builder.cljc

@@ -8,7 +8,6 @@
   (:require
    [app.common.data :as d]
    [app.common.data.macros :as dm]
-   [app.common.features :as cfeat]
    [app.common.files.changes :as cfc]
    [app.common.files.helpers :as cfh]
    [app.common.geom.matrix :as gmt]
@@ -19,24 +18,26 @@
    [app.common.schema :as sm]
    [app.common.types.component :as ctk]
    [app.common.types.file :as ctf]
+   [app.common.types.path :as path]
    [app.common.types.shape.layout :as ctl]
    [app.common.types.tokens-lib :as ctob]
    [app.common.uuid :as uuid]))
 
 ;; Auxiliary functions to help create a set of changes (undo + redo)
 
-(sm/register!
- ^{::sm/type ::changes}
- [:map {:title "changes"}
-  [:redo-changes vector?]
-  [:undo-changes seq?]
-  [:origin {:optional true} any?]
-  [:save-undo? {:optional true} boolean?]
-  [:stack-undo? {:optional true} boolean?]
-  [:undo-group {:optional true} any?]])
+(def schema:changes
+  (sm/register!
+   ^{::sm/type ::changes}
+   [:map {:title "changes"}
+    [:redo-changes vector?]
+    [:undo-changes seq?]
+    [:origin {:optional true} ::sm/any]
+    [:save-undo? {:optional true} boolean?]
+    [:stack-undo? {:optional true} boolean?]
+    [:undo-group {:optional true} ::sm/any]]))
 
 (def check-changes!
-  (sm/check-fn ::changes))
+  (sm/check-fn schema:changes))
 
 (defn empty-changes
   ([origin page-id]
@@ -84,8 +85,7 @@
 
 (defn with-objects
   [changes objects]
-  (let [fdata (binding [cfeat/*current* #{"components/v2"}]
-                (ctf/make-file-data (uuid/next) uuid/zero))
+  (let [fdata (ctf/make-file-data (uuid/next) uuid/zero)
         fdata (assoc-in fdata [:pages-index uuid/zero :objects] objects)]
     (vary-meta changes assoc
                ::file-data fdata
@@ -126,42 +126,55 @@
 ; TODO: remove this when not needed
 (defn- assert-page-id!
   [changes]
-  (dm/assert!
-   "Give a page-id or call (with-page) before using this function"
-   (contains? (meta changes) ::page-id)))
+  (assert
+   (contains? (meta changes) ::page-id)
+   "Give a page-id or call (with-page) before using this function"))
+
+(defn- assert-page!
+  [changes]
+  (assert
+   (contains? (meta changes) ::page)
+   "Give a page or call (with-page) before using this function"))
 
 (defn- assert-container-id!
   [changes]
-  (dm/assert!
-   "Give a page-id or call (with-container) before using this function"
+  (assert
    (or (contains? (meta changes) ::page-id)
-       (contains? (meta changes) ::component-id))))
+       (contains? (meta changes) ::component-id))
+   "Give a page-id or call (with-container) before using this function"))
 
 (defn- assert-objects!
   [changes]
-  (dm/assert!
-   "Call (with-objects) before using this function"
-   (contains? (meta changes) ::file-data)))
+  (assert
+   (contains? (meta changes) ::file-data)
+   "Call (with-objects) before using this function"))
 
 (defn- assert-library!
   [changes]
-  (dm/assert!
-   "Call (with-library-data) before using this function"
-   (contains? (meta changes) ::library-data)))
+  (assert
+   (contains? (meta changes) ::library-data)
+   "Call (with-library-data) before using this function"))
+
+(defn- assert-file-data!
+  [changes]
+  (assert
+   (contains? (meta changes) ::file-data)
+   "Call (with-file-data) before using this function"))
 
 (defn- lookup-objects
   [changes]
   (let [data (::file-data (meta changes))]
     (dm/get-in data [:pages-index uuid/zero :objects])))
 
-(defn- apply-changes-local
-  [changes]
-  (dm/assert!
-   "expected valid changes"
-   (check-changes! changes))
+(defn apply-changes-local
+  [changes & {:keys [apply-to-library?]}]
+  (assert
+   (check-changes! changes)
+   "expected valid changes")
 
   (if-let [file-data (::file-data (meta changes))]
-    (let [index         (::applied-changes-count (meta changes))
+    (let [library-data  (::library-data (meta changes))
+          index         (::applied-changes-count (meta changes))
           redo-changes  (:redo-changes changes)
           new-changes   (if (< index (count redo-changes))
                           (->> (subvec (:redo-changes changes) index)
@@ -169,8 +182,12 @@
                                          (assoc :page-id uuid/zero)
                                          (dissoc :component-id))))
                           [])
-          new-file-data (cfc/process-changes file-data new-changes)]
+          new-file-data (cfc/process-changes file-data new-changes)
+          new-library-data (if apply-to-library?
+                             (cfc/process-changes library-data new-changes)
+                             library-data)]
       (vary-meta changes assoc ::file-data new-file-data
+                 ::library-data new-library-data
                  ::applied-changes-count (count redo-changes)))
     changes))
 
@@ -192,6 +209,7 @@
 
 (defn mod-page
   ([changes options]
+   (assert-page! changes)
    (let [page (::page (meta changes))]
      (mod-page changes page options)))
 
@@ -222,6 +240,7 @@
   ([changes type id namespace key value]
    (set-plugin-data changes type id nil namespace key value))
   ([changes type id page-id namespace key value]
+   (assert-file-data! changes)
    (let [data (::file-data (meta changes))
          old-val
          (case type
@@ -288,6 +307,8 @@
 
 (defn set-guide
   [changes id guide]
+  (assert-page-id! changes)
+  (assert-page! changes)
   (let [page-id (::page-id (meta changes))
         page    (::page (meta changes))
         old-val (dm/get-in page [:guides id])]
@@ -301,8 +322,11 @@
                                     :page-id page-id
                                     :id id
                                     :params old-val}))))
+
 (defn set-flow
   [changes id flow]
+  (assert-page-id! changes)
+  (assert-page! changes)
   (let [page-id (::page-id (meta changes))
         page    (::page (meta changes))
         old-val (dm/get-in page [:flows id])
@@ -321,6 +345,8 @@
 
 (defn set-comment-thread-position
   [changes {:keys [id frame-id position] :as thread}]
+  (assert-page-id! changes)
+  (assert-page! changes)
   (let [page-id (::page-id (meta changes))
         page    (::page (meta changes))
 
@@ -342,6 +368,8 @@
 
 (defn set-default-grid
   [changes type params]
+  (assert-page-id! changes)
+  (assert-page! changes)
   (let [page-id (::page-id (meta changes))
         page    (::page (meta changes))
         old-val (dm/get-in page [:grids type])
@@ -436,8 +464,8 @@
 
            (some? index)
            (assoc :index index)
-           (:component-swap options)
-           (assoc :component-swap true)
+           (:allow-altering-copies options)
+           (assoc :allow-altering-copies true)
            (:ignore-touched options)
            (assoc :ignore-touched true))
 
@@ -445,12 +473,14 @@
          (fn [undo-changes shape]
            (let [prev-sibling (cfh/get-prev-sibling objects (:id shape))]
              (conj undo-changes
-                   {:type :mov-objects
-                    :page-id (::page-id (meta changes))
-                    :parent-id (:parent-id shape)
-                    :shapes [(:id shape)]
-                    :after-shape prev-sibling
-                    :index 0}))) ; index is used in case there is no after-shape (moving bottom shapes)
+                   (cond-> {:type :mov-objects
+                            :page-id (::page-id (meta changes))
+                            :parent-id (:parent-id shape)
+                            :shapes [(:id shape)]
+                            :after-shape prev-sibling
+                            :index 0} ; index is used in case there is no after-shape (moving bottom shapes)
+                     (:allow-altering-copies options)
+                     (assoc :allow-altering-copies true)))))
 
          restore-touched-change
          {:type :mod-obj
@@ -475,9 +505,12 @@
           (let [old-val (get old attr)
                 new-val (get new attr)]
             (not= old-val new-val)))
-        new-obj (if with-objects?
-                  (update-fn object objects)
-                  (update-fn object))]
+
+        new-obj
+        (if with-objects?
+          (update-fn object objects)
+          (update-fn object))]
+
     (when-not (= object new-obj)
       (let [attrs (or attrs (d/concat-set (keys object) (keys new-obj)))]
         (filter (partial changed? object new-obj) attrs)))))
@@ -653,10 +686,14 @@
                                  (empty? children) ;; a parent with no children will be deleted,
                                  nil               ;; so it does not need resize
 
-                                 (= (:type parent) :bool)
-                                 (gsh/update-bool-selrect parent children objects)
+                                 (cfh/bool-shape? parent)
+                                 (path/update-bool-shape parent objects)
 
-                                 (= (:type parent) :group)
+                                 (cfh/group-shape? parent)
+                                 ;; FIXME: this functions should be
+                                 ;; normalized in the same way as
+                                 ;; update-bool in order to make all
+                                 ;; this code consistent
                                  (if (:masked-group parent)
                                    (gsh/update-mask-selrect parent children)
                                    (gsh/update-group-selrect parent children)))]
@@ -762,56 +799,39 @@
         (update :undo-changes conj {:type :add-typography :typography prev-typography})
         (apply-changes-local))))
 
-(defn add-temporary-token-theme
-  [changes token-theme]
-  (-> changes
-      (update :redo-changes conj {:type :add-temporary-token-theme :token-theme token-theme})
-      (update :undo-changes conj {:type :delete-temporary-token-theme :id (:id token-theme) :name (:name token-theme)})
-      (apply-changes-local)))
-
 (defn update-active-token-themes
-  [changes token-active-theme-ids prev-token-active-theme-ids]
+  [changes active-theme-paths prev-active-theme-paths]
   (-> changes
-      (update :redo-changes conj {:type :update-active-token-themes :theme-ids token-active-theme-ids})
-      (update :undo-changes conj {:type :update-active-token-themes :theme-ids prev-token-active-theme-ids})
+      (update :redo-changes conj {:type :update-active-token-themes :theme-paths active-theme-paths})
+      (update :undo-changes conj {:type :update-active-token-themes :theme-paths prev-active-theme-paths})
       (apply-changes-local)))
 
-(defn add-token-theme
-  [changes token-theme]
-  (-> changes
-      (update :redo-changes conj {:type :add-token-theme :token-theme token-theme})
-      (update :undo-changes conj {:type :del-token-theme :group (:group token-theme) :name (:name token-theme)})
-      (apply-changes-local)))
-
-(defn update-token-theme
-  [changes token-theme prev-token-theme]
-  (let [name (or (:name prev-token-theme)
-                 (:name token-theme))
-        group (or (:group prev-token-theme)
-                  (:group token-theme))]
-    (-> changes
-        (update :redo-changes conj {:type :mod-token-theme :group group :name name :token-theme token-theme})
-        (update :undo-changes conj {:type :mod-token-theme :group group :name name :token-theme (or prev-token-theme token-theme)})
-        (apply-changes-local))))
-
-(defn delete-token-theme
-  [changes group name]
+(defn set-token-theme [changes group theme-name theme]
   (assert-library! changes)
   (let [library-data (::library-data (meta changes))
-        prev-token-theme (some-> (get library-data :tokens-lib)
-                                 (ctob/get-theme group name))]
+        prev-theme (some-> (get library-data :tokens-lib)
+                           (ctob/get-theme group theme-name))]
     (-> changes
-        (update :redo-changes conj {:type :del-token-theme :group group :name name})
-        (update :undo-changes conj {:type :add-token-theme :token-theme prev-token-theme})
+        (update :redo-changes conj {:type :set-token-theme
+                                    :theme-name theme-name
+                                    :group group
+                                    :theme theme})
+        (update :undo-changes conj (if prev-theme
+                                     {:type :set-token-theme
+                                      :group group
+                                      :theme-name (or
+                                                   ;; Undo of edit
+                                                   (:name theme)
+                                                   ;; Undo of delete
+                                                   theme-name)
+                                      :theme prev-theme}
+                                     ;; Undo of create
+                                     {:type :set-token-theme
+                                      :group group
+                                      :theme-name theme-name
+                                      :theme nil}))
         (apply-changes-local))))
 
-(defn add-token-set
-  [changes token-set]
-  (-> changes
-      (update :redo-changes conj {:type :add-token-set :token-set token-set})
-      (update :undo-changes conj {:type :del-token-set :name (:name token-set)})
-      (apply-changes-local)))
-
 (defn rename-token-set-group
   [changes set-group-path set-group-fname]
   (let [undo-path (ctob/replace-last-path-name set-group-path set-group-fname)
@@ -821,62 +841,39 @@
         (update :undo-changes conj {:type :rename-token-set-group :set-group-path undo-path :set-group-fname undo-fname})
         (apply-changes-local))))
 
-(defn update-token-set
-  [changes token-set prev-token-set]
-  (-> changes
-      (update :redo-changes conj {:type :mod-token-set :name (:name prev-token-set) :token-set token-set})
-      (update :undo-changes conj {:type :mod-token-set :name (:name token-set) :token-set (or prev-token-set token-set)})
-      (apply-changes-local)))
-
-(defn delete-token-set-path
-  [changes group? path]
-  (assert-library! changes)
-  (let [;; TODO Move leaking prefix to library
-        prefixed-path (if group?
-                        (ctob/set-group-path->set-group-prefixed-path path)
-                        (ctob/set-full-path->set-prefixed-full-path path))
-        prefixed-path-str (ctob/join-set-path prefixed-path)
-        library-data (::library-data (meta changes))
-        prev-token-sets (some-> (get library-data :tokens-lib)
-                                (ctob/get-path-sets prefixed-path-str))]
-    (-> changes
-        (update :redo-changes conj {:type :del-token-set-path :path prefixed-path-str})
-        (update :undo-changes conj {:type :add-token-sets :token-sets prev-token-sets})
-        (apply-changes-local))))
-
-(defn move-token-set-before
+(defn move-token-set
   [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?] :as opts}]
   (-> changes
-      (update :redo-changes conj {:type :move-token-set-before
+      (update :redo-changes conj {:type :move-token-set
                                   :from-path from-path
                                   :to-path to-path
                                   :before-path before-path
-                                  :before-group? before-group?})
-      (update :undo-changes conj {:type :move-token-set-before
+                                  :before-group before-group?})
+      (update :undo-changes conj {:type :move-token-set
                                   :from-path to-path
                                   :to-path from-path
                                   :before-path prev-before-path
-                                  :before-group? prev-before-group?})
+                                  :before-group prev-before-group?})
       (apply-changes-local)))
 
-(defn move-token-set-group-before
+(defn move-token-set-group
   [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?]}]
-  (prn prev-before-path prev-before-group?)
   (-> changes
-      (update :redo-changes conj {:type :move-token-set-group-before
+      (update :redo-changes conj {:type :move-token-set-group
                                   :from-path from-path
                                   :to-path to-path
                                   :before-path before-path
-                                  :before-group? before-group?})
-      (update :undo-changes conj {:type :move-token-set-group-before
+                                  :before-group before-group?})
+      (update :undo-changes conj {:type :move-token-set-group
                                   :from-path to-path
                                   :to-path from-path
                                   :before-path prev-before-path
-                                  :before-group? prev-before-group?})
+                                  :before-group prev-before-group?})
       (apply-changes-local)))
 
 (defn set-tokens-lib
   [changes tokens-lib]
+  (assert-library! changes)
   (let [library-data (::library-data (meta changes))
         prev-tokens-lib (get library-data :tokens-lib)]
     (-> changes
@@ -884,36 +881,84 @@
         (update :undo-changes conj {:type :set-tokens-lib :tokens-lib prev-tokens-lib})
         (apply-changes-local))))
 
-(defn add-token
-  [changes set-name token]
-  (-> changes
-      (update :redo-changes conj {:type :add-token :set-name set-name :token token})
-      (update :undo-changes conj {:type :del-token :set-name set-name :name (:name token)})
-      (apply-changes-local)))
-
-(defn update-token
-  [changes set-name token prev-token]
-  (-> changes
-      (update :redo-changes conj {:type :mod-token :set-name set-name :name (:name prev-token) :token token})
-      (update :undo-changes conj {:type :mod-token :set-name set-name :name (:name token) :token (or prev-token token)})
-      (apply-changes-local)))
-
-(defn delete-token
-  [changes set-name token-name]
+(defn set-token [changes set-name token-name token]
   (assert-library! changes)
   (let [library-data (::library-data (meta changes))
         prev-token (some-> (get library-data :tokens-lib)
                            (ctob/get-set set-name)
                            (ctob/get-token token-name))]
     (-> changes
-        (update :redo-changes conj {:type :del-token :set-name set-name :name token-name})
-        (update :undo-changes conj {:type :add-token :set-name set-name :token prev-token})
+        (update :redo-changes conj {:type :set-token
+                                    :set-name set-name
+                                    :token-name token-name
+                                    :token token})
+        (update :undo-changes conj (if prev-token
+                                     {:type :set-token
+                                      :set-name set-name
+                                      :token-name (or
+                                                   ;; Undo of edit
+                                                   (:name token)
+                                                   ;; Undo of delete
+                                                   token-name)
+                                      :token prev-token}
+                                     ;; Undo of create token
+                                     {:type :set-token
+                                      :set-name set-name
+                                      :token-name token-name
+                                      :token nil}))
+        (apply-changes-local))))
+
+(defn rename-token-set
+  [changes name new-name]
+
+  (assert-library! changes)
+  (let [library-data   (::library-data (meta changes))
+        prev-token-set (some-> (get library-data :tokens-lib)
+                               (ctob/get-set name))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token-set
+                                    :set-name name
+                                    :token-set (ctob/rename prev-token-set new-name)
+                                    :group? false})
+        (update :undo-changes conj {:type :set-token-set
+                                    :set-name new-name
+                                    :token-set prev-token-set
+                                    :group? false})
+        (apply-changes-local))))
+
+(defn set-token-set
+  [changes set-name group? token-set]
+  (assert-library! changes)
+  (let [library-data   (::library-data (meta changes))
+        prev-token-set (some-> (get library-data :tokens-lib)
+                               (ctob/get-set set-name))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token-set
+                                    :set-name set-name
+                                    :token-set token-set
+                                    :group? group?})
+        (update :undo-changes conj (if prev-token-set
+                                     {:type :set-token-set
+                                      :set-name (if token-set
+                                                  ;; Undo of edit
+                                                  (ctob/get-name token-set)
+                                                  ;; Undo of delete
+                                                  set-name)
+                                      :token-set prev-token-set
+                                      :group? group?}
+                                     ;; Undo of create
+                                     {:type :set-token-set
+                                      :set-name set-name
+                                      :token-set nil
+                                      :group? group?}))
         (apply-changes-local))))
 
 (defn add-component
-  ([changes id path name new-shapes updated-shapes main-instance-id main-instance-page]
-   (add-component changes id path name new-shapes updated-shapes main-instance-id main-instance-page nil))
-  ([changes id path name new-shapes updated-shapes main-instance-id main-instance-page annotation]
+  ([changes id path name updated-shapes main-instance-id main-instance-page]
+   (add-component changes id path name updated-shapes main-instance-id main-instance-page nil nil nil))
+  ([changes id path name updated-shapes main-instance-id main-instance-page annotation]
+   (add-component changes id path name updated-shapes main-instance-id main-instance-page annotation nil nil))
+  ([changes id path name updated-shapes main-instance-id main-instance-page annotation variant-id variant-properties & {:keys [apply-changes-local-library?]}]
    (assert-page-id! changes)
    (assert-objects! changes)
    (let [page-id (::page-id (meta changes))
@@ -953,8 +998,10 @@
                                       :main-instance-id main-instance-id
                                       :main-instance-page main-instance-page
                                       :annotation annotation}
-                               (some? new-shapes)  ;; this will be null in components-v2
-                               (assoc :shapes (vec new-shapes))))
+                               (some? variant-id)
+                               (assoc :variant-id variant-id)
+                               (seq variant-properties)
+                               (assoc :variant-properties variant-properties)))
                        (into (map mk-change) updated-shapes))))
          (update :undo-changes
                  (fn [undo-changes]
@@ -967,33 +1014,51 @@
                                    (map mk-change))
                              updated-shapes))))
 
-         (apply-changes-local)))))
+         (apply-changes-local {:apply-to-library? apply-changes-local-library?})))))
 
 (defn update-component
-  [changes id update-fn]
+  [changes id update-fn & {:keys [apply-changes-local-library?]}]
   (assert-library! changes)
   (let [library-data   (::library-data (meta changes))
         prev-component (get-in library-data [:components id])
         new-component  (update-fn prev-component)]
     (if prev-component
       (-> changes
-          (update :redo-changes conj {:type :mod-component
-                                      :id id
-                                      :name (:name new-component)
-                                      :path (:path new-component)
-                                      :main-instance-id (:main-instance-id new-component)
-                                      :main-instance-page (:main-instance-page new-component)
-                                      :annotation (:annotation new-component)
-                                      :objects (:objects new-component) ;; this won't exist in components-v2 (except for deleted components)
-                                      :modified-at (:modified-at new-component)})
-          (update :undo-changes conj {:type :mod-component
-                                      :id id
-                                      :name (:name prev-component)
-                                      :path (:path prev-component)
-                                      :main-instance-id (:main-instance-id prev-component)
-                                      :main-instance-page (:main-instance-page prev-component)
-                                      :annotation (:annotation prev-component)
-                                      :objects (:objects prev-component)}))
+          (update :redo-changes conj (cond-> {:type :mod-component
+                                              :id id
+                                              :name (:name new-component)
+                                              :path (:path new-component)
+                                              :main-instance-id (:main-instance-id new-component)
+                                              :main-instance-page (:main-instance-page new-component)
+                                              :annotation (:annotation new-component)
+                                              :objects (:objects new-component) ;; for deleted components
+                                              :modified-at (:modified-at new-component)}
+                                       (some? (:variant-id new-component))
+                                       (assoc :variant-id (:variant-id new-component))
+                                       (nil? (:variant-id new-component))
+                                       (dissoc :variant-id)
+                                       (seq (:variant-properties new-component))
+                                       (assoc :variant-properties (:variant-properties new-component))
+                                       (not (seq (:variant-properties new-component)))
+                                       (dissoc :variant-properties)))
+          (update :undo-changes conj (cond-> {:type :mod-component
+                                              :id id
+                                              :name (:name prev-component)
+                                              :path (:path prev-component)
+                                              :main-instance-id (:main-instance-id prev-component)
+                                              :main-instance-page (:main-instance-page prev-component)
+                                              :annotation (:annotation prev-component)
+                                              :objects (:objects prev-component)}
+                                       (some? (:variant-id prev-component))
+                                       (assoc :variant-id (:variant-id prev-component))
+                                       (nil? (:variant-id prev-component))
+                                       (dissoc :variant-id)
+                                       (seq (:variant-properties prev-component))
+                                       (assoc :variant-properties (:variant-properties prev-component))
+                                       (not (seq (:variant-properties prev-component)))
+                                       (dissoc :variant-properties)))
+          (cond-> apply-changes-local-library?
+            (apply-changes-local {:apply-to-library? true})))
       changes)))
 
 (defn delete-component
@@ -1007,7 +1072,7 @@
                                   :page-id page-id})))
 
 (defn restore-component
-  [changes id page-id main-instance]
+  [changes id page-id delta]
   (assert-library! changes)
   (-> changes
       (update :redo-changes conj {:type :restore-component
@@ -1015,7 +1080,34 @@
                                   :page-id page-id})
       (update :undo-changes conj {:type :del-component
                                   :id id
-                                  :main-instance main-instance})))
+                                  :delta delta})))
+
+(defn reorder-children
+  [changes id children]
+  (assert-page-id! changes)
+  (assert-objects! changes)
+
+  (let [page-id (::page-id (meta changes))
+        objects (lookup-objects changes)
+        shape (get objects id)
+        old-children (:shapes shape)
+
+        redo-change
+        {:type :reorder-children
+         :parent-id (:id shape)
+         :page-id page-id
+         :shapes children}
+
+        undo-change
+        {:type :reorder-children
+         :parent-id (:id shape)
+         :page-id page-id
+         :shapes old-children}]
+
+    (-> changes
+        (update :redo-changes conj redo-change)
+        (update :undo-changes conj undo-change)
+        (apply-changes-local))))
 
 (defn reorder-grid-children
   [changes ids]
@@ -1055,3 +1147,32 @@
              (reduce reorder-grid changes))]
 
     changes))
+
+(defn get-library-data
+  [changes]
+  (::library-data (meta changes)))
+
+(defn get-objects
+  [changes]
+  (dm/get-in (::file-data (meta changes)) [:pages-index uuid/zero :objects]))
+
+(defn get-page
+  [changes]
+  (::page (meta changes)))
+
+(defn get-page-id
+  [changes]
+  (::page-id (meta changes)))
+
+(defn set-base-font-size
+  [changes new-base-font-size]
+  (assert-file-data! changes)
+  (let [file-data  (::file-data (meta changes))
+        previous-font-size (ctf/get-base-font-size file-data)]
+    (-> changes
+        (update :redo-changes conj {:type :set-base-font-size
+                                    :base-font-size new-base-font-size})
+
+        (update :undo-changes conj {:type :set-base-font-size
+                                    :base-font-size previous-font-size})
+        (apply-changes-local))))

common/src/app/common/files/helpers.cljc

@@ -9,7 +9,6 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.geom.shapes.common :as gco]
-   [app.common.schema :as sm]
    [app.common.uuid :as uuid]
    [clojure.set :as set]
    [clojure.walk :as walk]
@@ -118,6 +117,12 @@
   ([shape]
    (d/not-empty? (:shapes shape))))
 
+(defn has-layout?
+  "Returns true if the provided shape has a layout assigned"
+  [objects id]
+  (let [shape (get objects id)]
+    (boolean (and shape (:layout shape)))))
+
 (defn group-like-shape?
   ([objects id]
    (group-like-shape? (get objects id)))
@@ -128,13 +133,41 @@
 
 ;; ---- ACCESSORS
 
-(defn get-children-ids
+(defn get-selected-type
+  "Returns the type of the shape if only one, or :multiple if more
+  than one"
+  [objects selected]
+  (if (= 1 (count selected))
+    (let [shape (get objects (first selected))]
+      (:type shape))
+    :multiple))
+
+(defn get-shape-type
+  "Returns the type of the shape, or 'root' if it's Root Frame, always
+  as string"
   [objects id]
-  (letfn [(get-children-ids-rec [id processed]
-            (when (not (contains? processed id))
-              (when-let [shapes (-> (get objects id) :shapes (some-> vec))]
-                (into shapes (mapcat #(get-children-ids-rec % (conj processed id))) shapes))))]
-    (get-children-ids-rec id #{})))
+  (let [shape (get objects id)]
+    (if (root? shape)
+      :root
+      (dm/get-prop shape :type))))
+
+(defn get-children-ids
+  "Returns the ids of all the descendants of the shape identified
+  by the id. Optionally, you can pass an ignore function to indicate
+  when to ignore a descendant (and all its descendants)"
+  ([objects id]
+   (get-children-ids objects id {}))
+  ([objects id {:keys [ignore-children-fn]
+                ;;ignore-children-fn should receive a shape and return a boolean
+                :or {ignore-children-fn (constantly false)}}]
+   (letfn [(get-children-ids-rec [id processed]
+             (when-not (contains? processed id)
+               (when-let [shapes (as-> (get objects id) $
+                                   (:shapes $)
+                                   (remove ignore-children-fn $)
+                                   (some-> $ vec))]
+                 (into shapes (mapcat #(get-children-ids-rec % (conj processed id))) shapes))))]
+     (get-children-ids-rec id #{}))))
 
 (defn get-children-ids-with-self
   [objects id]
@@ -201,7 +234,7 @@
         result))))
 
 (defn get-parent-seq
-  "Returns a vector of parents of the specified shape."
+  "Returns a lazy seq of parents of the specified shape."
   ([objects shape-id]
    (get-parent-seq objects (get objects shape-id) shape-id))
 
@@ -284,6 +317,22 @@
       :else
       (get-root-frame objects (:frame-id frame)))))
 
+(defn get-parent-frame
+  "Similar to `get-frame, but always return the parent frame. When root
+  frame is provided, then itself is returned."
+  [objects shape-or-id]
+  (cond
+    (map? shape-or-id)
+    (get objects (dm/get-prop shape-or-id :frame-id))
+
+    (= uuid/zero shape-or-id)
+    (get objects uuid/zero)
+
+    :else
+    (some->> shape-or-id
+             (get objects)
+             (get-frame objects))))
+
 (defn valid-frame-target?
   [objects parent-id shape-id]
   (let [shape (get objects shape-id)]
@@ -400,31 +449,57 @@
                    elements)]
     (into #{} (keep :name) elements)))
 
-(defn- extract-numeric-suffix
-  [basename]
-  (if-let [[_ p1 p2] (re-find #"(.*) ([0-9]+)$" basename)]
-    [p1 (+ 1 (d/parse-integer p2))]
-    [basename 1]))
+(defn- name-seq
+  "Creates a lazy, infinite sequence of names starting with `base-name`,
+   followed by variants with suffixes applied. The sequence follows this pattern:
+   - `base-name`
+   - `(str base-name (suffix-fn 1))`
+   - `(str base-name (suffix-fn 2))`
+   - `(str base-name (suffix-fn 3))`, etc."
+  [base-name suffix-fn]
+  (cons base-name
+        (map #(str/concat base-name (suffix-fn %))
+             (iterate inc 1))))
 
 (defn generate-unique-name
-  "A unique name generator"
-  [used basename]
+  "Generates a unique name by selecting the first available name from a generated sequence.
+   The sequence consists of `base-name` and its variants, avoiding conflicts with `existing-names`.
+
+   Parameters:
+   - `base-name` - string used as the base for name generation.
+   - `existing-names` - a collection of existing names to check for uniqueness.
+   - Options:
+     - `:suffix-fn` - a function that generates suffixes, given an integer (default: `get-suffix`).
+     - `:immediate-suffix?` - if `true`, the base name is considered taken, and suffixing starts immediately.
+
+   Returns:
+   - A unique name not present in `existing-names`."
+  [base-name existing-names & {:keys [suffix-fn immediate-suffix? suffix]}]
   (dm/assert!
    "expected a set of strings"
-   (sm/check-set-of-strings! used))
+   (coll? existing-names))
 
   (dm/assert!
    "expected a string for `basename`."
-   (string? basename))
+   (string? base-name))
+  (let [suffix-fn (if suffix-fn
+                    suffix-fn
+                    (if suffix
+                      (fn [copy-count]
+                        (str/concat "-"
+                                    suffix
+                                    (when (> copy-count 1)
+                                      (str "-" copy-count))))
+                      (fn [copy-count]
+                        (str/concat " " copy-count))))
 
-  (if-not (contains? used basename)
-    basename
-    (let [[prefix initial] (extract-numeric-suffix basename)]
-      (loop [counter initial]
-        (let [candidate (str prefix " " counter)]
-          (if (contains? used candidate)
-            (recur (inc counter))
-            candidate))))))
+        existing-name-set (cond-> (set existing-names)
+                            immediate-suffix? (conj base-name))
+        names (name-seq base-name suffix-fn)]
+
+    (->> names
+         (remove #(contains? existing-name-set %))
+         first)))
 
 (defn walk-pages
   "Go through all pages of a file and apply a function to each one"
@@ -585,6 +660,9 @@
               (map? (:fill-image form))
               (update-in [:fill-image :id] lookup-index)
 
+              (map? (:stroke-image form))
+              (update-in [:stroke-image :id] lookup-index)
+
               ;; This covers old shapes and the new :fills.
               (uuid? (:fill-color-ref-file form))
               (update :fill-color-ref-file lookup-index)

common/src/app/common/files/repair.cljc

@@ -96,7 +96,7 @@
   (log/dbg :hint "repairing shape :invalid-parent" :id (:id shape) :name (:name shape) :page-id page-id)
   (-> (pcb/empty-changes nil page-id)
       (pcb/with-file-data file-data)
-      (pcb/change-parent (:parent-id args) [shape] nil {:component-swap true})))
+      (pcb/change-parent (:parent-id args) [shape] nil {:allow-altering-copies true})))
 
 (defmethod repair-error :frame-not-found
   [_ {:keys [shape page-id] :as error} file-data _]
@@ -387,7 +387,7 @@
     (-> (pcb/empty-changes nil page-id)
         (pcb/with-file-data file-data)
         (pcb/update-shapes [(:id shape)] repair-shape)
-        (pcb/change-parent uuid/zero [shape] nil {:component-swap true}))))
+        (pcb/change-parent uuid/zero [shape] nil {:allow-altering-copies true}))))
 
 (defmethod repair-error :root-copy-not-allowed
   [_ {:keys [shape page-id] :as error} file-data _]
@@ -572,6 +572,46 @@
         (pcb/with-file-data file-data)
         (pcb/update-shapes [(:id shape)] repair-shape))))
 
+(defmethod repair-error :not-a-variant
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :invalid-variant-id
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :invalid-variant-properties
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :variant-not-main
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :parent-not-variant
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :variant-bad-name
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :variant-bad-variant-name
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
+(defmethod repair-error :variant-component-bad-name
+  [_ error file _]
+  (log/error :hint "Variant error code, we don't want to auto repair it for now" :code (:code error))
+  file)
+
 (defmethod repair-error :default
   [_ error file _]
   (log/error :hint "Unknown error code, don't know how to repair" :code (:code error))