WIP

Make it more scallable and make it easily extensible

.circleci/config.yml

@@ -226,29 +226,14 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
 
-      # Build frontend
       - run:
-          name: "frontend build"
+          name: "integration tests"
           working_directory: "./frontend"
           command: |
             yarn install
             yarn run build:app:assets
             yarn run build:app
             yarn run build:app:libs
-
-          # Build the wasm bundle
-      - run:
-          name: "wasm build"
-          working_directory: "./render-wasm"
-          command: |
-            EMSDK_QUIET=1 . /opt/emsdk/emsdk_env.sh
-            ./build release
-
-      # Run integration tests
-      - run:
-          name: "integration tests"
-          working_directory: "./frontend"
-          command: |
             yarn run playwright install chromium
             yarn run test:e2e -x --workers=4
 

.github/workflows/build-bundles.yml

@@ -1,45 +1,18 @@
-name: Bundles Builder
+name: Build and Upload Penpot Bundles
 
 on:
   # Create bundle from manual action
   workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
   workflow_call:
     inputs:
       gh_ref:
-        description: 'Name of the branch or ref'
+        description: 'Name of the branch'
         type: string
         required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
 
 jobs:
-  build-bundle:
-    name: Build and Upload Penpot Bundle
+  build-bundles:
+    name: Build and Upload Penpot Bundles
     runs-on: ubuntu-24.04
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -56,12 +29,13 @@ jobs:
       - name: Extract some useful variables
         id: vars
         run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+          echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "gh_branch=${{ github.base_ref || github.ref_name }}" >> $GITHUB_OUTPUT
 
-      - name: Build bundle
-        env:
-          BUILD_WASM: ${{ inputs.build_wasm }}
-          BUILD_STORYBOOK: ${{ inputs.build_storybook }}
+      - name: Set up Docker Buildx for multi-arch build
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run manage.sh build-bundle from host
         run: ./manage.sh build-bundle
 
       - name: Prepare directories for zipping
@@ -69,14 +43,15 @@ jobs:
           mkdir zips
           mv bundles penpot
 
-      - name: Create zip bundle
+      - name: Create zip bundles
         run: |
-          echo "📦 Packaging Penpot bundle..."
+          echo "📦 Packaging Penpot bundles..."
           zip -r zips/penpot.zip penpot
 
       - name: Upload Penpot bundle to S3
         run: |
-          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip
+          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_branch}}-latest.zip
+          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.commit_hash }}.zip
 
       - name: Notify Mattermost
         if: failure()
@@ -85,5 +60,5 @@ jobs:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
           TEXT: |
             ❌ *[PENPOT] Error during the execution of the job*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_branch}}`
             🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/build-develop.yml

@@ -1,21 +1,12 @@
-name: _DEVELOP
+name: Build and Upload Penpot DEVELOP Bundles
 
 on:
   schedule:
     - cron: '16 5-20 * * 1-5'
 
 jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: "develop"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
+  build-develop-bundle:
+    uses: ./.github/workflows/build-bundles.yml
     secrets: inherit
     with:
       gh_ref: "develop"

.github/workflows/build-docker-devenv.yml

@@ -1,36 +0,0 @@
-name: DevEnv Docker Image Builder
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build-and-push:
-    name: Build and push DevEnv Docker image
-    environment: release-admins
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.PUB_DOCKER_USERNAME }}
-          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      - name: Build and push DevEnv Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'penpotapp/devenv'
-        with:
-          context: ./docker/devenv/
-          file: ./docker/devenv/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ env.DOCKER_IMAGE }}:latest
-          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache,mode=max

.github/workflows/build-docker.yml

@@ -1,101 +0,0 @@
-name: Docker Images Builder
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-
-jobs:
-  build-and-push:
-    name: Build and Push Penpot Docker Images
-    runs-on: ubuntu-24.04-arm
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.gh_ref }}
-
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Download Penpot Bundles
-        env:
-          FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        run: |
-          pushd docker/images
-          aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
-          unzip $FILE_NAME > /dev/null
-          mv penpot/backend bundle-backend
-          mv penpot/frontend bundle-frontend
-          mv penpot/exporter bundle-exporter
-          popd
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.DOCKER_REGISTRY }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build and push Backend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'backend'
-          BUNDLE_PATH: './bundle-backend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.backend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Frontend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'frontend'
-          BUNDLE_PATH: './bundle-frontend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.frontend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Exporter Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'exporter'
-          BUNDLE_PATH: './bundle-exporter'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.exporter
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max

.github/workflows/build-staging.yml

@@ -1,21 +1,12 @@
-name: _STAGING
+name: Build and Upload Penpot STAGING Bundles
 
 on:
   schedule:
-    - cron: '36 5-20 * * 1-5'
+    - cron: '0 5 * * 1-5'
 
 jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: "staging"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
+  build-staging-bundle:
+    uses: ./.github/workflows/build-bundles.yml
     secrets: inherit
     with:
       gh_ref: "staging"

.github/workflows/build-tag.yml

@@ -1,30 +0,0 @@
-name: _TAG
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-      build_wasm: "no"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-
-  # publish-final-tag:
-  #   if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta')  && contains(github.ref_name, '.') }}
-  #   needs: build-docker
-  #   uses: ./.github/workflows/release.yml
-  #   secrets: inherit
-  #   with:
-  #     gh_ref: ${{ github.ref_name }}

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind):)\s["A-Z].*[^.]$'
+          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/release.yml

@@ -1,95 +0,0 @@
-name: Release Publisher
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-
-permissions:
-  contents: write
-
-jobs:
-  release:
-    environment: release-admins
-    runs-on: ubuntu-24.04
-    outputs:
-      version: ${{ steps.vars.outputs.gh_ref }}
-      release_notes: ${{ steps.extract_release_notes.outputs.release_notes }}
-    steps:
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ steps.vars.outputs.gh_ref }}
-
-      # # --- Publicly release the docker images ---
-      # - name: Login to private registry
-      #   uses: docker/login-action@v3
-      #   with:
-      #     registry: ${{ secrets.DOCKER_REGISTRY }}
-      #     username: ${{ secrets.DOCKER_USERNAME }}
-      #     password: ${{ secrets.DOCKER_PASSWORD }}
-
-      # - name: Login to DockerHub
-      #   uses: docker/login-action@v3
-      #   with:
-      #     username: ${{ secrets.PUB_DOCKER_USERNAME }}
-      #     password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      # - name: Publish docker images to DockerHub
-      #   env:
-      #     TAG: ${{ steps.vars.outputs.gh_ref }}
-      #     REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
-      #     HUB: ${{ secrets.PUB_DOCKER_HUB }}
-      #   run: |
-      #     IMAGES=("frontend" "backend" "exporter")
-      #     EXTRA_TAGS=("main" "latest")
-
-      #     for image in "${IMAGES[@]}"; do
-      #       docker pull "$REGISTRY/penpotapp/$image:$TAG"
-      #       docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$TAG"
-      #       docker push "penpotapp/$image:$TAG"
-
-      #       for tag in "${EXTRA_TAGS[@]}"; do
-      #         docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$tag"
-      #         docker push "penpotapp/$image:$tag"
-      #       done
-      #     done
-
-      # --- Release notes extraction ---
-      - name: Extract release notes from CHANGES.md
-        id: extract_release_notes
-        env:
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-        run: |
-          RELEASE_NOTES=$(awk "/^## $TAG$/{flag=1; next} /^## /{flag=0} flag" CHANGES.md | awk '{$1=$1};1')
-          if [ -z "$RELEASE_NOTES" ]; then
-            RELEASE_NOTES="No changes for $TAG according to CHANGES.md"
-          fi
-          echo "release_notes<<EOF" >> $GITHUB_OUTPUT
-          echo "$RELEASE_NOTES" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-
-      # --- Create GitHub release ---
-      - name: Create GitHub release
-        uses: softprops/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ steps.vars.outputs.gh_ref }}
-          name: ${{ steps.vars.outputs.gh_ref }}
-          body: ${{ steps.extract_release_notes.outputs.release_notes }}

.gitignore

@@ -31,7 +31,6 @@
 /.clj-kondo/.cache
 /_dump
 /notes
-/playground/
 /backend/*.md
 /backend/*.sql
 /backend/*.txt

.nvmrc

@@ -1 +1 @@
-v22.19.0
+v22.13.1

CHANGES.md

@@ -1,86 +1,16 @@
 # CHANGELOG
 
-## 2.11.0 (Unreleased)
+## 2.10.0 (Unreleased)
 
 ### :rocket: Epics and highlights
 
-- Deprecated configuration variables with the prefix `PENPOT_ASSETS_*`, and will be
-  removed in future versions:
-
-  - The `PENPOT_ASSETS_STORAGE_BACKEND` becomes `PENPOT_OBJECTS_STORAGE_BACKEND` and its
-    values passes from (`assets-fs` or `assets-s3`) to (`fs` or `s3`)
-  - The `PENPOT_STORAGE_ASSETS_FS_DIRECTORY` becomes `PENPOT_OBJECTS_STORAGE_FS_DIRECTORY`
-  - The `PENPOT_STORAGE_ASSETS_S3_BUCKET` becomes `PENPOT_OBJECTS_STORAGE_S3_BUCKET`
-  - The `PENPOT_STORAGE_ASSETS_S3_REGION` becomes `PENPOT_OBJECTS_STORAGE_S3_REGION`
-  - The `PENPOT_STORAGE_ASSETS_S3_ENDPOINT` becomes `PENPOT_OBJECTS_STORAGE_S3_ENDPOINT`
-  - The `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` replaced (see below)
-
-- Add `PENPOT_NETTY_IO_THREADS` and `PENPOT_EXECUTOR_THREADS` variables to provide the
-  control over concurrency of the shared resources used by netty. Penpot uses the netty IO
-  threads for AWS S3 SDK and Redis/Valkey communication, and the EXEC threads to perform
-  out of HTTP serving threads tasks such that cache invalidation, S3 response completion,
-  configuration reloading and many other auxiliar tasks. By default they use a half number
-  if available cpus with a minumum of 2 for both executors. You should not touch that
-  variables unless you are know what you are doing.
-
-- Replace the `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` with a more general configuration
-  `PENPOT_NETTY_IO_THREADS` used to configure a shared netty resources across different
-  services which use netty internally (redis connection, S3 SDK client). This
-  configuration is not very commonly used so don't expected real impact on any user.
-
-### :heart: Community contributions (Thank you!)
-
-### :sparkles: New features & Enhancements
-- Show current Penpot version [Taiga #11603](https://tree.taiga.io/project/penpot/us/11603)
-- Switch several variant copies at the same time [Taiga #11411](https://tree.taiga.io/project/penpot/us/11411)
-- Invitations management improvements [Taiga #3479](https://tree.taiga.io/project/penpot/us/3479)
-- Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
-
-### :bug: Bugs fixed
-
-- Fix selection problems when devtools open [Taiga #11950](https://tree.taiga.io/project/penpot/issue/11950)
-- Fix long font names overlap [Taiga #11844](https://tree.taiga.io/project/penpot/issue/11844)
-- Fix paste behavior according to the selected element [Taiga #11979](https://tree.taiga.io/project/penpot/issue/11979)
-- Fix problem with export size [Github #7160](https://github.com/penpot/penpot/issues/7160)
-- Fix multi level library dependencies [Taiga #12155](https://tree.taiga.io/project/penpot/issue/12155)
-- Fix component context menu options order in assets tab [Taiga #11941](https://tree.taiga.io/project/penpot/issue/11941)
-
-## 2.10.0
-
-### :rocket: Epics and highlights
-
-- Variants
-
 ### :boom: Breaking changes & Deprecations
 
 ### :heart: Community contributions (Thank you!)
 
 ### :sparkles: New features & Enhancements
-
-- Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
 - Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
 - Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
-- New Number token - unitless values  [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
-- New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
-- New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
-- New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
-- New letter spacing token [Taiga #10940](https://tree.taiga.io/project/penpot/us/10940)
-- New font weight token [Taiga #10939](https://tree.taiga.io/project/penpot/us/10939)
-- Upgrade Node to v22.18.0 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Upgrade the base docker image for penpot frontend to v1.29.1 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Create variant from an existing component [Taiga #2088](https://tree.taiga.io/project/penpot/us/2088)
-- Create variant from an existing variant [Taiga #8282](https://tree.taiga.io/project/penpot/us/8282)
-- Actions over a component with variants [Taiga #10503](https://tree.taiga.io/project/penpot/us/10503)
-- Create a variant by dragging a component into a component with variants [Taiga #8134](https://tree.taiga.io/project/penpot/us/8134)
-- Transform a variant into an individual component [Taiga #8141](https://tree.taiga.io/project/penpot/us/8141)
-- Delete variant [Taiga #6890](https://tree.taiga.io/project/penpot/us/6890)
-- Restore an orphaned copy of a variant [Taiga #10446](https://tree.taiga.io/project/penpot/us/10446)
-- Add, Edit & Delete variant properties name and value [Taiga #6892](https://tree.taiga.io/project/penpot/us/6892)
-- Retrieve variants [Taiga #6888](https://tree.taiga.io/project/penpot/us/6888)
-- Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
-- Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
-- Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
-- Fix problem with component swapping panel [Taiga #12175](https://tree.taiga.io/project/penpot/issue/12175)
 
 ### :bug: Bugs fixed
 
@@ -93,25 +23,8 @@
 - Fix font size/variant not updated when editing a text [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11552)
 - Fix issue where Alt + arrow keys shortcut interferes with letter-spacing when moving text layers [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11771)
 - Fix consistency issues on how font variants are visualized [Taiga #11499](https://tree.taiga.io/project/penpot/us/11499)
-- Fix parsing rx and ry SVG values for rect radius [Taiga #11861](https://tree.taiga.io/project/penpot/issue/11861)
-- Fix misleading affordance in saved versions [Taiga #11887](https://tree.taiga.io/project/penpot/issue/11887)
-- Fix pasting RTF text crashes penpot [Taiga #11717](https://tree.taiga.io/project/penpot/issue/11717)
-- Fix navigation arrows in Libraries & Templates carousel [Taiga #10609](https://tree.taiga.io/project/penpot/issue/10609)
-- Fix applying tokens with zero value to size [Taiga #11618](https://tree.taiga.io/project/penpot/issue/11618)
-- Fix typo [Taiga #11969](https://tree.taiga.io/project/penpot/issue/11969)
-- Fix typo [Taiga #11970](https://tree.taiga.io/project/penpot/issue/11970)
-- Fix typos [Taiga #11971](https://tree.taiga.io/project/penpot/issue/11971)
-- Fix inconsistent naming for "Flatten" [Taiga #8371](https://tree.taiga.io/project/penpot/issue/8371)
-- Layout item tokens should be unapplied when moving out of a layout [Taiga #11012](https://tree.taiga.io/project/penpot/issue/11012)
-- Fix incorrect date displayed for support plan [Taiga #11986](https://tree.taiga.io/project/penpot/issue/11986)
-- Fix can't import 'borderWidth' type token [#132](https://github.com/tokens-studio/penpot/issues/132)
-- Fix moving elements up or down while pressing alt [Taiga Issue #11992](https://tree.taiga.io/project/penpot/issue/11992)
-- Fix conflicting shortcuts (remove dec/inc line height and letter spacing) [Taiga #12102](https://tree.taiga.io/project/penpot/issue/12102)
-- Fix conflicting shortcuts (remove text-align shortcuts) [Taiga #12047](https://tree.taiga.io/project/penpot/issue/12047)
-- Fix export file with empty tokens library [Taiga #12137](https://tree.taiga.io/project/penpot/issue/12137)
-- Fix context menu on spacing tokens [Taiga #12141](https://tree.taiga.io/project/penpot/issue/12141)
 
-## 2.9.0
+## 2.9.0 (Unreleased)
 
 ### :rocket: Epics and highlights
 
@@ -139,8 +52,6 @@
 - Add the ability to show login dialog on profile settings [Github #6871](https://github.com/penpot/penpot/pull/6871)
 - Improve the application of tokens with object specific tokens [Taiga #10209](https://tree.taiga.io/project/penpot/us/10209)
 - Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
-- Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
-
 
 ### :bug: Bugs fixed
 
@@ -189,7 +100,7 @@
 
 **Penpot Library**
 
-The initial prototype is completly reworked to provide a more consistent API
+The initial prototype is completly reworked for provide a more consistent API
 and to have proper validation and params decoding. All the details can be found
 on [its own changelog](library/CHANGES.md)
 

README.md

@@ -77,14 +77,17 @@ Provide your team or organization with a completely owned collaborative design t
 ### Integrations ###
 Penpot offers integration into the development toolchain, thanks to its support for webhooks and an API accessible through access tokens.
 
-###  Building Design Systems: design tokens, components and variants  ###
-Penpot brings design systems to code-minded teams: a single source of truth with native Design Tokens, Components, and Variants for scalable, reusable, and consistent UI across projects and platforms.
+### What’s great for design ###
+With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
+
+### Design Tokens ###
+With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
 
 
 <br />
 
 <p align="center">
- <img src="https://github.com/user-attachments/assets/cce75ad6-f783-473f-8803-da9eb8255fef">
+  <img src="https://img.plasmic.app/img-optimizer/v1/img?src=https%3A%2F%2Fimg.plasmic.app%2Fimg-optimizer%2Fv1%2Fimg%2F9dd677c36afb477e9666ccd1d3f009ad.png" alt="Open Source" style="width: 65%;">
 </p>
 
 <br />

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.2"}
+  org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.7-4"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,7 +17,7 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.8.1.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.7.0.RELEASE"}
   ;; Minimal dependencies required by lettuce, we need to include them
   ;; explicitly because clojure dependency management does not support
   ;; yet the BOM format.
@@ -28,30 +28,29 @@
   com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.6"
-   :git/sha "94dc017"
+  {:git/tag "v11.4"
+   :git/sha "ce50d42"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
   com.github.seancorfield/next.jdbc
-  {:mvn/version "1.3.1070"}
-
+  {:mvn/version "1.3.1002"}
   metosin/reitit-core {:mvn/version "0.9.1"}
-  nrepl/nrepl {:mvn/version "1.4.0"}
+  nrepl/nrepl {:mvn/version "1.3.1"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.7"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}
+  org.postgresql/postgresql {:mvn/version "42.7.6"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "7.0.2"}
+  com.zaxxer/HikariCP {:mvn/version "6.3.0"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.2"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.21.2"}
+  org.jsoup/jsoup {:mvn/version "1.20.1"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -61,12 +60,12 @@
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
-  dawran6/emoji {:mvn/version "0.2.0"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.4"}
+  dawran6/emoji {:mvn/version "0.1.5"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.3"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.33.10"}}
+  software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -81,14 +80,12 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:mvn/version "0.10.10"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test
   {:main-opts ["-m" "kaocha.runner"]
-   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
-              "--sun-misc-unsafe-memory-access=allow"
-              "--enable-native-access=ALL-UNNAMED"]
+   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"]
    :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
 
   :outdated

backend/dev/user.clj

@@ -6,14 +6,12 @@
 
 (ns user
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.debug :as debug]
    [app.common.exceptions :as ex]
    [app.common.files.helpers :as cfh]
    [app.common.fressian :as fres]
    [app.common.geom.matrix :as gmt]
-   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.perf :as perf]
    [app.common.pprint :as pp]
@@ -21,9 +19,8 @@
    [app.common.schema.desc-js-like :as smdj]
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.generators :as sg]
-   [app.common.schema.openapi :as oapi]
    [app.common.spec :as us]
-   [app.common.time :as ct]
+   [app.common.json :as json]
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
@@ -33,9 +30,9 @@
    [app.srepl.helpers :as srepl.helpers]
    [app.srepl.main :as srepl]
    [app.util.blob :as blob]
+   [app.common.time :as ct]
    [clj-async-profiler.core :as prof]
    [clojure.contrib.humanize :as hum]
-   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]

backend/resources/app/email/invite-to-team/en.subj

@@ -1 +1 @@
-{{invited-by|abbreviate:25}} has invited you to join the team “{{ team|abbreviate:25 }}”
+Invitation to join {{team}}

backend/scripts/repl

@@ -31,7 +31,8 @@ export PENPOT_FLAGS="\
        disable-tiered-file-data-storage \
        enable-file-validation \
        enable-file-schema-validation \
-       enable-subscriptions";
+       enable-subscriptions \
+       disable-subscriptions-old";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"
@@ -77,14 +78,10 @@ export JAVA_OPTS="\
        -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
        -Djdk.tracePinnedThreads=full \
        -Dim4java.useV7=true \
-       -XX:+UseShenandoahGC \
        -XX:+EnableDynamicAgentLoading \
        -XX:-OmitStackTraceInFastThrow  \
-       -XX:+UnlockExperimentalVMOptions \
        -XX:+UnlockDiagnosticVMOptions \
        -XX:+DebugNonSafepoints \
-       -XX:ShenandoahGCMode=generational \
-       -XX:+UseCompactObjectHeaders \
        --sun-misc-unsafe-memory-access=allow \
        --enable-preview \
        --enable-native-access=ALL-UNNAMED";

backend/scripts/start-dev

@@ -24,7 +24,8 @@ export PENPOT_FLAGS="\
        disable-tiered-file-data-storage \
        enable-file-validation \
        enable-file-schema-validation \
-       enable-subscriptions";
+       enable-subscriptions \
+       disable-subscriptions-old";
 
 # Default deletion delay for devenv
 export PENPOT_DELETION_DELAY="24h"

backend/src/app/binfile/common.clj

@@ -19,7 +19,6 @@
    [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.common.weak :as weak]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
@@ -34,7 +33,8 @@
    [clojure.set :as set]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
-   [datoteka.io :as io]))
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
 
 (set! *warn-on-reflection* true)
 
@@ -141,13 +141,11 @@
   ([index coll attr]
    (reduce #(index-object %1 %2 attr) index coll)))
 
-(defn decode-row
-  [{:keys [data changes features] :as row}]
+(defn- decode-row-features
+  [{:keys [features] :as row}]
   (when row
     (cond-> row
-      features (assoc :features (db/decode-pgarray features #{}))
-      changes  (assoc :changes (blob/decode changes))
-      data     (assoc :data (blob/decode data)))))
+      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
 
 (def sql:get-minimal-file
   "SELECT f.id,
@@ -161,6 +159,7 @@
   [cfg id & {:as opts}]
   (db/get-with-sql cfg [sql:get-minimal-file id] opts))
 
+;; DEPRECATED
 (defn decode-file
   "A general purpose file decoding function that resolves all external
   pointers, run migrations and return plain vanilla file map"
@@ -168,7 +167,8 @@
   (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)]
     (let [file (->> file
                     (fmigr/resolve-applied-migrations cfg)
-                    (fdata/resolve-file-data cfg))
+                    (fdata/resolve-file-data cfg)
+                    (fdata/decode-file-data cfg))
           libs (delay (get-resolved-file-libraries cfg file))]
 
       (-> file
@@ -179,6 +179,119 @@
           (update :data assoc :id id)
           (cond-> migrate? (fmg/migrate-file libs))))))
 
+(def sql:get-file
+  "SELECT f.id,
+          f.project_id,
+          f.created_at,
+          f.modified_at,
+          f.deleted_at,
+          f.name,
+          f.is_shared,
+          f.has_media_trimmed,
+          f.revn,
+          f.data AS legacy_data,
+          f.ignore_sync_until,
+          f.comment_thread_seqn,
+          f.features,
+          f.version,
+          f.vern,
+          p.team_id,
+          coalesce(fd.backend, 'db') AS backend,
+          fd.metadata AS metadata,
+          fd.data AS data
+     FROM file AS f
+     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
+    INNER JOIN project AS p ON (p.id = f.project_id)
+   WHERE f.id = ?")
+
+(defn- migrate-file
+  [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)
+            pmap/*tracked* (pmap/create-tracked)]
+    (let [libs (delay (get-resolved-file-libraries cfg file))
+          ;; For avoid unnecesary overhead of creating multiple
+          ;; pointers and handly internally with objects map in their
+          ;; worst case (when probably all shapes and all pointers
+          ;; will be readed in any case), we just realize/resolve them
+          ;; before applying the migration to the file.
+          file (-> (fdata/realize cfg file)
+                   (fmg/migrate-file libs))]
+
+      (if (or read-only? (db/read-only? conn))
+        file
+        (do ;; When file is migrated, we break the rule of no
+          ;; perform mutations on get operations and update the
+          ;; file with all migrations applied
+          (update-file! cfg file)
+          (fmigr/resolve-applied-migrations cfg file))))))
+
+;; FIXME: filter by project-id
+(defn- get-file*
+  [{:keys [::db/conn] :as cfg} id
+   {:keys [#_project-id
+           migrate?
+           realize?
+           decode?
+           skip-locked?
+           include-deleted?
+           throw-if-not-exists?
+           lock-for-update?]
+    :or {lock-for-update? false
+         migrate? true
+         decode? true
+         include-deleted? false
+         throw-if-not-exists? true
+         realize? false}
+    :as options}]
+
+  (assert (db/connection? conn) "expected cfg with valid connection")
+
+  (let [sql
+        (if lock-for-update?
+          (str sql:get-file " FOR UPDATE of f")
+          sql:get-file)
+
+        sql
+        (if skip-locked?
+          (str sql " SKIP LOCKED")
+          sql)
+
+        file
+        (db/get-with-sql conn [sql id]
+                         {::db/throw-if-not-exists false
+                          ::db/remove-deleted (not include-deleted?)})
+
+        file
+        (-> file
+            (d/update-when :features db/decode-pgarray #{})
+            (d/update-when :metadata fdata/decode-metadata))]
+
+    (if file
+      (let [file
+            (->> file
+                 (fmigr/resolve-applied-migrations cfg)
+                 (fdata/resolve-file-data cfg))
+
+            will-migrate?
+            (and migrate? (fmg/need-migration? file))]
+
+        (if decode?
+          (cond->> (fdata/decode-file-data cfg file)
+            (and realize? (not will-migrate?))
+            (fdata/realize cfg)
+
+            will-migrate?
+            (migrate-file cfg options))
+
+          file))
+
+      (when-not (or skip-locked? (not throw-if-not-exists?))
+        (ex/raise :type :not-found
+                  :code :object-not-found
+                  :hint "database object not found"
+                  :table :file
+                  :file-id id)))))
+
 (defn get-file
   "Get file, resolve all features and apply migrations.
 
@@ -186,10 +299,7 @@
   operations on file, because it removes the ovehead of lazy fetching
   and decoding."
   [cfg file-id & {:as opts}]
-  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                 (when-let [row (db/get* conn :file {:id file-id}
-                                         (assoc opts ::db/remove-deleted false))]
-                   (decode-file cfg row opts)))))
+  (db/run! cfg get-file* file-id opts))
 
 (defn clean-file-features
   [file]
@@ -213,12 +323,12 @@
   (let [conn (db/get-connection cfg)
         ids  (db/create-array conn "uuid" ids)]
     (->> (db/exec! conn [sql:get-teams ids])
-         (map decode-row))))
+         (map decode-row-features))))
 
 (defn get-team
   [cfg team-id]
   (-> (db/get cfg :team {:id team-id})
-      (decode-row)))
+      (decode-row-features)))
 
 (defn get-fonts
   [cfg team-id]
@@ -475,7 +585,7 @@
         (vary-meta dissoc ::fmg/migrated))))
 
 (defn encode-file
-  [cfg {:keys [id features] :as file}]
+  [{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}]
   (let [file (if (and (contains? features "fdata/objects-map")
                       (:data file))
                (fdata/enable-objects-map file)
@@ -492,26 +602,48 @@
 
     (-> file
         (d/update-when :features into-array)
-        (d/update-when :data blob/encode))))
+        (d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data)))))))
 
 (defn- file->params
   [file]
   (-> (select-keys file file-attrs)
+      (assoc :data nil)
       (dissoc :team-id)
       (dissoc :migrations)))
 
+(defn file->file-data-params
+  [{:keys [id backend] :as file} & {:as opts}]
+  (let [created-at  (or (:created-at file) (ct/now))
+        modified-at (or (:modified-at file) created-at)
+        backend     (if (and (::overwrite-storage-backend opts) backend)
+                      backend
+                      (cf/get :file-storage-backend))]
+
+    (d/without-nils
+     {:id id
+      :type "main"
+      :file-id id
+      :data (:data file)
+      :metadata (:metadata file)
+      :backend backend
+      :created-at created-at
+      :modified-at modified-at})))
+
 (defn insert-file!
   "Insert a new file into the database table. Expectes a not-encoded file.
   Returns nil."
   [{:keys [::db/conn] :as cfg} file & {:as opts}]
-
   (when (:migrations file)
     (fmigr/upsert-migrations! conn file))
 
   (let [file (encode-file cfg file)]
     (db/insert! conn :file
                 (file->params file)
-                {::db/return-keys false})
+                (assoc opts ::db/return-keys false))
+
+    (->> (file->file-data-params file)
+         (fdata/update! cfg))
+
     nil))
 
 (defn update-file!
@@ -525,21 +657,25 @@
   (let [file
         (encode-file cfg file)
 
-        params
-        (file->params (dissoc file :id))]
+        file-params
+        (file->params (dissoc file :id))
 
-    (db/update! conn :file params
+        file-data-params
+        (file->file-data-params file)]
+
+    (db/update! conn :file file-params
                 {:id id}
                 {::db/return-keys false})
 
+    (fdata/update! cfg file-data-params)
     nil))
 
 (defn save-file!
   "Applies all the final validations and perist the file, binfile
   specific, should not be used outside of binfile domain.
-
   Returns nil"
   [{:keys [::timestamp] :as cfg} file & {:as opts}]
+
   (assert (ct/inst? timestamp) "expected valid timestamp")
 
   (let [file (-> file
@@ -603,22 +739,14 @@
          ;; FIXME: :is-indirect set to false to all rows looks
          ;; completly useless
          (map #(assoc % :is-indirect false))
-         (map decode-row))
+         (map decode-row-features))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
+;; FIXME: this will use a lot of memory if file uses too many big
+;; libraries, we should load required libraries on demand
 (defn get-resolved-file-libraries
-  "Get all file libraries including itself. Returns an instance of
-  LoadableWeakValueMap that allows do not have strong references to
-  the loaded libraries and reduce possible memory pressure on having
-  all this libraries loaded at same time on processing file validation
-  or file migration.
-
-  This still requires at least one library at time to be loaded while
-  access to it is performed, but it improves considerable not having
-  the need of loading all the libraries at the same time."
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [library-ids (->> (get-file-libraries conn (:id file))
-                         (map :id)
-                         (cons (:id file)))
-        load-fn     #(get-file cfg % :migrate? false)]
-    (weak/loadable-weak-value-map library-ids load-fn {id file})))
+  "A helper for preload file libraries"
+  [{:keys [::db/conn] :as cfg} file]
+  (->> (get-file-libraries conn (:id file))
+       (into [file] (map #(get-file cfg (:id %))))
+       (d/index-by :id)))

backend/src/app/binfile/migrations.clj

@@ -36,6 +36,11 @@
       "fdata/shape-data-type"
       nil
 
+      ;; There is no migration needed, but we don't want to allow
+      ;; copy paste nor import of variant files into no-variant teams
+      "variants/v1"
+      nil
+
       (ex/raise :type :internal
                 :code :no-migration-defined
                 :hint (str/ffmt "no migation for feature '%' on file importation" feature)

backend/src/app/binfile/v1.clj

@@ -346,7 +346,7 @@
           thumbnails (->> (bfc/get-file-object-thumbnails cfg file-id)
                           (mapv #(dissoc % :file-id)))
 
-          file       (cond-> (bfc/get-file cfg file-id)
+          file       (cond-> (bfc/get-file cfg file-id :realize? true)
                        detach?
                        (-> (ctf/detach-external-references file-id)
                            (dissoc :libraries))

backend/src/app/binfile/v2.clj

@@ -153,7 +153,7 @@
 
 (defn- write-file!
   [cfg file-id]
-  (let [file   (bfc/get-file cfg file-id)
+  (let [file   (bfc/get-file cfg file-id :realize? true)
         thumbs (bfc/get-file-object-thumbnails cfg file-id)
         media  (bfc/get-file-media cfg file)
         rels   (bfc/get-files-rels cfg #{file-id})]

backend/src/app/binfile/v3.clj

@@ -27,7 +27,7 @@
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
-   [app.common.types.tokens-lib :as ctob]
+   [app.common.types.tokens-lib :as cto]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -41,10 +41,8 @@
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
-   java.io.File
    java.io.InputStream
    java.io.OutputStreamWriter
-   java.lang.AutoCloseable
    java.util.zip.ZipEntry
    java.util.zip.ZipFile
    java.util.zip.ZipOutputStream))
@@ -105,25 +103,25 @@
   (sm/encoder ctp/schema:page sm/json-transformer))
 
 (def encode-shape
-  (sm/encoder cts/schema:shape sm/json-transformer))
+  (sm/encoder ::cts/shape sm/json-transformer))
 
 (def encode-media
-  (sm/encoder ctf/schema:media sm/json-transformer))
+  (sm/encoder ::ctf/media sm/json-transformer))
 
 (def encode-component
-  (sm/encoder ctc/schema:component sm/json-transformer))
+  (sm/encoder ::ctc/component sm/json-transformer))
 
 (def encode-color
   (sm/encoder ctcl/schema:library-color sm/json-transformer))
 
 (def encode-typography
-  (sm/encoder cty/schema:typography sm/json-transformer))
+  (sm/encoder ::cty/typography sm/json-transformer))
 
 (def encode-tokens-lib
-  (sm/encoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/encoder ::cto/tokens-lib sm/json-transformer))
 
 (def encode-plugin-data
-  (sm/encoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
 
 (def encode-storage-object
   (sm/encoder schema:storage-object sm/json-transformer))
@@ -140,7 +138,7 @@
   (sm/decoder ctf/schema:media sm/json-transformer))
 
 (def decode-component
-  (sm/decoder ctc/schema:component sm/json-transformer))
+  (sm/decoder ::ctc/component sm/json-transformer))
 
 (def decode-color
   (sm/decoder ctcl/schema:library-color sm/json-transformer))
@@ -149,19 +147,19 @@
   (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
-  (sm/decoder ctp/schema:page sm/json-transformer))
+  (sm/decoder ::ctp/page sm/json-transformer))
 
 (def decode-shape
-  (sm/decoder cts/schema:shape sm/json-transformer))
+  (sm/decoder ::cts/shape sm/json-transformer))
 
 (def decode-typography
-  (sm/decoder cty/schema:typography sm/json-transformer))
+  (sm/decoder ::cty/typography sm/json-transformer))
 
 (def decode-tokens-lib
-  (sm/decoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/decoder cto/schema:tokens-lib sm/json-transformer))
 
 (def decode-plugin-data
-  (sm/decoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/decoder ::ctpg/plugin-data sm/json-transformer))
 
 (def decode-storage-object
   (sm/decoder schema:storage-object sm/json-transformer))
@@ -175,31 +173,31 @@
   (sm/check-fn schema:manifest))
 
 (def validate-file
-  (sm/check-fn ctf/schema:file))
+  (sm/check-fn ::ctf/file))
 
 (def validate-page
-  (sm/check-fn ctp/schema:page))
+  (sm/check-fn ::ctp/page))
 
 (def validate-shape
-  (sm/check-fn cts/schema:shape))
+  (sm/check-fn ::cts/shape))
 
 (def validate-media
-  (sm/check-fn ctf/schema:media))
+  (sm/check-fn ::ctf/media))
 
 (def validate-color
   (sm/check-fn ctcl/schema:library-color))
 
 (def validate-component
-  (sm/check-fn ctc/schema:component))
+  (sm/check-fn ::ctc/component))
 
 (def validate-typography
-  (sm/check-fn cty/schema:typography))
+  (sm/check-fn ::cty/typography))
 
 (def validate-tokens-lib
-  (sm/check-fn ctob/schema:tokens-lib))
+  (sm/check-fn ::cto/tokens-lib))
 
 (def validate-plugin-data
-  (sm/check-fn ctpg/schema:plugin-data))
+  (sm/check-fn ::ctpg/plugin-data))
 
 (def validate-storage-object
   (sm/check-fn schema:storage-object))
@@ -224,9 +222,11 @@
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
-  (let [detach?  (and (not embed-assets) (not include-libraries))]
+  (let [detach? (and (not embed-assets) (not include-libraries))]
     (db/tx-run! cfg (fn [cfg]
-                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
+                      (cond-> (bfc/get-file cfg file-id
+                                            {:realize? true
+                                             :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
                             (dissoc :libraries))
@@ -253,9 +253,9 @@
         (write-entry! output path params)
 
         (with-open [input (sto/get-object-data storage sobject)]
-          (.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
+          (.putNextEntry output (ZipEntry. (str "objects/" id ext)))
           (io/copy input output :size (:size sobject))
-          (.closeEntry ^ZipOutputStream output))))))
+          (.closeEntry output))))))
 
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
@@ -349,8 +349,7 @@
             typography (encode-typography object)]
         (write-entry! output path typography)))
 
-    (when (and tokens-lib
-               (not (ctob/empty-lib? tokens-lib)))
+    (when tokens-lib
       (let [path           (str "files/" file-id "/tokens.json")
             encoded-tokens (encode-tokens-lib tokens-lib)]
         (write-entry! output path encoded-tokens)))))
@@ -450,7 +449,7 @@
 (defn- read-manifest
   [^ZipFile input]
   (let [entry (get-zip-entry input "manifest.json")]
-    (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+    (with-open [reader (zip-entry-reader input entry)]
       (let [manifest (json/read reader :key-fn json/read-kebab-key)]
         (decode-manifest manifest)))))
 
@@ -540,12 +539,12 @@
 
 (defn- read-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
 (defn- read-plain-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader)))
 
 (defn- read-file
@@ -1009,8 +1008,8 @@
     (try
       (l/info :hint "start exportation" :export-id (str id))
       (binding [bfc/*state* (volatile! (bfc/initial-state))]
-        (with-open [^AutoCloseable output (io/output-stream output)]
-          (with-open [^AutoCloseable output (ZipOutputStream. output)]
+        (with-open [output (io/output-stream output)]
+          (with-open [output (ZipOutputStream. output)]
             (let [cfg (assoc cfg ::output output)]
               (export-files cfg)
               (export-storage-objects cfg)))))
@@ -1054,7 +1053,7 @@
 
     (l/info :hint "import: started" :id (str id))
     (try
-      (with-open [input (ZipFile. ^File (fs/file input))]
+      (with-open [input (ZipFile. (fs/file input))]
         (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
@@ -1069,6 +1068,6 @@
 
 (defn get-manifest
   [path]
-  (with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
+  (with-open [input (ZipFile. (fs/file path))]
     (-> (read-manifest input)
         (validate-manifest))))

backend/src/app/config.clj

@@ -52,6 +52,8 @@
 
    :redis-uri "redis://redis/0"
 
+   :file-storage-backend "db"
+
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
@@ -96,7 +98,7 @@
     [:http-server-max-body-size {:optional true} ::sm/int]
     [:http-server-max-multipart-body-size {:optional true} ::sm/int]
     [:http-server-io-threads {:optional true} ::sm/int]
-    [:http-server-max-worker-threads {:optional true} ::sm/int]
+    [:http-server-worker-threads {:optional true} ::sm/int]
 
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
@@ -105,7 +107,8 @@
     [:auto-file-snapshot-timeout {:optional true} ::ct/duration]
 
     [:media-max-file-size {:optional true} ::sm/int]
-    [:deletion-delay {:optional true} ::ct/duration] ;; REVIEW
+    [:deletion-delay {:optional true} ::ct/duration]
+    [:file-clean-delay {:optional true} ::ct/duration]
     [:telemetry-enabled {:optional true} ::sm/boolean]
     [:default-blob-version {:optional true} ::sm/int]
     [:allow-demo-users {:optional true} ::sm/boolean]
@@ -210,25 +213,26 @@
     [:prepl-host {:optional true} :string]
     [:prepl-port {:optional true} ::sm/int]
 
+    [:file-storage-backend :string]
+
     [:media-directory {:optional true} :string] ;; REVIEW
     [:media-uri {:optional true} :string]
     [:assets-path {:optional true} :string]
 
-    [:netty-io-threads {:optional true} ::sm/int]
-    [:executor-threads {:optional true} ::sm/int]
-
-    ;; DEPRECATED
+    ;; Legacy, will be removed in 2.5
     [:assets-storage-backend {:optional true} :keyword]
     [:storage-assets-fs-directory {:optional true} :string]
     [:storage-assets-s3-bucket {:optional true} :string]
     [:storage-assets-s3-region {:optional true} :keyword]
     [:storage-assets-s3-endpoint {:optional true} ::sm/uri]
+    [:storage-assets-s3-io-threads {:optional true} ::sm/int]
 
     [:objects-storage-backend {:optional true} :keyword]
     [:objects-storage-fs-directory {:optional true} :string]
     [:objects-storage-s3-bucket {:optional true} :string]
     [:objects-storage-s3-region {:optional true} :keyword]
-    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]]))
+    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
+    [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
 (defn- parse-flags
   [config]
@@ -301,6 +305,11 @@
   (or (c/get config :deletion-delay)
       (ct/duration {:days 7})))
 
+(defn get-file-clean-delay
+  []
+  (or (c/get config :file-clean-delay)
+      (ct/duration {:days 2})))
+
 (defn get
   "A configuration getter. Helps code be more testable."
   ([key]

backend/src/app/db/sql.clj

@@ -53,15 +53,8 @@
          opts (cond-> opts
                 (::order-by opts)   (assoc :order-by (::order-by opts))
                 (::columns opts)    (assoc :columns (::columns opts))
-
-                (or (::db/for-update opts)
-                    (::for-update opts))
-                (assoc :suffix "FOR UPDATE")
-
-                (or (::db/for-share opts)
-                    (::for-share opts))
-                (assoc :suffix "FOR SHARE"))]
-
+                (::for-update opts) (assoc :suffix "FOR UPDATE")
+                (::for-share opts)  (assoc :suffix "FOR SHARE"))]
      (sql/for-query table where-params opts))))
 
 (defn update

backend/src/app/features/fdata.clj

@@ -12,22 +12,18 @@
    [app.common.files.helpers :as cfh]
    [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
-   [app.common.types.objects-map :as omap]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.types.path :as path]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap.legacy]
-   [app.util.pointer-map :as pmap]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; OFFLOAD
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn offloaded?
-  [file]
-  (= "objects-storage" (:data-backend file)))
+   [app.util.objects-map :as omap]
+   [app.util.pointer-map :as pmap]
+   [app.worker :as wrk]
+   [promesa.exec :as px]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OBJECTS-MAP
@@ -37,7 +33,10 @@
   [file & _opts]
   (let [update-page
         (fn [page]
-          (update page :objects omap/wrap))
+          (if (and (pmap/pointer-map? page)
+                   (not (pmap/loaded? page)))
+            page
+            (update page :objects omap/wrap)))
 
         update-data
         (fn [fdata]
@@ -47,20 +46,6 @@
         (update :data update-data)
         (update :features conj "fdata/objects-map"))))
 
-(defn disable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (update page :objects #(into {} %)))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features disj "fdata/objects-map"))))
-
 (defn process-objects
   "Apply a function to all objects-map on the file. Usualy used for convert
   the objects-map instances to plain maps"
@@ -70,42 +55,30 @@
             (fn [page]
               (update page :objects
                       (fn [objects]
-                        (if (or (omap/objects-map? objects)
-                                (omap.legacy/objects-map? objects))
+                        (if (omap/objects-map? objects)
                           (update-fn objects)
                           objects)))))
     fdata))
 
+
+(defn realize-objects
+  "Process a file and remove all instances of objects mao realizing them
+  to a plain data. Used in operation where is more efficient have the
+  whole file loaded in memory or we going to persist it in an
+  alterantive storage."
+  [_cfg file]
+  (update file :data process-objects (partial into {})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; POINTER-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn get-file-data
-  "Get file data given a file instance."
-  [system file]
-  (if (offloaded? file)
-    (let [storage (sto/resolve system ::db/reuse-conn true)]
-      (->> (sto/get-object storage (:data-ref-id file))
-           (sto/get-object-bytes storage)))
-    (:data file)))
-
-(defn resolve-file-data
-  [system file]
-  (let [data (get-file-data system file)]
-    (assoc file :data data)))
-
-(defn decode-file-data
-  [_system {:keys [data] :as file}]
-  (cond-> file
-    (bytes? data)
-    (assoc :data (blob/decode data))))
-
 (defn load-pointer
   "A database loader pointer helper"
-  [system file-id id]
-  (let [fragment (db/get* system :file-data-fragment
-                          {:id id :file-id file-id}
-                          {::sql/columns [:data :data-backend :data-ref-id :id]})]
+  [cfg file-id id]
+  (let [fragment (db/get* cfg :file-data
+                          {:id id :file-id file-id :type "fragment"}
+                          {::sql/columns [:content :backend :id]})]
 
     (l/trc :hint "load pointer"
            :file-id (str file-id)
@@ -119,22 +92,22 @@
                 :file-id file-id
                 :fragment-id id))
 
-    (let [data (get-file-data system fragment)]
-      ;; FIXME: conditional thread scheduling for decoding big objects
-      (blob/decode data))))
+    ;; FIXME: conditional thread scheduling for decoding big objects
+    (blob/decode (:data fragment))))
 
 (defn persist-pointers!
   "Persist all currently tracked pointer objects"
-  [system file-id]
-  (let [conn (db/get-connection system)]
+  [cfg file-id]
+  (let [conn (db/get-connection cfg)]
     (doseq [[id item] @pmap/*tracked*]
       (when (pmap/modified? item)
         (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
         (let [content (-> item deref blob/encode)]
-          (db/insert! conn :file-data-fragment
+          (db/insert! conn :file-data
                       {:id id
                        :file-id file-id
-                       :data content}))))))
+                       :type "fragment"
+                       :content content}))))))
 
 (defn process-pointers
   "Apply a function to all pointers on the file. Usuly used for
@@ -148,6 +121,14 @@
         (d/update-vals update-fn')
         (update :pages-index d/update-vals update-fn'))))
 
+(defn realize-pointers
+  "Process a file and remove all instances of pointers realizing them to
+  a plain data. Used in operation where is more efficient have the
+  whole file loaded in memory."
+  [cfg {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial load-pointer cfg id)]
+    (update file :data process-pointers deref)))
+
 (defn get-used-pointer-ids
   "Given a file, return all pointer ids used in the data."
   [fdata]
@@ -211,3 +192,314 @@
         (update :features disj "fdata/path-data")
         (update :migrations disj "0003-convert-path-content")
         (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GENERAL PURPOSE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn realize
+  "A helper that combines realize-pointers and realize-objects"
+  [cfg file]
+  (->> file
+       (realize-pointers cfg)
+       (realize-objects cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; STORAGE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmulti resolve-file-data
+  (fn [_cfg file] (or (get file :backend) "db")))
+
+(defmethod resolve-file-data "db"
+  [_cfg {:keys [legacy-data data] :as file}]
+  (if (and (some? legacy-data) (not data))
+    (-> file
+        (assoc :data legacy-data)
+        (dissoc :legacy-data))
+    (dissoc file :legacy-data)))
+
+(defmethod resolve-file-data "storage"
+  [cfg object]
+  (let [storage (sto/resolve cfg ::db/reuse-conn true)
+        ref-id  (-> object :metadata :storage-ref-id)
+        data    (->> (sto/get-object storage ref-id)
+                     (sto/get-object-bytes storage))]
+    (-> object
+        (assoc :data data)
+        (dissoc :legacy-data))))
+
+(defn decode-file-data
+  [{:keys [::wrk/executor]} {:keys [data] :as file}]
+  (cond-> file
+    (bytes? data)
+    (assoc :data (px/invoke! executor #(blob/decode data)))))
+
+(def ^:private sql:insert-file-data
+  "INSERT INTO file_data (file_id, id, created_at, modified_at,
+                          type, backend, metadata, data)
+   VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+
+(def ^:private sql:upsert-file-data
+  (str sql:insert-file-data
+       " ON CONFLICT (file_id, id)
+         DO UPDATE SET modified_at=?,
+                       backend=?,
+                       metadata=?,
+                       data=?;"))
+
+(defn- create-in-database
+  [cfg {:keys [id file-id created-at modified-at type backend data metadata]}]
+  (let [metadata    (some-> metadata db/json)
+        created-at  (or created-at (ct/now))
+        modified-at (or modified-at created-at)]
+    (db/exec-one! cfg [sql:insert-file-data
+                       file-id id
+                       created-at
+                       modified-at
+                       type
+                       backend
+                       metadata
+                       data])))
+
+(defn- upsert-in-database
+  [cfg {:keys [id file-id created-at modified-at type backend data metadata]}]
+  (let [metadata    (some-> metadata db/json)
+        created-at  (or created-at (ct/now))
+        modified-at (or modified-at created-at)]
+
+    (db/exec-one! cfg [sql:upsert-file-data
+                       file-id id
+                       created-at
+                       modified-at
+                       type
+                       backend
+                       metadata
+                       data
+                       modified-at
+                       backend
+                       metadata
+                       data])))
+
+(defmulti ^:private handle-persistence
+  (fn [_cfg params] (:backend params)))
+
+(defmethod handle-persistence "db"
+  [_ params]
+  (dissoc params :metadata))
+
+(defmethod handle-persistence "storage"
+  [{:keys [::sto/storage] :as cfg}
+   {:keys [id file-id data] :as params}]
+
+  (let [content  (sto/content data)
+        sobject  (sto/put-object! storage
+                                  {::sto/content content
+                                   ::sto/touch true
+                                   :bucket "file-data"
+                                   :content-type "application/octet-stream"
+                                   :file-id file-id
+                                   :id id})
+        metadata {:storage-ref-id (:id sobject)}]
+    (-> params
+        (assoc :metadata metadata)
+        (assoc :data nil))))
+
+(defn- process-metadata
+  [cfg metadata]
+  (when-let [storage-id (:storage-ref-id metadata)]
+    (let [storage (sto/resolve cfg ::db/reuse-conn true)]
+      (sto/touch-object! storage storage-id))))
+
+(defn- default-backend
+  [backend]
+  (or backend (cf/get :file-storage-backend "db")))
+
+(def ^:private schema:metadata
+  [:map {:title "Metadata"}
+   [:storage-ref-id {:optional true} ::sm/uuid]])
+
+(def decode-metadata-with-schema
+  (sm/decoder schema:metadata sm/json-transformer))
+
+(defn decode-metadata
+  [metadata]
+  (some-> metadata
+          (db/decode-json-pgobject)
+          (decode-metadata-with-schema)))
+
+(def ^:private schema:update-params
+  [:map {:closed true}
+   [:id ::sm/uuid]
+   [:type [:enum "main" "snapshot"]]
+   [:file-id ::sm/uuid]
+   [:backend {:optional true} [:enum "db" "storage"]]
+   [:metadata {:optional true} [:maybe schema:metadata]]
+   [:data {:optional true} bytes?]
+   [:created-at {:optional true} ::ct/inst]
+   [:modified-at {:optional true} ::ct/inst]])
+
+(def ^:private check-update-params
+  (sm/check-fn schema:update-params :hint "invalid params received for update"))
+
+(defn update!
+  [cfg params & {:keys [throw-if-not-exists?]}]
+  (let [params (-> (check-update-params params)
+                   (update :backend default-backend))]
+
+    (some->> (:metadata params) (process-metadata cfg))
+    (let [result (handle-persistence cfg params)
+          result (if throw-if-not-exists?
+                   (create-in-database cfg result)
+                   (upsert-in-database cfg result))]
+      (-> result db/get-update-count pos?))))
+
+(defn create!
+  [cfg params]
+  (update! cfg params :throw-on-conflict? true))
+
+(def ^:private schema:delete-params
+  [:map {:closed true}
+   [:id ::sm/uuid]
+   [:type [:enum "main" "snapshot"]]
+   [:file-id ::sm/uuid]])
+
+(def check-delete-params
+  (sm/check-fn schema:delete-params :hint "invalid params received for delete"))
+
+(defn delete!
+  [cfg params]
+  (when-let [fdata (db/get* cfg :file-data
+                            (check-delete-params params))]
+
+    (some->> (get fdata :metadata)
+             (decode-metadata)
+             (process-metadata cfg))
+
+    (-> (db/delete! cfg :file-data params)
+        (db/get-update-count)
+        (pos?))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SCRIPTS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def ^:private sql:get-unmigrated-files
+  "SELECT f.id, f.data, f.created_at, f.modified_at
+     FROM file AS f
+    WHERE f.data IS NOT NULL
+    ORDER BY f.modified_at ASC
+    LIMIT ?
+      FOR UPDATE
+     SKIP LOCKED")
+
+(defn migrate-files-to-storage
+  "Migrate the current existing files to store data in new storage
+  tables."
+  [system & {:keys [chunk-size] :or {chunk-size 100}}]
+  (db/tx-run! system
+              (fn [{:keys [::db/conn]}]
+                (reduce (fn [total {:keys [id data index created-at modified-at]}]
+                          (l/dbg :hint "migrating file" :file-id (str id))
+                          (db/update! conn :file {:data nil} {:id id} ::db/return-keys false)
+                          (db/insert! conn :file-data
+                                      {:backend "db"
+                                       :metadata nil
+                                       :type "main"
+                                       :data data
+                                       :created-at created-at
+                                       :modified-at modified-at
+                                       :file-id id
+                                       :id id}
+                                      {::db/return-keys false})
+                          (inc total))
+                        0
+                        (db/plan conn [sql:get-unmigrated-files chunk-size]
+                                 {:fetch-size 1})))))
+
+(def ^:private sql:get-migrated-files
+  "SELECT f.id, f.data
+     FROM file_data AS f
+    WHERE f.data IS NOT NULL
+      AND f.id = f.file_id
+    ORDER BY f.id ASC
+    LIMIT ?
+      FOR UPDATE
+     SKIP LOCKED")
+
+(defn rollback-files-from-storage
+  "Migrate back to the file table storage."
+  [system & {:keys [chunk-size] :or {chunk-size 100}}]
+  (db/tx-run! system
+              (fn [{:keys [::db/conn]}]
+                (reduce (fn [total {:keys [id data]}]
+                          (l/dbg :hint "rollback file" :file-id (str id))
+                          (db/update! conn :file {:data data} {:id id} ::db/return-keys false)
+                          (db/delete! conn :file-data {:id id} ::db/return-keys false)
+                          (inc total))
+                        0
+                        (db/plan conn [sql:get-migrated-files chunk-size]
+                                 {:fetch-size 1})))))
+
+(def ^:private sql:get-unmigrated-snapshots
+  "SELECT fc.id, fc.data, fc.file_id, fc.created_at, fc.updated_at AS modified_at
+     FROM file_change AS fc
+    WHERE fc.data IS NOT NULL
+      AND f.label IS NOT NULL
+    ORDER BY f.id ASC
+    LIMIT ?
+      FOR UPDATE
+     SKIP LOCKED")
+
+(defn migrate-snapshots-to-storage
+  "Migrate the current existing files to store data in new storage
+  tables."
+  [system & {:keys [chunk-size] :or {chunk-size 100}}]
+  (db/tx-run! system
+              (fn [{:keys [::db/conn]}]
+                (reduce (fn [total {:keys [id file-id data created-at modified-at]}]
+                          (l/dbg :hint "migrating snapshot" :file-id (str file-id) :id (str id))
+                          (db/update! conn :file-change {:data nil} {:id id :file-id file-id} ::db/return-keys false)
+                          (db/insert! conn :file-data
+                                      {:backend "db"
+                                       :metadata nil
+                                       :type "snapshot"
+                                       :data data
+                                       :created-at created-at
+                                       :modified-at modified-at
+                                       :file-id file-id
+                                       :id id}
+                                      {::db/return-keys false})
+                          (inc total))
+                        0
+                        (db/plan conn [sql:get-unmigrated-snapshots chunk-size]
+                                 {:fetch-size 1})))))
+
+(def ^:private sql:get-migrated-snapshots
+  "SELECT f.id, f.data, f.file_id
+     FROM file_data AS f
+    WHERE f.data IS NOT NULL
+      AND f.type = 'snapshot'
+      AND f.id != f.file_id
+    ORDER BY f.id ASC
+    LIMIT ?
+      FOR UPDATE
+     SKIP LOCKED")
+
+(defn rollback-snapshots-from-storage
+  "Migrate back to the file table storage."
+  [system & {:keys [chunk-size] :or {chunk-size 100}}]
+  (db/tx-run! system
+              (fn [{:keys [::db/conn]}]
+                (db/exec! conn ["SET statement_timeout = 0"])
+                (db/exec! conn ["SET idle_in_transaction_session_timeout = 0"])
+
+                (reduce (fn [total {:keys [id file-id data]}]
+                          (l/dbg :hint "rollback snapshot" :file-id (str id) :id (str id))
+                          (db/update! conn :file-change {:data data} {:id id :file-id file-id} ::db/return-keys false)
+                          (db/delete! conn :file-data {:id id :file-id file-id} ::db/return-keys false)
+                          (inc total))
+                        0
+                        (db/plan conn [sql:get-migrated-snapshots chunk-size]
+                                 {:fetch-size 1})))))

backend/src/app/features/file_snapshots.clj

@@ -0,0 +1,373 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.file-snapshots
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.features :as-alias cfeat]
+   [app.common.files.migrations :as fmg]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as-alias sql]
+   [app.features.fdata :as fdata]
+   [app.storage :as sto]
+   [app.util.blob :as blob]
+   [app.worker :as wrk]
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
+
+(def sql:snapshots
+  "SELECT c.id,
+          c.label,
+          c.created_at,
+          c.updated_at AS modified_at,
+          c.deleted_at,
+          c.profile_id,
+          c.created_by,
+          c.locked_by,
+          c.revn,
+          c.features,
+          c.migrations,
+          c.version,
+          c.file_id,
+          c.data AS legacy_data,
+          fd.data AS data,
+          coalesce(fd.backend, 'db') AS backend,
+          fd.metadata AS metadata
+     FROM file_change AS c
+     LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id
+                                   AND fd.id = c.id
+                                   AND fd.type = 'snapshot')
+    WHERE c.label IS NOT NULL")
+
+(def ^:private sql:get-snapshot
+  (str sql:snapshots " AND c.file_id = ? AND c.id = ?"))
+
+(def ^:private sql:get-snapshots
+  (str sql:snapshots " AND c.file_id = ?"))
+
+(def ^:private sql:get-snapshot-without-data
+  (str "WITH snapshots AS (" sql:snapshots ")"
+       "SELECT c.id,
+               c.label,
+               c.revn,
+               c.created_at,
+               c.modified_at,
+               c.deleted_at,
+               c.profile_id,
+               c.created_by,
+               c.features,
+               c.metadata,
+               c.migrations,
+               c.version,
+               c.file_id
+         FROM snapshots AS c
+        WHERE c.id = ?"))
+
+(defn- decode-snapshot
+  [snapshot]
+  (some-> snapshot (-> (d/update-when :metadata fdata/decode-metadata)
+                       (d/update-when :migrations db/decode-pgarray [])
+                       (d/update-when :features db/decode-pgarray #{}))))
+
+(def sql:get-minimal-file
+  "SELECT f.id,
+          f.revn,
+          f.modified_at,
+          f.deleted_at,
+          fd.backend AS backend,
+          fd.metadata AS metadata
+     FROM file AS f
+     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
+    WHERE f.id = ?")
+
+(defn get-minimal-file
+  [cfg id & {:as opts}]
+  (-> (db/get-with-sql cfg [sql:get-minimal-file id] opts)
+      (d/update-when :metadata fdata/decode-metadata)))
+
+(defn get-minimal-snapshot
+  [cfg snapshot-id]
+  (-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id])
+      (decode-snapshot)))
+
+(defn get-snapshot
+  "Get snapshot with decoded data"
+  [cfg file-id snapshot-id]
+  (->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id])
+       (decode-snapshot)
+       (fdata/resolve-file-data cfg)
+       (fdata/decode-file-data cfg)))
+
+(def ^:private sql:get-visible-snapshots
+  (str "WITH "
+       "snapshots1 AS ( " sql:snapshots "),"
+       "snapshots2 AS (
+          SELECT c.id,
+                 c.label,
+                 c.version,
+                 c.created_at,
+                 c.modified_at,
+                 c.created_by,
+                 c.locked_by,
+                 c.profile_id
+            FROM snapshots1 AS c
+           WHERE c.file_id = ?
+             AND (c.deleted_at IS NULL OR deleted_at > now())
+       ), snapshots3 AS (
+          (SELECT * FROM snapshots2 WHERE created_by = 'system' LIMIT 1000)
+          UNION ALL
+          (SELECT * FROM snapshots2 WHERE created_by != 'system' LIMIT 1000)
+       )
+       SELECT * FROM snapshots3
+        ORDER BY created_at DESC;"))
+
+(defn get-visible-snapshots
+  "Return a list of snapshots fecheable from the API, it has a limited
+  set of fields and applies big but safe limits over all available
+  snapshots. It return a ordered vector by the snapshot date of
+  creation."
+  [cfg file-id]
+  (->> (db/exec! cfg [sql:get-visible-snapshots file-id])
+       (mapv decode-snapshot)))
+
+(def ^:private schema:decoded-file
+  [:map {:title "DecodedFile"}
+   [:id ::sm/uuid]
+   [:revn :int]
+   [:vern :int]
+   [:data :map]
+   [:version :int]
+   [:features ::cfeat/features]
+   [:migrations [::sm/set :string]]])
+
+(def ^:private schema:snapshot
+  [:map {:title "Snapshot"}
+   [:id ::sm/uuid]
+   [:revn [::sm/int {:min 0}]]
+   [:version [::sm/int {:min 0}]]
+   [:features ::cfeat/features]
+   [:migrations [::sm/set ::sm/text]]
+   [:profile-id {:optional true} ::sm/uuid]
+   [:label ::sm/text]
+   [:file-id ::sm/uuid]
+   [:created-by [:enum "system" "user" "admin"]]
+   [:deleted-at {:optional true} ::ct/inst]
+   [:modified-at ::ct/inst]
+   [:created-at ::ct/inst]])
+
+(def ^:private schema:snapshot-params
+  [:map {:title "SnapshotParams"}
+   [:id ::sm/uuid]
+   [:file-id ::sm/uuid]
+   [:label ::sm/text]
+   [:modified-at {:optional true} ::ct/inst]])
+
+(def ^:private check-snapshot
+  (sm/check-fn schema:snapshot))
+
+(def ^:private check-snapshot-params
+  (sm/check-fn schema:snapshot-params))
+
+(def ^:private check-decoded-file
+  (sm/check-fn schema:decoded-file))
+
+(defn- generate-snapshot-label
+  []
+  (let [ts (-> (ct/now)
+               (ct/format-inst)
+               (str/replace #"[T:\.]" "-")
+               (str/rtrim "Z"))]
+    (str "snapshot-" ts)))
+
+(defn create!
+  "Create a file snapshot; expects a non-encoded file."
+  [cfg file & {:keys [label created-by deleted-at profile-id session-id]
+               :or {deleted-at :default
+                    created-by "system"}}]
+
+  (let [file        (check-decoded-file file)
+
+
+        snapshot-id (uuid/next)
+        created-at  (ct/now)
+        deleted-at  (cond
+                      (= deleted-at :default)
+                      (ct/plus (ct/now) (cf/get-deletion-delay))
+
+                      (ct/inst? deleted-at)
+                      deleted-at
+
+                      :else
+                      nil)
+
+        label       (or label (generate-snapshot-label))
+        data        (px/invoke! (::wrk/executor cfg) #(blob/encode (:data file)))
+        features    (:features file)
+        migrations  (:migrations file)
+
+        snapshot    {:id snapshot-id
+                     :revn (:revn file)
+                     :version (:version file)
+                     :file-id (:id file)
+                     :features features
+                     :migrations migrations
+                     :label label
+                     :created-at created-at
+                     :modified-at created-at
+                     :created-by created-by}
+
+        snapshot   (cond-> snapshot
+                     deleted-at
+                     (assoc :deleted-at deleted-at)
+
+                     :always
+                     (check-snapshot))]
+
+    (db/insert! cfg :file-change
+                (-> snapshot
+                    (update :features into-array)
+                    (update :migrations into-array)
+                    (assoc :updated-at created-at)
+                    (assoc :profile-id profile-id)
+                    (assoc :session-id session-id)
+                    (dissoc :modified-at))
+                {::db/return-keys false})
+
+    (fdata/create! cfg
+                   {:id snapshot-id
+                    :file-id (:id file)
+                    :type "snapshot"
+                    :data data
+                    :created-at created-at
+                    :modified-at created-at})
+
+    snapshot))
+
+(defn update!
+  [cfg params]
+
+  (let [{:keys [id file-id label modified-at]}
+        (check-snapshot-params params)
+
+        modified-at
+        (or modified-at (ct/now))]
+
+    (-> (db/update! cfg :file-change
+                    {:label label
+                     :created-by "user"
+                     :updated-at modified-at
+                     :deleted-at nil}
+                    {:file-id file-id
+                     :id id}
+                    {::db/return-keys false})
+        (db/get-update-count)
+        (pos?))))
+
+(defn restore!
+  [{:keys [::db/conn] :as cfg} file-id snapshot-id]
+  (let [file (get-minimal-file conn file-id {::db/for-update true})
+        vern (rand-int Integer/MAX_VALUE)
+
+        storage
+        (sto/resolve cfg {::db/reuse-conn true})
+
+        snapshot
+        (get-snapshot cfg file-id snapshot-id)]
+
+    (when-not snapshot
+      (ex/raise :type :not-found
+                :code :snapshot-not-found
+                :hint "unable to find snapshot with the provided label"
+                :snapshot-id snapshot-id
+                :file-id file-id))
+
+    (when-not (:data snapshot)
+      (ex/raise :type :internal
+                :code :snapshot-without-data
+                :hint "snapshot has no data"
+                :label (:label snapshot)
+                :file-id file-id))
+
+    (let [;; If the snapshot has applied migrations stored, we reuse
+          ;; them, if not, we take a safest set of migrations as
+          ;; starting point. This is because, at the time of
+          ;; implementing snapshots, migrations were not taken into
+          ;; account so we need to make this backward compatible in
+          ;; some way.
+          migrations
+          (or (:migrations snapshot)
+              (fmg/generate-migrations-from-version 67))
+
+          file
+          (-> file
+              (update :revn inc)
+              (assoc :migrations migrations)
+              (assoc :data (:data snapshot))
+              (assoc :vern vern)
+              (assoc :version (:version snapshot))
+              (assoc :has-media-trimmed false)
+              (assoc :modified-at (:modified-at snapshot))
+              (assoc :features (:features snapshot)))]
+
+      (l/dbg :hint "restoring snapshot"
+             :file-id (str file-id)
+             :label (:label snapshot)
+             :snapshot-id (str (:id snapshot)))
+
+      ;; In the same way, on reseting the file data, we need to restore
+      ;; the applied migrations on the moment of taking the snapshot
+      (bfc/update-file! cfg file ::bfc/reset-migrations true)
+
+      ;; FIXME: this should be separated functions, we should not have
+      ;; inline sql here.
+
+      ;; clean object thumbnails
+      (let [sql (str "update file_tagged_object_thumbnail "
+                     "   set deleted_at = now() "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/touch-object! storage media-id)))
+
+      ;; clean file thumbnails
+      (let [sql (str "update file_thumbnail "
+                     "   set deleted_at = now() "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/touch-object! storage media-id)))
+
+      vern)))
+
+(defn delete!
+  [cfg {:keys [id file-id]}]
+  (let [deleted-at (ct/now)]
+    (db/update! cfg :file-change
+                {:deleted-at deleted-at}
+                {:id id :file-id file-id}
+                {::db/return-keys false})
+    true))
+
+
+(defn reduce-snapshots
+  "Process the file snapshots using efficient reduction."
+  [cfg file-id xform f init]
+  (let [conn  (db/get-connection cfg)
+        xform (comp
+               (map (partial fdata/resolve-file-data cfg))
+               (map (partial fdata/decode-file-data cfg))
+               xform)]
+
+    (->> (db/plan conn [sql:get-snapshots file-id] {:fetch-size 1})
+         (transduce xform f init))))
+

backend/src/app/http.clj

@@ -17,7 +17,6 @@
    [app.http.awsns :as-alias awsns]
    [app.http.debug :as-alias debug]
    [app.http.errors :as errors]
-   [app.http.management :as mgmt]
    [app.http.middleware :as mw]
    [app.http.session :as session]
    [app.http.websocket :as-alias ws]
@@ -26,7 +25,9 @@
    [app.rpc :as-alias rpc]
    [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
+   [app.worker :as wrk]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
    [yetti.adapter :as yt]
@@ -53,8 +54,6 @@
   [:map
    [::port ::sm/int]
    [::host ::sm/text]
-   [::io-threads {:optional true} ::sm/int]
-   [::max-worker-threads {:optional true} ::sm/int]
    [::max-body-size {:optional true} ::sm/int]
    [::max-multipart-body-size {:optional true} ::sm/int]
    [::router {:optional true} [:fn r/router?]]
@@ -65,41 +64,31 @@
   (assert (sm/check schema:server-params params)))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [::handler ::router ::host ::port ::mtx/metrics] :as cfg}]
+  [_ {:keys [::handler ::router ::host ::port ::wrk/executor] :as cfg}]
   (l/info :hint "starting http server" :port port :host host)
-  (let [on-dispatch
-        (fn [_ start-at-ns]
-          (let [timing (- (System/nanoTime) start-at-ns)
-                timing (int (/ timing 1000000))]
-            (mtx/run! metrics
-                      :id :http-server-dispatch-timing
-                      :val timing)))
+  (let [options {:http/port port
+                 :http/host host
+                 :http/max-body-size (::max-body-size cfg)
+                 :http/max-multipart-body-size (::max-multipart-body-size cfg)
+                 :xnio/direct-buffers false
+                 :xnio/io-threads (or (::io-threads cfg)
+                                      (max 3 (px/get-available-processors)))
+                 :xnio/dispatch executor
+                 :ring/compat :ring2
+                 :socket/backlog 4069}
 
-        options
-        {:http/port port
-         :http/host host
-         :http/max-body-size (::max-body-size cfg)
-         :http/max-multipart-body-size (::max-multipart-body-size cfg)
-         :xnio/direct-buffers false
-         :xnio/io-threads (::io-threads cfg)
-         :xnio/max-worker-threads (::max-worker-threads cfg)
-         :ring/compat :ring2
-         :events/on-dispatch on-dispatch
-         :socket/backlog 4069}
+        handler (cond
+                  (some? router)
+                  (router-handler router)
 
-        handler
-        (cond
-          (some? router)
-          (router-handler router)
+                  (some? handler)
+                  handler
 
-          (some? handler)
-          handler
+                  :else
+                  (throw (UnsupportedOperationException. "handler or router are required")))
 
-          :else
-          (throw (UnsupportedOperationException. "handler or router are required")))
-
-        server
-        (yt/server handler (d/without-nils options))]
+        options (d/without-nils options)
+        server  (yt/server handler options)]
 
     (assoc cfg ::server (yt/start! server))))
 
@@ -154,7 +143,6 @@
    [::debug/routes schema:routes]
    [::mtx/routes schema:routes]
    [::awsns/routes schema:routes]
-   [::mgmt/routes schema:routes]
    ::session/manager
    ::setup/props
    ::db/pool])
@@ -182,9 +170,6 @@
      ["/webhooks"
       (::awsns/routes cfg)]
 
-     ["/management"
-      (::mgmt/routes cfg)]
-
      (::ws/routes cfg)
 
      ["/api" {:middleware [[mw/cors]]}

backend/src/app/http/awsns.clj

@@ -17,9 +17,11 @@
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.worker :as-alias wrk]
    [clojure.data.json :as j]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [yetti.request :as yreq]
    [yetti.response :as-alias yres]))
 
@@ -38,8 +40,8 @@
   [_ cfg]
   (letfn [(handler [request]
             (let [data (-> request yreq/body slurp)]
-              (handle-request cfg data)
-              {::yres/status 200}))]
+              (px/run! :vthread (partial handle-request cfg data)))
+            {::yres/status 200})]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 

backend/src/app/http/management.clj

@@ -1,234 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.management
-  "Internal mangement HTTP API"
-  (:require
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.schema.generators :as sg]
-   [app.common.time :as ct]
-   [app.db :as db]
-   [app.main :as-alias main]
-   [app.rpc.commands.profile :as cmd.profile]
-   [app.setup :as-alias setup]
-   [app.tokens :as tokens]
-   [app.worker :as-alias wrk]
-   [integrant.core :as ig]
-   [yetti.response :as-alias yres]))
-
-;; ---- ROUTES
-
-(declare ^:private authenticate)
-(declare ^:private get-customer)
-(declare ^:private update-customer)
-
-(defmethod ig/assert-key ::routes
-  [_ params]
-  (assert (db/pool? (::db/pool params)) "expect valid database pool"))
-
-(def ^:private default-system
-  {:name ::default-system
-   :compile
-   (fn [_ _]
-     (fn [handler cfg]
-       (fn [request]
-         (handler cfg request))))})
-
-(def ^:private transaction
-  {:name ::transaction
-   :compile
-   (fn [data _]
-     (when (:transaction data)
-       (fn [handler]
-         (fn [cfg request]
-           (db/tx-run! cfg handler request)))))})
-
-(defmethod ig/init-key ::routes
-  [_ cfg]
-  ["" {:middleware [[default-system cfg]
-                    [transaction]]}
-   ["/authenticate"
-    {:handler authenticate
-     :allowed-methods #{:post}}]
-
-   ["/get-customer"
-    {:handler get-customer
-     :transaction true
-     :allowed-methods #{:post}}]
-
-   ["/update-customer"
-    {:handler update-customer
-     :allowed-methods #{:post}
-     :transaction true}]])
-
-;; ---- HELPERS
-
-(defn- coercer
-  [schema & {:as opts}]
-  (let [decode-fn (sm/decoder schema sm/json-transformer)
-        check-fn  (sm/check-fn schema opts)]
-    (fn [data]
-      (-> data decode-fn check-fn))))
-
-;; ---- API: AUTHENTICATE
-
-(defn- authenticate
-  [cfg request]
-  (let [token  (-> request :params :token)
-        props  (get cfg ::setup/props)
-        result (tokens/verify props {:token token :iss "authentication"})]
-    {::yres/status 200
-     ::yres/body result}))
-
-;; ---- API: GET-CUSTOMER
-
-(def ^:private schema:get-customer
-  [:map [:id ::sm/uuid]])
-
-(def ^:private coerce-get-customer-params
-  (coercer schema:get-customer
-           :type :validation
-           :hint "invalid data provided for `get-customer` rpc call"))
-
-(def ^:private sql:get-customer-slots
-  "WITH teams AS (
-    SELECT tpr.team_id AS id,
-           tpr.profile_id AS profile_id
-      FROM team_profile_rel AS tpr
-     WHERE tpr.is_owner IS true
-       AND tpr.profile_id = ?
-  ), teams_with_slots AS (
-    SELECT tpr.team_id AS id,
-           count(*) AS total
-      FROM team_profile_rel AS tpr
-     WHERE tpr.team_id IN (SELECT id FROM teams)
-       AND tpr.can_edit IS true
-     GROUP BY 1
-     ORDER BY 2
-  )
-  SELECT max(total) AS total FROM teams_with_slots;")
-
-(defn- get-customer-slots
-  [cfg profile-id]
-  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
-    (:total result)))
-
-(defn- get-customer
-  [cfg request]
-  (let [profile-id (-> request :params coerce-get-customer-params :id)
-        profile    (cmd.profile/get-profile cfg profile-id)
-        result     {:id (get profile :id)
-                    :name (get profile :fullname)
-                    :email (get profile :email)
-                    :num-editors (get-customer-slots cfg profile-id)
-                    :subscription (-> profile :props :subscription)}]
-    {::yres/status 200
-     ::yres/body result}))
-
-
-;; ---- API: UPDATE-CUSTOMER
-
-(def ^:private schema:timestamp
-  (sm/type-schema
-   {:type ::timestamp
-    :pred ct/inst?
-    :type-properties
-    {:title "inst"
-     :description "The same as :app.common.time/inst but encodes to epoch"
-     :error/message "should be an instant"
-     :gen/gen (->> (sg/small-int)
-                   (sg/fmap (fn [v] (ct/inst v))))
-     :decode/string ct/inst
-     :encode/string inst-ms
-     :decode/json ct/inst
-     :encode/json inst-ms}}))
-
-(def ^:private schema:subscription
-  [:map {:title "Subscription"}
-   [:id ::sm/text]
-   [:customer-id ::sm/text]
-   [:type [:enum
-           "unlimited"
-           "professional"
-           "enterprise"]]
-   [:status [:enum
-             "active"
-             "canceled"
-             "incomplete"
-             "incomplete_expired"
-             "past_due"
-             "paused"
-             "trialing"
-             "unpaid"]]
-
-   [:billing-period [:enum
-                     "month"
-                     "day"
-                     "week"
-                     "year"]]
-   [:quantity :int]
-   [:description [:maybe ::sm/text]]
-   [:created-at schema:timestamp]
-   [:start-date [:maybe schema:timestamp]]
-   [:ended-at [:maybe schema:timestamp]]
-   [:trial-end [:maybe schema:timestamp]]
-   [:trial-start [:maybe schema:timestamp]]
-   [:cancel-at [:maybe schema:timestamp]]
-   [:canceled-at [:maybe schema:timestamp]]
-   [:current-period-end [:maybe schema:timestamp]]
-   [:current-period-start [:maybe schema:timestamp]]
-   [:cancel-at-period-end :boolean]
-
-   [:cancellation-details
-    [:map {:title "CancellationDetails"}
-     [:comment [:maybe ::sm/text]]
-     [:reason [:maybe ::sm/text]]
-     [:feedback [:maybe
-                 [:enum
-                  "customer_service"
-                  "low_quality"
-                  "missing_feature"
-                  "other"
-                  "switched_service"
-                  "too_complex"
-                  "too_expensive"
-                  "unused"]]]]]])
-
-(def ^:private schema:update-customer
-  [:map
-   [:id ::sm/uuid]
-   [:subscription [:maybe schema:subscription]]])
-
-(def ^:private coerce-update-customer-params
-  (coercer schema:update-customer
-           :type :validation
-           :hint "invalid data provided for `update-customer` rpc call"))
-
-(defn- update-customer
-  [cfg request]
-  (let [{:keys [id subscription]}
-        (-> request :params coerce-update-customer-params)
-
-        {:keys [props] :as profile}
-        (cmd.profile/get-profile cfg id ::db/for-update true)
-
-        props
-        (assoc props :subscription subscription)]
-
-    (l/dbg :hint "update customer"
-           :profile-id (str id)
-           :subscription-type (get subscription :type)
-           :subscription-status (get subscription :status)
-           :subscription-quantity (get subscription :quantity))
-
-    (db/update! cfg :profile
-                {:props (db/tjson props)}
-                {:id id}
-                {::db/return-keys false})
-
-    {::yres/status 201
-     ::yres/body nil}))

backend/src/app/http/sse.clj

@@ -33,7 +33,7 @@
                  (println "event:" (d/name name))
                  (println "data:" (t/encode-str data {:type :json-verbose}))
                  (println))]
-      (.getBytes ^String data "UTF-8"))
+      (.getBytes data "UTF-8"))
     (catch Throwable cause
       (l/err :hint "unexpected error on encoding value on sse stream"
              :cause cause)
@@ -54,7 +54,7 @@
      ::yres/body (yres/stream-body
                   (fn [_ output]
                     (let [channel  (sp/chan :buf buf :xf (keep encode))
-                          listener (events/spawn-listener
+                          listener (events/start-listener
                                     channel
                                     (partial write! output)
                                     (partial pu/close! output))]

backend/src/app/main.clj

@@ -20,7 +20,6 @@
    [app.http.awsns :as http.awsns]
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
-   [app.http.management :as mgmt]
    [app.http.session :as-alias session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
@@ -42,7 +41,6 @@
    [app.svgo :as-alias svgo]
    [app.util.cron]
    [app.worker :as-alias wrk]
-   [app.worker.executor]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -149,11 +147,23 @@
     ::mdef/labels []
     ::mdef/type :histogram}
 
-   :http-server-dispatch-timing
-   {::mdef/name "penpot_http_server_dispatch_timing"
-    ::mdef/help "Histogram of dispatch handler"
-    ::mdef/labels []
-    ::mdef/type :histogram}})
+   :executors-active-threads
+   {::mdef/name "penpot_executors_active_threads"
+    ::mdef/help "Current number of threads available in the executor service."
+    ::mdef/labels ["name"]
+    ::mdef/type :gauge}
+
+   :executors-completed-tasks
+   {::mdef/name "penpot_executors_completed_tasks_total"
+    ::mdef/help "Approximate number of completed tasks by the executor."
+    ::mdef/labels ["name"]
+    ::mdef/type :counter}
+
+   :executors-running-threads
+   {::mdef/name "penpot_executors_running_threads"
+    ::mdef/help "Current number of threads with state RUNNING."
+    ::mdef/labels ["name"]
+    ::mdef/type :gauge}})
 
 (def system-config
   {::db/pool
@@ -165,12 +175,14 @@
     ::db/max-size   (cf/get :database-max-pool-size 60)
     ::mtx/metrics   (ig/ref ::mtx/metrics)}
 
-   ;; Default netty IO pool (shared between several services)
-   ::wrk/netty-io-executor
-   {:threads (cf/get :netty-io-threads)}
+   ;; Default thread pool for IO operations
+   ::wrk/executor
+   {}
 
-   ::wrk/netty-executor
-   {:threads (cf/get :executor-threads)}
+   ::wrk/monitor
+   {::mtx/metrics  (ig/ref ::mtx/metrics)
+    ::wrk/executor (ig/ref ::wrk/executor)
+    ::wrk/name     "default"}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -184,19 +196,14 @@
    ::rds/redis
    {::rds/uri      (cf/get :redis-uri)
     ::mtx/metrics  (ig/ref ::mtx/metrics)
-
-    ::wrk/netty-executor
-    (ig/ref ::wrk/netty-executor)
-
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
+    ::wrk/executor (ig/ref ::wrk/executor)}
 
    ::mbus/msgbus
-   {::wrk/executor  (ig/ref ::wrk/netty-executor)
+   {::wrk/executor  (ig/ref ::wrk/executor)
     ::rds/redis     (ig/ref ::rds/redis)}
 
    :app.storage.tmp/cleaner
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    ::sto.gc-deleted/handler
    {::db/pool      (ig/ref ::db/pool)
@@ -224,10 +231,9 @@
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
     ::http/io-threads              (cf/get :http-server-io-threads)
-    ::http/max-worker-threads      (cf/get :http-server-max-worker-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
     ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)
-    ::mtx/metrics                  (ig/ref ::mtx/metrics)}
+    ::wrk/executor                 (ig/ref ::wrk/executor)}
 
    ::ldap/provider
    {:host           (cf/get :ldap-host)
@@ -267,10 +273,6 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   ::mgmt/routes
-   {::db/pool            (ig/ref ::db/pool)
-    ::setup/props        (ig/ref ::setup/props)}
-
    :app.http/router
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
@@ -279,7 +281,6 @@
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
-    ::mgmt/routes        (ig/ref ::mgmt/routes)
     ::http.debug/routes  (ig/ref ::http.debug/routes)
     ::http.assets/routes (ig/ref ::http.assets/routes)
     ::http.ws/routes     (ig/ref ::http.ws/routes)
@@ -305,17 +306,17 @@
 
    ::rpc/climit
    {::mtx/metrics        (ig/ref ::mtx/metrics)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::wrk/executor       (ig/ref ::wrk/executor)
     ::climit/config      (cf/get :rpc-climit-config)
     ::climit/enabled     (contains? cf/flags :rpc-climit)}
 
    :app.rpc/rlimit
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.rpc/methods
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::wrk/executor       (ig/ref ::wrk/executor)
     ::session/manager    (ig/ref ::session/manager)
     ::ldap/provider      (ig/ref ::ldap/provider)
     ::sto/storage        (ig/ref ::sto/storage)
@@ -469,14 +470,13 @@
                             (cf/get :objects-storage-s3-bucket))
     ::sto.s3/io-threads (or (cf/get :storage-assets-s3-io-threads)
                             (cf/get :objects-storage-s3-io-threads))
-
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
+    ::wrk/executor      (ig/ref ::wrk/executor)}
 
    :app.storage.fs/backend
    {::sto.fs/directory (or (cf/get :storage-assets-fs-directory)
                            (cf/get :objects-storage-fs-directory))}})
 
+
 (def worker-config
   {::wrk/cron
    {::wrk/registry            (ig/ref ::wrk/registry)

backend/src/app/media.clj

@@ -38,13 +38,15 @@
    org.im4java.core.Info))
 
 (def schema:upload
-  [:map {:title "Upload"}
-   [:filename :string]
-   [:size ::sm/int]
-   [:path ::fs/path]
-   [:mtype {:optional true} :string]
-   [:headers {:optional true}
-    [:map-of :string :string]]])
+  (sm/register!
+   ^{::sm/type ::upload}
+   [:map {:title "Upload"}
+    [:filename :string]
+    [:size ::sm/int]
+    [:path ::fs/path]
+    [:mtype {:optional true} :string]
+    [:headers {:optional true}
+     [:map-of :string :string]]]))
 
 (def ^:private schema:input
   [:map {:title "Input"}
@@ -116,7 +118,7 @@
 (defn- parse-svg
   [text]
   (let [text (strip-doctype text)]
-    (dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
+    (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
       (xml/parse istream secure-parser-factory))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/migrations.clj

@@ -446,8 +446,8 @@
    {:name "0140-add-locked-by-column-to-file-change-table"
     :fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}
 
-   {:name "0141-add-idx-to-file-library-rel"
-    :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}])
+   {:name "0141-add-file-data-table.sql"
+    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0141-add-file-data-table.sql

@@ -0,0 +1,33 @@
+CREATE TABLE file_data (
+  file_id uuid NOT NULL REFERENCES file(id) DEFERRABLE,
+  id uuid NOT NULL,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  type text NULL,
+  backend text NULL,
+
+  metadata jsonb NULL,
+  data bytea NULL,
+
+  PRIMARY KEY (file_id, id)
+
+) PARTITION BY HASH (file_id, id);
+
+CREATE TABLE file_data_00 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 0);
+CREATE TABLE file_data_01 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 1);
+CREATE TABLE file_data_02 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 2);
+CREATE TABLE file_data_03 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 3);
+CREATE TABLE file_data_04 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 4);
+CREATE TABLE file_data_05 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 5);
+CREATE TABLE file_data_06 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 6);
+CREATE TABLE file_data_07 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 7);
+CREATE TABLE file_data_08 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 8);
+CREATE TABLE file_data_09 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 9);
+CREATE TABLE file_data_10 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 10);
+CREATE TABLE file_data_11 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 11);
+CREATE TABLE file_data_12 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 12);
+CREATE TABLE file_data_13 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 13);
+CREATE TABLE file_data_14 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 14);
+CREATE TABLE file_data_15 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 15);

backend/src/app/migrations/sql/0141-add-idx-to-file-library-rel.sql

@@ -1,2 +0,0 @@
-CREATE INDEX IF NOT EXISTS file_library_rel__library_file_id__idx
-    ON file_library_rel (library_file_id);

backend/src/app/msgbus.clj

@@ -216,7 +216,8 @@
   (rds/add-listener sconn (create-listener rcv-ch))
 
   (px/thread
-    {:name "penpot/msgbus"}
+    {:name "penpot/msgbus/io-loop"
+     :virtual true}
     (try
       (loop []
         (let [timeout-ch (sp/timeout-chan 1000)

backend/src/app/redis.clj

@@ -21,7 +21,8 @@
    [clojure.java.io :as io]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [promesa.core :as p])
+   [promesa.core :as p]
+   [promesa.exec :as px])
   (:import
    clojure.lang.MapEntry
    io.lettuce.core.KeyValue
@@ -44,10 +45,8 @@
    io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
    io.lettuce.core.resource.ClientResources
    io.lettuce.core.resource.DefaultClientResources
-   io.netty.channel.nio.NioEventLoopGroup
    io.netty.util.HashedWheelTimer
    io.netty.util.Timer
-   io.netty.util.concurrent.EventExecutorGroup
    java.lang.AutoCloseable
    java.time.Duration))
 
@@ -112,15 +111,20 @@
 
 (defmethod ig/expand-key ::redis
   [k v]
-  {k (-> (d/without-nils v)
-         (assoc ::timeout (ct/duration "10s")))})
+  (let [cpus    (px/get-available-processors)
+        threads (max 1 (int (* cpus 0.2)))]
+    {k (-> (d/without-nils v)
+           (assoc ::timeout (ct/duration "10s"))
+           (assoc ::io-threads (max 3 threads))
+           (assoc ::worker-threads (max 3 threads)))}))
 
 (def ^:private schema:redis-params
   [:map {:title "redis-params"}
-   ::wrk/netty-io-executor
-   ::wrk/netty-executor
+   ::wrk/executor
    ::mtx/metrics
    [::uri ::sm/uri]
+   [::worker-threads ::sm/int]
+   [::io-threads ::sm/int]
    [::timeout ::ct/duration]])
 
 (defmethod ig/assert-key ::redis
@@ -137,30 +141,17 @@
 
 (defn- initialize-resources
   "Initialize redis connection resources"
-  [{:keys [::uri ::mtx/metrics ::wrk/netty-io-executor ::wrk/netty-executor] :as params}]
+  [{:keys [::uri ::io-threads ::worker-threads ::wrk/executor ::mtx/metrics] :as params}]
 
   (l/inf :hint "initialize redis resources"
-         :uri (str uri))
+         :uri (str uri)
+         :io-threads io-threads
+         :worker-threads worker-threads)
 
   (let [timer     (HashedWheelTimer.)
         resources (.. (DefaultClientResources/builder)
-                      (eventExecutorGroup ^EventExecutorGroup netty-executor)
-
-                      ;; We provide lettuce with a shared event loop
-                      ;; group instance instead of letting lettuce to
-                      ;; create its own
-                      (eventLoopGroupProvider
-                       (reify io.lettuce.core.resource.EventLoopGroupProvider
-                         (allocate [_ _] netty-io-executor)
-                         (threadPoolSize [_]
-                           (.executorCount ^NioEventLoopGroup netty-io-executor))
-                         (release [_ _ _ _ _]
-                           ;; Do nothing
-                           )
-                         (shutdown [_ _ _ _]
-                           ;; Do nothing
-                           )))
-
+                      (ioThreadPoolSize ^long io-threads)
+                      (computationThreadPoolSize ^long worker-threads)
                       (timer ^Timer timer)
                       (build))
 
@@ -175,7 +166,7 @@
                     (l/trace :hint "evict connection (cache)" :key key :reason cause)
                     (some-> val d/close!))
 
-        cache     (cache/create :executor netty-executor
+        cache     (cache/create :executor executor
                                 :on-remove on-remove
                                 :keepalive "5m")]
     (reify

backend/src/app/rpc.clj

@@ -239,7 +239,6 @@
           'app.rpc.commands.files
           'app.rpc.commands.files-create
           'app.rpc.commands.files-share
-          'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
           'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails

backend/src/app/rpc/climit.clj

@@ -21,6 +21,7 @@
    [clojure.set :as set]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [promesa.exec.bulkhead :as pbh])
   (:import
    clojure.lang.ExceptionInfo
@@ -288,9 +289,13 @@
           (get-limits cfg)))
 
 (defn invoke!
-  "Run a function in context of climit."
-  [{:keys [::rpc/climit] :as cfg} f params]
+  "Run a function in context of climit.
+  Intended to be used in virtual threads."
+  [{:keys [::executor ::rpc/climit] :as cfg} f params]
   (let [f (if climit
-            (build-exec-chain cfg f)
+            (let [f (if (some? executor)
+                      (fn [cfg params] (px/await! (px/submit! executor (fn [] (f cfg params)))))
+                      f)]
+              (build-exec-chain cfg f))
             f)]
     (f cfg params)))

backend/src/app/rpc/commands/auth.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.auth
   (:require
-   [app.auth :as auth]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -63,7 +62,7 @@
               (ex/raise :type :validation
                         :code :account-without-password
                         :hint "the current account does not have password")
-              (let [result (auth/verify-password password (:password profile))]
+              (let [result (profile/verify-password cfg password (:password profile))]
                 (when (:update result)
                   (l/trc :hint "updating profile password"
                          :id (str (:id profile))
@@ -157,7 +156,7 @@
               (:profile-id tdata)))
 
           (update-password [conn profile-id]
-            (let [pwd (auth/derive-password password)]
+            (let [pwd (profile/derive-password cfg password)]
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
@@ -379,7 +378,7 @@
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
-                                             (update :password auth/derive-password))
+                                             (update :password #(profile/derive-password cfg %)))
                                profile   (->> (create-profile! conn params)
                                               (create-profile-rels! conn))]
                            (vary-meta profile assoc :created true))))

backend/src/app/rpc/commands/binfile.clj

@@ -28,6 +28,7 @@
    [app.tasks.file-gc]
    [app.util.services :as sv]
    [app.worker :as-alias wrk]
+   [promesa.exec :as px]
    [yetti.response :as yres]))
 
 (set! *warn-on-reflection* true)
@@ -93,7 +94,7 @@
 ;; --- Command: import-binfile
 
 (defn- import-binfile
-  [{:keys [::db/pool] :as cfg} {:keys [profile-id project-id version name file]}]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
   (let [team   (teams/get-team pool
                                :profile-id profile-id
                                :project-id project-id)
@@ -104,9 +105,13 @@
                    (assoc ::bfc/name name)
                    (assoc ::bfc/input (:path file)))
 
+        ;; NOTE: the importation process performs some operations that are
+        ;; not very friendly with virtual threads, and for avoid
+        ;; unexpected blocking of other concurrent operations we dispatch
+        ;; that operation to a dedicated executor.
         result (case (int version)
-                 1 (bf.v1/import-files! cfg)
-                 3 (bf.v3/import-files! cfg))]
+                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
+                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
 
     (db/update! pool :project
                 {:modified-at (ct/now)}
@@ -122,7 +127,7 @@
    [:project-id ::sm/uuid]
    [:file-id {:optional true} ::sm/uuid]
    [:version {:optional true} ::sm/int]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::import-binfile
   "Import a penpot file in a binary format. If `file-id` is provided,

backend/src/app/rpc/commands/comments.clj

@@ -184,8 +184,8 @@
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
       (let [file (->> file
-                      (files/decode-row)
-                      (feat.fdata/resolve-file-data cfg))
+                      (feat.fdata/resolve-file-data cfg)
+                      (feat.fdata/decode-file-data cfg))
             data (get file :data)]
         (-> file
             (assoc :page-name (dm/get-in data [:pages-index page-id :name]))

backend/src/app/rpc/commands/demo.clj

@@ -7,7 +7,6 @@
 (ns app.rpc.commands.demo
   "A demo specific mutations."
   (:require
-   [app.auth :refer [derive-password]]
    [app.common.exceptions :as ex]
    [app.common.time :as ct]
    [app.config :as cf]
@@ -15,6 +14,7 @@
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.auth :as auth]
+   [app.rpc.commands.profile :as profile]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
    [buddy.core.codecs :as bc]
@@ -46,7 +46,7 @@
                   :fullname fullname
                   :is-active true
                   :deleted-at (ct/in-future (cf/get-deletion-delay))
-                  :password (derive-password password)
+                  :password (profile/derive-password cfg password)
                   :props {}}
         profile  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
                                    (->> (auth/create-profile! conn params)

backend/src/app/rpc/commands/files.clj

@@ -24,7 +24,6 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
-   [app.features.file-migrations :as feat.fmigr]
    [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
@@ -54,12 +53,10 @@
   (ct/duration {:days 7}))
 
 (defn decode-row
-  [{:keys [data changes features] :as row}]
+  [{:keys [features] :as row}]
   (when row
     (cond-> row
-      features (assoc :features (db/decode-pgarray features #{}))
-      changes  (assoc :changes (blob/decode changes))
-      data     (assoc :data (blob/decode data)))))
+      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
 
 (defn check-version!
   [file]
@@ -77,7 +74,6 @@
 
 ;; --- FILE PERMISSIONS
 
-
 (def ^:private sql:file-permissions
   "select fpr.is_owner,
           fpr.is_admin,
@@ -195,7 +191,7 @@
 
 (def schema:permissions-mixin
   [:map {:title "PermissionsMixin"}
-   [:permissions perms/schema:permissions]])
+   [:permissions ::perms/permissions]])
 
 (def schema:file-with-permissions
   [:merge {:title "FileWithPermissions"}
@@ -209,85 +205,9 @@
    [:id ::sm/uuid]
    [:project-id {:optional true} ::sm/uuid]])
 
-(defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
-            pmap/*tracked* (pmap/create-tracked)]
-    (let [libs (delay (bfc/get-resolved-file-libraries cfg file))
-          ;; For avoid unnecesary overhead of creating multiple pointers and
-          ;; handly internally with objects map in their worst case (when
-          ;; probably all shapes and all pointers will be readed in any
-          ;; case), we just realize/resolve them before applying the
-          ;; migration to the file
-          file (-> file
-                   (update :data feat.fdata/process-pointers deref)
-                   (update :data feat.fdata/process-objects (partial into {}))
-                   (fmg/migrate-file libs))]
-
-      (if (or read-only? (db/read-only? conn))
-        file
-        (let [;; When file is migrated, we break the rule of no perform
-              ;; mutations on get operations and update the file with all
-              ;; migrations applied
-              file (if (contains? (:features file) "fdata/objects-map")
-                     (feat.fdata/enable-objects-map file)
-                     file)
-              file (if (contains? (:features file) "fdata/pointer-map")
-                     (feat.fdata/enable-pointer-map file)
-                     file)]
-
-          (db/update! conn :file
-                      {:data (blob/encode (:data file))
-                       :version (:version file)
-                       :features (db/create-array conn "text" (:features file))}
-                      {:id id}
-                      {::db/return-keys false})
-
-          (when (contains? (:features file) "fdata/pointer-map")
-            (feat.fdata/persist-pointers! cfg id))
-
-          (feat.fmigr/upsert-migrations! conn file)
-          (feat.fmigr/resolve-applied-migrations cfg file))))))
-
-(defn get-file
-  [{:keys [::db/conn] :as cfg} id
-   & {:keys [project-id
-             migrate?
-             include-deleted?
-             lock-for-update?
-             preload-pointers?]
-      :or {include-deleted? false
-           lock-for-update? false
-           migrate? true
-           preload-pointers? false}
-      :as options}]
-
-  (assert (db/connection? conn) "expected cfg with valid connection")
-
-  (let [params (merge {:id id}
-                      (when (some? project-id)
-                        {:project-id project-id}))
-        file   (->> (db/get conn :file params
-                            {::db/check-deleted (not include-deleted?)
-                             ::db/remove-deleted (not include-deleted?)
-                             ::sql/for-update lock-for-update?})
-                    (feat.fmigr/resolve-applied-migrations cfg)
-                    (feat.fdata/resolve-file-data cfg)
-                    (decode-row))
-
-        file   (if (and migrate? (fmg/need-migration? file))
-                 (migrate-file cfg file options)
-                 file)]
-
-    (if preload-pointers?
-      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-        (update file :data feat.fdata/process-pointers deref))
-
-      file)))
-
 (defn get-minimal-file
   [cfg id & {:as opts}]
-  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern :data-ref-id :data-backend])]
+  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern])]
     (db/get cfg :file {:id id} opts)))
 
 (defn- get-minimal-file-with-perms
@@ -327,33 +247,22 @@
                                :project-id project-id
                                :file-id id)
 
-          file (-> (get-file cfg id :project-id project-id)
+          file (-> (bfc/get-file cfg id
+                                 :project-id project-id)
                    (assoc :permissions perms)
-                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
           (cfeat/check-client-features! (:features params))
           (cfeat/check-file-features! (:features file)))
 
-      (as-> file file
-        ;; This operation is needed for backward comapatibility with
-        ;; frontends that does not support pointer-map resolution
-        ;; mechanism; this just resolves the pointers on backend and
-        ;; return a complete file
-        (if (and (contains? (:features file) "fdata/pointer-map")
-                 (not (contains? (:features params) "fdata/pointer-map")))
-          (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-            (update file :data feat.fdata/process-pointers deref))
-          file)
-
-        ;; This operation is needed for backward comapatibility with
-        ;; frontends that does not support objects-map mechanism; this
-        ;; just converts all objects map instaces to plain maps
-        (if (and (contains? (:features file) "fdata/objects-map")
-                 (not (contains? (:features params) "fdata/objects-map")))
-          (update file :data feat.fdata/process-objects (partial into {}))
-          file)))))
+      ;; This operation is needed for backward comapatibility with frontends that
+      ;; does not support pointer-map resolution mechanism; this just resolves the
+      ;; pointers on backend and return a complete file.
+      (if (and (contains? (:features file) "fdata/pointer-map")
+               (not (contains? (:features params) "fdata/pointer-map")))
+        (feat.fdata/realize-pointers cfg file)
+        file))))
 
 ;; --- COMMAND QUERY: get-file-fragment (by id)
 
@@ -362,7 +271,7 @@
    [:id ::sm/uuid]
    [:file-id ::sm/uuid]
    [:created-at ::ct/inst]
-   [:content ::sm/any]])
+   [:data any?]])
 
 (def schema:get-file-fragment
   [:map {:title "get-file-fragment"}
@@ -372,10 +281,8 @@
 
 (defn- get-file-fragment
   [cfg file-id fragment-id]
-  (let [resolve-file-data (partial feat.fdata/resolve-file-data cfg)]
-    (some-> (db/get cfg :file-data-fragment {:file-id file-id :id fragment-id})
-            (resolve-file-data)
-            (update :data blob/decode))))
+  (some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"})
+          (update :data blob/decode)))
 
 (sv/defmethod ::get-file-fragment
   "Retrieve a file fragment by its ID. Only authenticated users."
@@ -465,42 +372,8 @@
     (:has-libraries row)))
 
 
-;; --- COMMAND QUERY: get-library-usage
-
-
-(declare get-library-usage)
-
-(def schema:get-library-usage
-  [:map {:title "get-library-usage"}
-   [:file-id ::sm/uuid]])
-:sample
-(sv/defmethod ::get-library-usage
-  "Gets the number of files that use the specified library."
-  {::doc/added "2.10.0"
-   ::sm/params schema:get-library-usage
-   ::sm/result ::sm/int}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! pool profile-id file-id)
-    (get-library-usage conn file-id)))
-
-(def ^:private sql:get-library-usage
-  "SELECT COUNT(*) AS used
-     FROM file_library_rel AS flr
-     JOIN file AS fl ON (flr.library_file_id = fl.id)
-    WHERE flr.library_file_id = ?::uuid
-      AND (fl.deleted_at IS NULL OR
-           fl.deleted_at > now())")
-
-(defn- get-library-usage
-  [conn file-id]
-  (let [row (db/exec-one! conn [sql:get-library-usage file-id])]
-    {:used-in (:used row)}))
-
-
 ;; --- QUERY COMMAND: get-page
 
-
 (defn- prune-objects
   "Given the page data and the object-id returns the page data with all
   other not needed objects removed from the `:objects` data
@@ -534,7 +407,7 @@
 
   (let [perms (get-permissions conn profile-id file-id share-id)
 
-        file  (get-file cfg file-id :read-only? true)
+        file  (bfc/get-file cfg file-id :read-only? true)
 
         proj  (db/get conn :project {:id (:project-id file)})
 
@@ -590,34 +463,8 @@
 
 ;; --- COMMAND QUERY: get-team-shared-files
 
-(defn- components-and-variants
-  "Return a set with all the variant-ids, and a list of components, but with
-   only one component by variant"
-  [components]
-  (let [{:keys [variant-ids components]}
-        (reduce (fn [{:keys [variant-ids components] :as acc} {:keys [variant-id] :as component}]
-                  (cond
-                    (nil? variant-id)
-                    {:variant-ids variant-ids :components (conj components component)}
-                    (contains? variant-ids variant-id)
-                    acc
-                    :else
-                    {:variant-ids (conj variant-ids variant-id) :components (conj components component)}))
-                {:variant-ids #{} :components []}
-                components)]
-    {:components components
-     :variant-ids variant-ids}))
-
-;;coalesce(string_agg(flr.library_file_id::text, ','), '') as library_file_ids
 (def ^:private sql:team-shared-files
-  "with file_library_agg as (
-    select flr.file_id,
-           coalesce(array_agg(flr.library_file_id) filter (where flr.library_file_id is not null), '{}') as library_file_ids
-    from file_library_rel flr
-    group by flr.file_id
-   )
-
-   select f.id,
+  "select f.id,
           f.revn,
           f.vern,
           f.data,
@@ -630,12 +477,10 @@
           f.version,
           f.is_shared,
           ft.media_id,
-          p.team_id,
-          fla.library_file_ids
+          p.team_id
      from file as f
     inner join project as p on (p.id = f.project_id)
      left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn and ft.deleted_at is null)
-     left join file_library_agg as fla on fla.file_id = f.id
     where f.is_shared = true
       and f.deleted_at is null
       and p.deleted_at is null
@@ -651,13 +496,10 @@
                :sample (into [] (take limit sorted-assets))}))]
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-      (let [load-objects       (fn [component]
-                                 (ctf/load-component-objects data component))
-            comps-and-variants (components-and-variants (ctkl/components-seq data))
-            components         (into {} (map (juxt :id identity) (:components comps-and-variants)))
-            components-sample  (-> (assets-sample components 4)
-                                   (update :sample #(mapv load-objects %))
-                                   (assoc :variants-count (-> comps-and-variants :variant-ids count)))]
+      (let [load-objects      (fn [component]
+                                (ctf/load-component-objects data component))
+            components-sample (-> (assets-sample (ctkl/components data) 4)
+                                  (update :sample #(mapv load-objects %)))]
         {:components components-sample
          :media (assets-sample (:media data) 3)
          :colors (assets-sample (:colors data) 3)
@@ -679,8 +521,6 @@
                                (dissoc :media-id)
                                (assoc :thumbnail-id media-id))
                            (dissoc row :media-id))))
-                  (map (fn [row]
-                         (update row :library-file-ids db/decode-pgarray #{})))
                   (map #(assoc % :library-summary (get-library-summary cfg %)))
                   (map #(dissoc % :data))))))
 
@@ -713,7 +553,6 @@
 
 ;; --- COMMAND QUERY: Files that use this File library
 
-
 (def ^:private sql:library-using-files
   "SELECT f.id,
           f.name
@@ -786,7 +625,6 @@
 
 ;; --- COMMAND QUERY: get-file-summary
 
-
 (defn- get-file-summary
   [{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}]
   (check-read-permissions! conn profile-id id)
@@ -795,22 +633,20 @@
                              :project-id project-id
                              :file-id id)
 
-        file (get-file cfg id
-                       :project-id project-id
-                       :read-only? true)]
+        file (bfc/get-file cfg id
+                           :project-id project-id
+                           :read-only? true)]
 
     (-> (cfeat/get-team-enabled-features cf/flags team)
         (cfeat/check-client-features! (:features params))
         (cfeat/check-file-features! (:features file)))
 
     (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-      (let [components-and-variants (components-and-variants (ctkl/components-seq (:data file)))]
-        {:name             (:name file)
-         :components-count (-> components-and-variants :components count)
-         :variants-count   (-> components-and-variants :variant-ids count)
-         :graphics-count   (count (get-in file [:data :media] []))
-         :colors-count     (count (get-in file [:data :colors] []))
-         :typography-count (count (get-in file [:data :typographies] []))}))))
+      {:name             (:name file)
+       :components-count (count (ctkl/components-seq (:data file)))
+       :graphics-count   (count (get-in file [:data :media] []))
+       :colors-count     (count (get-in file [:data :colors] []))
+       :typography-count (count (get-in file [:data :typographies] []))})))
 
 (sv/defmethod ::get-file-summary
   "Retrieve a file summary by its ID. Only authenticated users."
@@ -822,7 +658,6 @@
 
 ;; --- COMMAND QUERY: get-file-info
 
-
 (defn- get-file-info
   [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
   (db/get* conn :file
@@ -887,7 +722,7 @@
 
 ;; --- MUTATION COMMAND: set-file-shared
 
-(def sql:get-referenced-files
+(def ^:private sql:get-referenced-files
   "SELECT f.id
      FROM file_library_rel AS flr
     INNER JOIN file AS f ON (f.id = flr.file_id)
@@ -898,56 +733,51 @@
 (defn- absorb-library-by-file!
   [cfg ldata file-id]
 
-  (dm/assert!
-   "expected cfg with valid connection"
-   (db/connection-map? cfg))
+  (assert (db/connection-map? cfg)
+          "expected cfg with valid connection")
 
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)
             pmap/*tracked* (pmap/create-tracked)]
-    (let [file (-> (get-file cfg file-id
-                             :include-deleted? true
-                             :lock-for-update? true)
+    (let [file (-> (bfc/get-file cfg file-id
+                                 :include-deleted? true
+                                 :lock-for-update? true)
                    (update :data ctf/absorb-assets ldata))]
 
       (l/trc :hint "library absorbed"
              :library-id (str (:id ldata))
              :file-id (str file-id))
 
-      (db/update! cfg :file
-                  {:revn (inc (:revn file))
-                   :data (blob/encode (:data file))
-                   :modified-at (ct/now)
-                   :has-media-trimmed false}
-                  {:id file-id})
-
-      (feat.fdata/persist-pointers! cfg file-id))))
+      (bfc/update-file! cfg {:id file-id
+                             :migrations (:migrations file)
+                             :revn (inc (:revn file))
+                             :data (:data file)
+                             :modified-at (ct/now)
+                             :has-media-trimmed false}))))
 
 (defn- absorb-library
   "Find all files using a shared library, and absorb all library assets
   into the file local libraries"
-  [cfg {:keys [id] :as library}]
+  [cfg {:keys [id data] :as library}]
 
-  (dm/assert!
-   "expected cfg with valid connection"
-   (db/connection-map? cfg))
+  (assert (db/connection-map? cfg)
+          "expected cfg with valid connection")
 
-  (let [ldata (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                (-> library :data (feat.fdata/process-pointers deref)))
-        ids   (->> (db/exec! cfg [sql:get-referenced-files id])
-                   (map :id))]
+  (let [ids (->> (db/exec! cfg [sql:get-referenced-files id])
+                 (sequence bfc/xf-map-id))]
 
     (l/trc :hint "absorbing library"
            :library-id (str id)
            :files (str/join "," (map str ids)))
 
-    (run! (partial absorb-library-by-file! cfg ldata) ids)
+    (run! (partial absorb-library-by-file! cfg data) ids)
     library))
 
 (defn absorb-library!
   [{:keys [::db/conn] :as cfg} id]
-  (let [file (-> (get-file cfg id
-                           :lock-for-update? true
-                           :include-deleted? true)
+  (let [file (-> (bfc/get-file cfg id
+                               :realize? true
+                               :lock-for-update? true
+                               :include-deleted? true)
                  (check-version!))
 
         proj (db/get* conn :project {:id (:project-id file)}
@@ -1077,7 +907,6 @@
    [:library-id ::sm/uuid]])
 
 (sv/defmethod ::link-file-to-library
-  "Link a file to a library. Returns the recursive list of libraries used by that library"
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:link-file-to-library}
@@ -1091,8 +920,7 @@
               (fn [{:keys [::db/conn]}]
                 (check-edition-permissions! conn profile-id file-id)
                 (check-edition-permissions! conn profile-id library-id)
-                (link-file-to-library conn params)
-                (bfc/get-libraries cfg [library-id]))))
+                (link-file-to-library conn params))))
 
 ;; --- MUTATION COMMAND: unlink-file-from-library
 

backend/src/app/rpc/commands/files_create.clj

@@ -8,6 +8,7 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.features :as cfeat]
+   [app.common.files.migrations :as fmg]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.types.file :as ctf]
@@ -51,6 +52,7 @@
                                :revn revn
                                :is-shared is-shared
                                :features features
+                               :migrations fmg/available-migrations
                                :ignore-sync-until ignore-sync-until
                                :created-at modified-at
                                :deleted-at deleted-at}
@@ -66,7 +68,7 @@
                   {:modified-at (ct/now)}
                   {:id project-id})
 
-      file)))
+      (bfc/get-file cfg (:id file)))))
 
 (def ^:private schema:create-file
   [:map {:title "create-file"}
@@ -112,15 +114,14 @@
     ;; FIXME: IMPORTANT: this code can have race conditions, because
     ;; we have no locks for updating team so, creating two files
     ;; concurrently can lead to lost team features updating
+
     (when-let [features (-> features
                             (set/difference (:features team))
                             (set/difference cfeat/no-team-inheritable-features)
                             (not-empty))]
-      (let [features (-> features
-                         (set/union (:features team))
-                         (set/difference cfeat/no-team-inheritable-features)
-                         (into-array))]
-
+      (let [features (->> features
+                          (set/union (:features team))
+                          (db/create-array conn "text"))]
         (db/update! conn :team
                     {:features features}
                     {:id (:id team)}

backend/src/app/rpc/commands/files_snapshot.clj

@@ -8,52 +8,17 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
-   [app.common.files.migrations :as fmg]
-   [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.fdata :as feat.fdata]
-   [app.features.file-migrations :refer [reset-migrations!]]
+   [app.features.file-snapshots :as fsnap]
    [app.main :as-alias main]
    [app.msgbus :as mbus]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
    [app.rpc.quotes :as quotes]
-   [app.storage :as sto]
-   [app.util.blob :as blob]
-   [app.util.services :as sv]
-   [cuerdas.core :as str]))
-
-(defn decode-row
-  [{:keys [migrations] :as row}]
-  (when row
-    (cond-> row
-      (some? migrations)
-      (assoc :migrations (db/decode-pgarray migrations)))))
-
-(def sql:get-file-snapshots
-  "WITH changes AS (
-      SELECT id, label, revn, created_at, created_by, profile_id, locked_by
-        FROM file_change
-       WHERE file_id = ?
-         AND data IS NOT NULL
-         AND (deleted_at IS NULL OR deleted_at > now())
-   ), versions AS (
-      (SELECT * FROM changes WHERE created_by = 'system' LIMIT 1000)
-      UNION ALL
-      (SELECT * FROM changes WHERE created_by != 'system' LIMIT 1000)
-   )
-   SELECT * FROM versions
-    ORDER BY created_at DESC;")
-
-(defn get-file-snapshots
-  [conn file-id]
-  (db/exec! conn [sql:get-file-snapshots file-id]))
+   [app.util.services :as sv]))
 
 (def ^:private schema:get-file-snapshots
   [:map {:title "get-file-snapshots"}
@@ -65,73 +30,7 @@
   [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-read-permissions! conn profile-id file-id)
-                 (get-file-snapshots conn file-id))))
-
-(defn- generate-snapshot-label
-  []
-  (let [ts (-> (ct/now)
-               (ct/format-inst)
-               (str/replace #"[T:\.]" "-")
-               (str/rtrim "Z"))]
-    (str "snapshot-" ts)))
-
-(defn create-file-snapshot!
-  [cfg file & {:keys [label created-by deleted-at profile-id]
-               :or {deleted-at :default
-                    created-by :system}}]
-
-  (assert (#{:system :user :admin} created-by)
-          "expected valid keyword for created-by")
-
-  (let [created-by
-        (name created-by)
-
-        deleted-at
-        (cond
-          (= deleted-at :default)
-          (ct/plus (ct/now) (cf/get-deletion-delay))
-
-          (ct/inst? deleted-at)
-          deleted-at
-
-          :else
-          nil)
-
-        label
-        (or label (generate-snapshot-label))
-
-        snapshot-id
-        (uuid/next)
-
-        data
-        (blob/encode (:data file))
-
-        features
-        (into-array (:features file))
-
-        migrations
-        (into-array (:migrations file))]
-
-    (l/dbg :hint "creating file snapshot"
-           :file-id (str (:id file))
-           :id (str snapshot-id)
-           :label label)
-
-    (db/insert! cfg :file-change
-                {:id snapshot-id
-                 :revn (:revn file)
-                 :data data
-                 :version (:version file)
-                 :features features
-                 :migrations migrations
-                 :profile-id profile-id
-                 :file-id (:id file)
-                 :label label
-                 :deleted-at deleted-at
-                 :created-by created-by}
-                {::db/return-keys false})
-
-    {:id snapshot-id :label label}))
+                 (fsnap/get-visible-snapshots conn file-id))))
 
 (def ^:private schema:create-file-snapshot
   [:map
@@ -144,7 +43,7 @@
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id label]}]
   (files/check-edition-permissions! conn profile-id file-id)
-  (let [file    (bfc/get-file cfg file-id)
+  (let [file    (bfc/get-file cfg file-id :realize? true)
         project (db/get-by-id cfg :project (:project-id file))]
 
     (-> cfg
@@ -155,96 +54,10 @@
         (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
                        {::quotes/id ::quotes/snapshots-per-team}))
 
-    (create-file-snapshot! cfg file
-                           {:label label
-                            :profile-id profile-id
-                            :created-by :user})))
-
-(defn restore-file-snapshot!
-  [{:keys [::db/conn ::mbus/msgbus] :as cfg} file-id snapshot-id]
-  (let [storage  (sto/resolve cfg {::db/reuse-conn true})
-        file     (files/get-minimal-file conn file-id {::db/for-update true})
-        vern     (rand-int Integer/MAX_VALUE)
-        snapshot (some->> (db/get* conn :file-change
-                                   {:file-id file-id
-                                    :id snapshot-id}
-                                   {::db/for-share true})
-                          (feat.fdata/resolve-file-data cfg)
-                          (decode-row))
-
-        ;; If snapshot has tracked applied migrations, we reuse them,
-        ;; if not we take a safest set of migrations as starting
-        ;; point. This is because, at the time of implementing
-        ;; snapshots, migrations were not taken into account so we
-        ;; need to make this backward compatible in some way.
-        file     (assoc file :migrations
-                        (or (:migrations snapshot)
-                            (fmg/generate-migrations-from-version 67)))]
-
-    (when-not snapshot
-      (ex/raise :type :not-found
-                :code :snapshot-not-found
-                :hint "unable to find snapshot with the provided label"
-                :snapshot-id snapshot-id
-                :file-id file-id))
-
-    (when-not (:data snapshot)
-      (ex/raise :type :validation
-                :code :snapshot-without-data
-                :hint "snapshot has no data"
-                :label (:label snapshot)
-                :file-id file-id))
-
-    (l/dbg :hint "restoring snapshot"
-           :file-id (str file-id)
-           :label (:label snapshot)
-           :snapshot-id (str (:id snapshot)))
-
-    ;; If the file was already offloaded, on restoring the snapshot we
-    ;; are going to replace the file data, so we need to touch the old
-    ;; referenced storage object and avoid possible leaks
-    (when (feat.fdata/offloaded? file)
-      (sto/touch-object! storage (:data-ref-id file)))
-
-    ;; In the same way, on reseting the file data, we need to restore
-    ;; the applied migrations on the moment of taking the snapshot
-    (reset-migrations! conn file)
-
-    (db/update! conn :file
-                {:data (:data snapshot)
-                 :revn (inc (:revn file))
-                 :vern vern
-                 :version (:version snapshot)
-                 :data-backend nil
-                 :data-ref-id nil
-                 :has-media-trimmed false
-                 :features (:features snapshot)}
-                {:id file-id})
-
-    ;; clean object thumbnails
-    (let [sql (str "update file_tagged_object_thumbnail "
-                   "   set deleted_at = now() "
-                   " where file_id=? returning media_id")
-          res (db/exec! conn [sql file-id])]
-      (doseq [media-id (into #{} (keep :media-id) res)]
-        (sto/touch-object! storage media-id)))
-
-    ;; clean file thumbnails
-    (let [sql (str "update file_thumbnail "
-                   "   set deleted_at = now() "
-                   " where file_id=? returning media_id")
-          res (db/exec! conn [sql file-id])]
-      (doseq [media-id (into #{} (keep :media-id) res)]
-        (sto/touch-object! storage media-id)))
-
-    ;; Send to the clients a notification to reload the file
-    (mbus/pub! msgbus
-               :topic (:id file)
-               :message {:type :file-restore
-                         :file-id (:id file)
-                         :vern vern})
-    {:id (:id snapshot)
-     :label (:label snapshot)}))
+    (fsnap/create! cfg file
+                   {:label label
+                    :profile-id profile-id
+                    :created-by "user"})))
 
 (def ^:private schema:restore-file-snapshot
   [:map {:title "restore-file-snapshot"}
@@ -253,88 +66,56 @@
 
 (sv/defmethod ::restore-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:restore-file-snapshot}
-  [cfg {:keys [::rpc/profile-id file-id id] :as params}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn] :as cfg}]
-                (files/check-edition-permissions! conn profile-id file-id)
-                (let [file (bfc/get-file cfg file-id)]
-                  (create-file-snapshot! cfg file
-                                         {:profile-id profile-id
-                                          :created-by :system})
-                  (restore-file-snapshot! cfg file-id id)))))
+   ::sm/params schema:restore-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id file-id id] :as params}]
+  (files/check-edition-permissions! conn profile-id file-id)
+  (let [file (bfc/get-file cfg file-id)]
+    (fsnap/create! cfg file
+                   {:profile-id profile-id
+                    :created-by "system"})
+    (let [vern (fsnap/restore! cfg file-id id)]
+      ;; Send to the clients a notification to reload the file
+      (mbus/pub! msgbus
+                 :topic (:id file)
+                 :message {:type :file-restore
+                           :file-id (:id file)
+                           :vern vern})
+      nil)))
 
 (def ^:private schema:update-file-snapshot
   [:map {:title "update-file-snapshot"}
    [:id ::sm/uuid]
    [:label ::sm/text]])
 
-(defn- update-file-snapshot!
-  [conn snapshot-id label]
-  (-> (db/update! conn :file-change
-                  {:label label
-                   :created-by "user"
-                   :deleted-at nil}
-                  {:id snapshot-id}
-                  {::db/return-keys true})
-      (dissoc :data :features :migrations)))
-
-(defn- get-snapshot
-  "Get a minimal snapshot from database and lock for update"
-  [conn id]
-  (db/get conn :file-change
-          {:id id}
-          {::sql/columns [:id :file-id :created-by :deleted-at :profile-id :locked-by]
-           ::db/for-update true}))
-
 (sv/defmethod ::update-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:update-file-snapshot}
-  [cfg {:keys [::rpc/profile-id id label]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn]}]
-                (let [snapshot (get-snapshot conn id)]
-                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
-                  (update-file-snapshot! conn id label)))))
+   ::sm/params schema:update-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id label]}]
+  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+    (fsnap/update! conn (assoc snapshot :label label))))
 
 (def ^:private schema:remove-file-snapshot
   [:map {:title "remove-file-snapshot"}
    [:id ::sm/uuid]])
 
-(defn- delete-file-snapshot!
-  [conn snapshot-id]
-  (db/update! conn :file-change
-              {:deleted-at (ct/now)}
-              {:id snapshot-id}
-              {::db/return-keys false})
-  nil)
-
 (sv/defmethod ::delete-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:remove-file-snapshot}
-  [cfg {:keys [::rpc/profile-id id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn]}]
-                (let [snapshot (get-snapshot conn id)]
-                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+   ::sm/params schema:remove-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-                  (when (not= (:created-by snapshot) "user")
-                    (ex/raise :type :validation
-                              :code :system-snapshots-cant-be-deleted
-                              :snapshot-id id
-                              :profile-id profile-id))
-
-                  ;; Check if version is locked by someone else
-                  (when (and (:locked-by snapshot)
-                             (not= (:locked-by snapshot) profile-id))
-                    (ex/raise :type :validation
-                              :code :snapshot-is-locked
-                              :hint "Cannot delete a locked version"
-                              :snapshot-id id
-                              :profile-id profile-id
-                              :locked-by (:locked-by snapshot)))
-
-                  (delete-file-snapshot! conn id)))))
+    (when (not= (:created-by snapshot) "user")
+      (ex/raise :type :validation
+                :code :system-snapshots-cant-be-deleted
+                :file-id (:file-id snapshot)
+                :snapshot-id id
+                :profile-id profile-id))
+    (fsnap/delete! conn snapshot)))
 
 ;;; Lock/unlock version endpoints
 
@@ -342,6 +123,7 @@
   [:map {:title "lock-file-snapshot"}
    [:id ::sm/uuid]])
 
+;; MOVE to fsnap
 (defn- lock-file-snapshot!
   [conn snapshot-id profile-id]
   (db/update! conn :file-change
@@ -352,44 +134,45 @@
 
 (sv/defmethod ::lock-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:lock-file-snapshot}
-  [cfg {:keys [::rpc/profile-id id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn]}]
-                (let [snapshot (get-snapshot conn id)]
-                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+   ::sm/params schema:lock-file-snapshot
+   ::db/transaction true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-                  (when (not= (:created-by snapshot) "user")
-                    (ex/raise :type :validation
-                              :code :system-snapshots-cant-be-locked
-                              :hint "Only user-created versions can be locked"
-                              :snapshot-id id
-                              :profile-id profile-id))
+    (when (not= (:created-by snapshot) "user")
+      (ex/raise :type :validation
+                :code :system-snapshots-cant-be-locked
+                :hint "Only user-created versions can be locked"
+                :snapshot-id id
+                :profile-id profile-id))
 
-                  ;; Only the creator can lock their own version
-                  (when (not= (:profile-id snapshot) profile-id)
-                    (ex/raise :type :validation
-                              :code :only-creator-can-lock
-                              :hint "Only the version creator can lock it"
-                              :snapshot-id id
-                              :profile-id profile-id
-                              :creator-id (:profile-id snapshot)))
+    ;; Only the creator can lock their own version
+    (when (not= (:profile-id snapshot) profile-id)
+      (ex/raise :type :validation
+                :code :only-creator-can-lock
+                :hint "Only the version creator can lock it"
+                :snapshot-id id
+                :profile-id profile-id
+                :creator-id (:profile-id snapshot)))
 
-                  ;; Check if already locked
-                  (when (:locked-by snapshot)
-                    (ex/raise :type :validation
-                              :code :snapshot-already-locked
-                              :hint "Version is already locked"
-                              :snapshot-id id
-                              :profile-id profile-id
-                              :locked-by (:locked-by snapshot)))
+    ;; Check if already locked
+    (when (:locked-by snapshot)
+      (ex/raise :type :validation
+                :code :snapshot-already-locked
+                :hint "Version is already locked"
+                :snapshot-id id
+                :profile-id profile-id
+                :locked-by (:locked-by snapshot)))
 
-                  (lock-file-snapshot! conn id profile-id)))))
+    (lock-file-snapshot! conn id profile-id)))
 
 (def ^:private schema:unlock-file-snapshot
   [:map {:title "unlock-file-snapshot"}
    [:id ::sm/uuid]])
 
+
+;; MOVE to fsnap
 (defn- unlock-file-snapshot!
   [conn snapshot-id]
   (db/update! conn :file-change
@@ -400,35 +183,34 @@
 
 (sv/defmethod ::unlock-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:unlock-file-snapshot}
-  [cfg {:keys [::rpc/profile-id id]}]
-  (db/tx-run! cfg
-              (fn [{:keys [::db/conn]}]
-                (let [snapshot (get-snapshot conn id)]
-                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+   ::sm/params schema:unlock-file-snapshot
+   ::db/transaction  true}
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-                  (when (not= (:created-by snapshot) "user")
-                    (ex/raise :type :validation
-                              :code :system-snapshots-cant-be-unlocked
-                              :hint "Only user-created versions can be unlocked"
-                              :snapshot-id id
-                              :profile-id profile-id))
+    (when (not= (:created-by snapshot) "user")
+      (ex/raise :type :validation
+                :code :system-snapshots-cant-be-unlocked
+                :hint "Only user-created versions can be unlocked"
+                :snapshot-id id
+                :profile-id profile-id))
 
-                  ;; Only the creator can unlock their own version
-                  (when (not= (:profile-id snapshot) profile-id)
-                    (ex/raise :type :validation
-                              :code :only-creator-can-unlock
-                              :hint "Only the version creator can unlock it"
-                              :snapshot-id id
-                              :profile-id profile-id
-                              :creator-id (:profile-id snapshot)))
+    ;; Only the creator can unlock their own version
+    (when (not= (:profile-id snapshot) profile-id)
+      (ex/raise :type :validation
+                :code :only-creator-can-unlock
+                :hint "Only the version creator can unlock it"
+                :snapshot-id id
+                :profile-id profile-id
+                :creator-id (:profile-id snapshot)))
 
-                  ;; Check if not locked
-                  (when (not (:locked-by snapshot))
-                    (ex/raise :type :validation
-                              :code :snapshot-not-locked
-                              :hint "Version is not locked"
-                              :snapshot-id id
-                              :profile-id profile-id))
+    ;; Check if not locked
+    (when (not (:locked-by snapshot))
+      (ex/raise :type :validation
+                :code :snapshot-not-locked
+                :hint "Version is not locked"
+                :snapshot-id id
+                :profile-id profile-id))
 
-                  (unlock-file-snapshot! conn id)))))
+    (unlock-file-snapshot! conn id)))

backend/src/app/rpc/commands/files_temp.clj

@@ -1,160 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.rpc.commands.files-temp
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.features :as cfeat]
-   [app.common.files.changes :as cpc]
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.uuid :as uuid]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.db.sql :as sql]
-   [app.features.fdata :as fdata]
-   [app.loggers.audit :as audit]
-   [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-create :as files.create]
-   [app.rpc.commands.files-update :as-alias files.update]
-   [app.rpc.commands.projects :as projects]
-   [app.rpc.commands.teams :as teams]
-   [app.rpc.doc :as-alias doc]
-   [app.rpc.helpers :as rph]
-   [app.util.blob :as blob]
-   [app.util.pointer-map :as pmap]
-   [app.util.services :as sv]
-   [clojure.set :as set]))
-
-;; --- MUTATION COMMAND: create-temp-file
-
-(def ^:private schema:create-temp-file
-  [:map {:title "create-temp-file"}
-   [:name [:string {:max 250}]]
-   [:project-id ::sm/uuid]
-   [:id {:optional true} ::sm/uuid]
-   [:is-shared ::sm/boolean]
-   [:features ::cfeat/features]
-   [:create-page ::sm/boolean]])
-
-(sv/defmethod ::create-temp-file
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::sm/params schema:create-temp-file
-   ::db/transaction true}
-  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
-  (projects/check-edition-permissions! conn profile-id project-id)
-  (let [team (teams/get-team conn :profile-id profile-id :project-id project-id)
-        ;; When we create files, we only need to respect the team
-        ;; features, because some features can be enabled
-        ;; globally, but the team is still not migrated properly.
-        input-features
-        (:features params #{})
-
-        ;; If the imported project doesn't contain v2 we need to remove it
-        team-features
-        (cond-> (cfeat/get-team-enabled-features cf/flags team)
-          (not (contains? input-features "components/v2"))
-          (disj "components/v2"))
-
-        ;; We also include all no migration features declared by
-        ;; client; that enables the ability to enable a runtime
-        ;; feature on frontend and make it permanent on file
-        features
-        (-> input-features
-            (set/intersection cfeat/no-migration-features)
-            (set/union team-features))
-
-        params
-        (-> params
-            (assoc :profile-id profile-id)
-            (assoc :deleted-at (ct/in-future {:days 1}))
-            (assoc :features features))]
-
-    (files.create/create-file cfg params)))
-
-;; --- MUTATION COMMAND: update-temp-file
-
-(def ^:private schema:update-temp-file
-  [:map {:title "update-temp-file"}
-   [:changes [:vector cpc/schema:change]]
-   [:revn [::sm/int {:min 0}]]
-   [:session-id ::sm/uuid]
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::update-temp-file
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::sm/params schema:update-temp-file}
-  [cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                    (db/insert! conn :file-change
-                                {:id (uuid/next)
-                                 :session-id session-id
-                                 :profile-id profile-id
-                                 :created-at (ct/now)
-                                 :file-id id
-                                 :revn revn
-                                 :data nil
-                                 :changes (blob/encode changes)})
-                    (rph/with-meta (rph/wrap nil)
-                      {::audit/replace-props {:file-id id
-                                              :revn revn}}))))
-
-;; --- MUTATION COMMAND: persist-temp-file
-
-(defn persist-temp-file
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
-  (let [file (files/get-file cfg id
-                             :migrate? false
-                             :lock-for-update? true)]
-
-    (when (nil? (:deleted-at file))
-      (ex/raise :type :validation
-                :code :cant-persist-already-persisted-file))
-
-    (let [changes (->> (db/cursor conn
-                                  (sql/select :file-change {:file-id id}
-                                              {:order-by [[:revn :asc]]})
-                                  {:chunk-size 10})
-                       (sequence (mapcat (comp blob/decode :changes))))
-
-          file    (update file :data cpc/process-changes changes)
-
-          file    (if (contains? (:features file) "fdata/objects-map")
-                    (fdata/enable-objects-map file)
-                    file)
-
-          file    (if (contains? (:features file) "fdata/pointer-map")
-                    (binding [pmap/*tracked* (pmap/create-tracked)]
-                      (let [file (fdata/enable-pointer-map file)]
-                        (fdata/persist-pointers! cfg id)
-                        file))
-                    file)]
-
-      ;; Delete changes from the changes history
-      (db/delete! conn :file-change {:file-id id})
-
-      (db/update! conn :file
-                  {:deleted-at nil
-                   :revn 1
-                   :data (blob/encode (:data file))}
-                  {:id id})
-      nil)))
-
-(def ^:private schema:persist-temp-file
-  [:map {:title "persist-temp-file"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::persist-temp-file
-  {::doc/added "1.17"
-   ::doc/module :files
-   ::sm/params schema:persist-temp-file}
-  [cfg {:keys [::rpc/profile-id id] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (files/check-edition-permissions! conn profile-id id)
-                    (persist-temp-file cfg params))))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.files-thumbnails
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
@@ -202,9 +203,9 @@
                                             :profile-id profile-id
                                             :file-id file-id)
 
-                       file (files/get-file cfg file-id
-                                            :preload-pointers? true
-                                            :read-only? true)]
+                       file (bfc/get-file cfg file-id
+                                          :realize? true
+                                          :read-only? true)]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
                        (cfeat/check-file-features! (:features file)))
@@ -271,7 +272,7 @@
   [:map {:title "create-file-object-thumbnail"}
    [:file-id ::sm/uuid]
    [:object-id [:string {:max 250}]]
-   [:media media/schema:upload]
+   [:media ::media/upload]
    [:tag {:optional true} [:string {:max 50}]]])
 
 (sv/defmethod ::create-file-object-thumbnail
@@ -339,6 +340,7 @@
         data  (-> (sto/content path)
                   (sto/wrap-with-hash hash))
         tnow  (ct/now)
+
         media (sto/put-object! storage
                                {::sto/content data
                                 ::sto/deduplicate? true
@@ -381,7 +383,7 @@
   [:map {:title "create-file-thumbnail"}
    [:file-id ::sm/uuid]
    [:revn ::sm/int]
-   [:media media/schema:upload]])
+   [:media ::media/upload]])
 
 (sv/defmethod ::create-file-thumbnail
   "Creates or updates the file thumbnail. Mainly used for paint the

backend/src/app/rpc/commands/files_update.clj

@@ -19,8 +19,9 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as feat.fdata]
+   [app.features.fdata :as fdata]
    [app.features.file-migrations :as feat.fmigr]
+   [app.features.file-snapshots :as fsnap]
    [app.features.logical-deletion :as ldel]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
@@ -33,11 +34,12 @@
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
-   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [clojure.set :as set]))
+   [app.worker :as wrk]
+   [clojure.set :as set]
+   [promesa.exec :as px]))
 
 (declare ^:private get-lagged-changes)
 (declare ^:private send-notifications!)
@@ -62,10 +64,10 @@
    [:revn {:min 0} ::sm/int]
    [:vern {:min 0} ::sm/int]
    [:features {:optional true} ::cfeat/features]
-   [:changes {:optional true} [:vector cpc/schema:change]]
+   [:changes {:optional true} [:vector ::cpc/change]]
    [:changes-with-metadata {:optional true}
     [:vector [:map
-              [:changes [:vector cpc/schema:change]]
+              [:changes [:vector ::cpc/change]]
               [:hint-origin {:optional true} :keyword]
               [:hint-events {:optional true} [:vector [:string {:max 250}]]]]]]
    [:skip-validate {:optional true} ::sm/boolean]])
@@ -74,7 +76,7 @@
   schema:update-file-result
   [:vector {:title "update-file-result"}
    [:map
-    [:changes [:vector cpc/schema:change]]
+    [:changes [:vector ::cpc/change]]
     [:file-id ::sm/uuid]
     [:id ::sm/uuid]
     [:revn {:min 0} ::sm/int]
@@ -127,76 +129,78 @@
    ::sm/params schema:update-file
    ::sm/result schema:update-file-result
    ::doc/module :files
-   ::doc/added "1.17"}
-  [{:keys [::mtx/metrics] :as cfg}
+   ::doc/added "1.17"
+   ::db/transaction true}
+  [{:keys [::mtx/metrics ::db/conn] :as cfg}
    {:keys [::rpc/profile-id id changes changes-with-metadata] :as params}]
-  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                    (files/check-edition-permissions! conn profile-id id)
-                    (db/xact-lock! conn id)
 
-                    (let [file     (get-file conn id)
-                          team     (teams/get-team conn
-                                                   :profile-id profile-id
-                                                   :team-id (:team-id file))
+  (files/check-edition-permissions! conn profile-id id)
+  (db/xact-lock! conn id)
 
-                          features (-> (cfeat/get-team-enabled-features cf/flags team)
-                                       (cfeat/check-client-features! (:features params))
-                                       (cfeat/check-file-features! (:features file)))
+  (let [file     (get-file cfg id)
+        team     (teams/get-team conn
+                                 :profile-id profile-id
+                                 :team-id (:team-id file))
 
-                          changes  (if changes-with-metadata
-                                     (->> changes-with-metadata (mapcat :changes) vec)
-                                     (vec changes))
+        features (-> (cfeat/get-team-enabled-features cf/flags team)
+                     (cfeat/check-client-features! (:features params))
+                     (cfeat/check-file-features! (:features file)))
 
-                          params   (-> params
-                                       (assoc :profile-id profile-id)
-                                       (assoc :features (set/difference features cfeat/frontend-only-features))
-                                       (assoc :team team)
-                                       (assoc :file file)
-                                       (assoc :changes changes))
+        changes  (if changes-with-metadata
+                   (->> changes-with-metadata (mapcat :changes) vec)
+                   (vec changes))
 
-                          cfg      (assoc cfg ::timestamp (ct/now))
+        params   (-> params
+                     (assoc :profile-id profile-id)
+                     (assoc :features (set/difference features cfeat/frontend-only-features))
+                     (assoc :team team)
+                     (assoc :file file)
+                     (assoc :changes changes))
 
-                          tpoint   (ct/tpoint)]
+        cfg      (assoc cfg ::timestamp (ct/now))
 
-                      (when (not= (:vern params)
-                                  (:vern file))
-                        (ex/raise :type :validation
-                                  :code :vern-conflict
-                                  :hint "A different version has been restored for the file."
-                                  :context {:incoming-revn (:revn params)
-                                            :stored-revn (:revn file)}))
+        tpoint   (ct/tpoint)]
 
-                      (when (> (:revn params)
-                               (:revn file))
-                        (ex/raise :type :validation
-                                  :code :revn-conflict
-                                  :hint "The incoming revision number is greater that stored version."
-                                  :context {:incoming-revn (:revn params)
-                                            :stored-revn (:revn file)}))
 
-                      ;; When newly computed features does not match exactly with
-                      ;; the features defined on team row, we update it
-                      (when-let [features (-> features
-                                              (set/difference (:features team))
-                                              (set/difference cfeat/no-team-inheritable-features)
-                                              (not-empty))]
-                        (let [features (-> features
-                                           (set/union (:features team))
-                                           (set/difference cfeat/no-team-inheritable-features)
-                                           (into-array))]
-                          (db/update! conn :team
-                                      {:features features}
-                                      {:id (:id team)}
-                                      {::db/return-keys false})))
+    (when (not= (:vern params)
+                (:vern file))
+      (ex/raise :type :validation
+                :code :vern-conflict
+                :hint "A different version has been restored for the file."
+                :context {:incoming-revn (:revn params)
+                          :stored-revn (:revn file)}))
 
-                      (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+    (when (> (:revn params)
+             (:revn file))
+      (ex/raise :type :validation
+                :code :revn-conflict
+                :hint "The incoming revision number is greater that stored version."
+                :context {:incoming-revn (:revn params)
+                          :stored-revn (:revn file)}))
 
-                      (binding [l/*context* (some-> (meta params)
-                                                    (get :app.http/request)
-                                                    (errors/request->context))]
-                        (-> (update-file* cfg params)
-                            (rph/with-defer #(let [elapsed (tpoint)]
-                                               (l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))))
+    ;; When newly computed features does not match exactly with
+    ;; the features defined on team row, we update it
+    (when-let [features (-> features
+                            (set/difference (:features team))
+                            (set/difference cfeat/no-team-inheritable-features)
+                            (not-empty))]
+      (let [features (->> features
+                          (set/union (:features team))
+                          (db/create-array conn "text"))]
+        (db/update! conn :team
+                    {:features features}
+                    {:id (:id team)}
+                    {::db/return-keys false})))
+
+
+    (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
+    (binding [l/*context* (some-> (meta params)
+                                  (get :app.http/request)
+                                  (errors/request->context))]
+      (-> (update-file* cfg params)
+          (rph/with-defer #(let [elapsed (tpoint)]
+                             (l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))
 
 (defn- update-file*
   "Internal function, part of the update-file process, that encapsulates
@@ -206,27 +210,47 @@
   Follow the inner implementation to `update-file-data!` function.
 
   Only intended for internal use on this module."
-  [{:keys [::db/conn ::timestamp] :as cfg}
+  [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
    {:keys [profile-id file team features changes session-id skip-validate] :as params}]
 
-  (let [;; Retrieve the file data
-        file  (feat.fmigr/resolve-applied-migrations cfg file)
-        file  (feat.fdata/resolve-file-data cfg file)
-        file  (assoc file :features
-                     (-> features
-                         (set/difference cfeat/frontend-only-features)
-                         (set/union (:features file))))]
+  (binding [pmap/*tracked* (pmap/create-tracked)
+            pmap/*load-fn* (partial fdata/load-pointer cfg (:id file))]
 
-    ;; We create a new lexycal scope for clearly delimit the result of
-    ;; executing this update file operation and all its side effects
-    (let [file (binding [cfeat/*current*  features
-                         cfeat/*previous* (:features file)]
-                 (update-file-data! cfg file
-                                    process-changes-and-validate
-                                    changes skip-validate))]
+    (let [file (assoc file :features
+                      (-> features
+                          (set/difference cfeat/frontend-only-features)
+                          (set/union (:features file))))
 
-      (feat.fmigr/upsert-migrations! conn file)
-      (persist-file! cfg file)
+          ;; We need to preserve the original revn for the response
+          revn
+          (get file :revn)
+
+          ;; We create a new lexical scope for clearly delimit the result of
+          ;; executing this update file operation and all its side effects
+          file
+          (px/invoke! executor
+                      (fn []
+                        ;; Process the file data on separated thread
+                        ;; for avoid to do the CPU intensive operation
+                        ;; on vthread.
+                        (binding [cfeat/*current*  features
+                                  cfeat/*previous* (:features file)]
+                          (update-file-data! cfg file
+                                             process-changes-and-validate
+                                             changes skip-validate))))
+
+          deleted-at
+          (ct/plus timestamp (ct/duration {:hours 1}))]
+
+      (when-let [file (::snapshot file)]
+        (let [deleted-at (ct/plus timestamp (ldel/get-deletion-delay team))
+              label      (str "internal/snapshot/" revn)]
+
+          (fsnap/create! cfg file
+                         {:label label
+                          :deleted-at deleted-at
+                          :profile-id profile-id
+                          :session-id session-id})))
 
       ;; Insert change (xlog) with deleted_at in a future data for
       ;; make them automatically eleggible for GC once they expires
@@ -236,34 +260,28 @@
                    :profile-id profile-id
                    :created-at timestamp
                    :updated-at timestamp
-                   :deleted-at (if (::snapshot-data file)
-                                 (ct/plus timestamp (ldel/get-deletion-delay team))
-                                 (ct/plus timestamp (ct/duration {:hours 1})))
+                   :deleted-at deleted-at
                    :file-id (:id file)
                    :revn (:revn file)
                    :version (:version file)
-                   :features (:features file)
-                   :label (::snapshot-label file)
-                   :data (::snapshot-data file)
+                   :features (into-array (:features file))
                    :changes (blob/encode changes)}
                   {::db/return-keys false})
 
+      (persist-file! cfg file)
+
       ;; Send asynchronous notifications
-      (send-notifications! cfg params file))
+      (send-notifications! cfg params file)
 
-    (when (feat.fdata/offloaded? file)
-      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-        (some->> (:data-ref-id file) (sto/touch-object! storage))))
-
-    (let [response {:revn (:revn file)
-                    :lagged (get-lagged-changes conn params)}]
-      (vary-meta response assoc ::audit/replace-props
-                 {:id         (:id file)
-                  :name       (:name file)
-                  :features   (:features file)
-                  :project-id (:project-id file)
-                  :team-id    (:team-id file)}))))
+      (with-meta {:revn revn :lagged (get-lagged-changes conn params)}
+        {::audit/replace-props
+         {:id         (:id file)
+          :name       (:name file)
+          :features   (:features file)
+          :project-id (:project-id file)
+          :team-id    (:team-id file)}}))))
 
+;: FIXME: DEPRECATED
 (defn update-file!
   "A public api that allows apply a transformation to a file with all context setup."
   [{:keys [::db/conn] :as cfg} file-id update-fn & args]
@@ -272,51 +290,42 @@
     (feat.fmigr/upsert-migrations! conn file)
     (persist-file! cfg file)))
 
-(def ^:private sql:get-file
-  "SELECT f.*, p.team_id
-     FROM file AS f
-     JOIN project AS p ON (p.id = f.project_id)
-    WHERE f.id = ?
-      AND (f.deleted_at IS NULL OR
-           f.deleted_at > now())
-      FOR KEY SHARE")
-
 (defn get-file
   "Get not-decoded file, only decodes the features set."
-  [conn id]
-  (let [file (db/exec-one! conn [sql:get-file id])]
-    (when-not file
-      (ex/raise :type :not-found
-                :code :object-not-found
-                :hint (format "file with id '%s' does not exists" id)))
-    (update file :features db/decode-pgarray #{})))
+  [cfg id]
+  ;; FIXME: lock for share
+  (bfc/get-file cfg id :decode? false :lock-for-update? true))
 
 (defn persist-file!
   "Function responsible of persisting already encoded file. Should be
   used together with `get-file` and `update-file-data!`.
 
   It also updates the project modified-at attr."
-  [{:keys [::db/conn ::timestamp]} file]
+  [{:keys [::db/conn ::timestamp] :as cfg} file]
   (let [;; The timestamp can be nil because this function is also
         ;; intended to be used outside of this module
-        modified-at (or timestamp (ct/now))]
+        modified-at
+        (or timestamp (ct/now))
+
+        file
+        (-> file
+            (dissoc ::snapshot)
+            (assoc :modified-at modified-at)
+            (assoc :has-media-trimmed false))]
 
     (db/update! conn :project
                 {:modified-at modified-at}
                 {:id (:project-id file)}
                 {::db/return-keys false})
 
-    (db/update! conn :file
-                {:revn (:revn file)
-                 :data (:data file)
-                 :version (:version file)
-                 :features (:features file)
-                 :data-backend nil
-                 :data-ref-id nil
-                 :modified-at modified-at
-                 :has-media-trimmed false}
-                {:id (:id file)}
-                {::db/return-keys false})))
+    (bfc/update-file! cfg file)))
+
+(defn- attach-snapshot
+  "Attach snapshot data to the file. This should be called before the
+  upcoming file operations are applied to the file."
+  [file migrated? cfg]
+  (let [snapshot (if migrated? file (update file :data (partial fdata/realize cfg)))]
+    (assoc file ::snapshot snapshot)))
 
 (defn- update-file-data!
   "Perform a file data transformation in with all update context setup.
@@ -328,52 +337,35 @@
   fdata/pointer-map modified fragments."
 
   [cfg {:keys [id] :as file} update-fn & args]
-  (binding [pmap/*tracked* (pmap/create-tracked)
-            pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-    (let [file (update file :data (fn [data]
-                                    (-> data
-                                        (blob/decode)
-                                        (assoc :id (:id file)))))
-          libs (delay (bfc/get-resolved-file-libraries cfg file))
+  (let [file (update file :data (fn [data]
+                                  (-> data
+                                      (blob/decode)
+                                      (assoc :id id))))
+        libs (delay (bfc/get-resolved-file-libraries cfg file))
 
-          ;; For avoid unnecesary overhead of creating multiple pointers
-          ;; and handly internally with objects map in their worst
-          ;; case (when probably all shapes and all pointers will be
-          ;; readed in any case), we just realize/resolve them before
-          ;; applying the migration to the file
-          file (if (fmg/need-migration? file)
-                 (-> file
-                     (update :data feat.fdata/process-pointers deref)
-                     (update :data feat.fdata/process-objects (partial into {}))
-                     (fmg/migrate-file libs))
-                 file)
+        need-migration?
+        (fmg/need-migration? file)
 
-          file (apply update-fn cfg file args)
+        take-snapshot?
+        (take-snapshot? file)
 
-          ;; TODO: reuse operations if file is migrated
-          ;; TODO: move encoding to a separated thread
-          file (if (take-snapshot? file)
-                 (let [tpoint   (ct/tpoint)
-                       snapshot (-> (:data file)
-                                    (feat.fdata/process-pointers deref)
-                                    (feat.fdata/process-objects (partial into {}))
-                                    (blob/encode))
-                       elapsed (tpoint)
-                       label   (str "internal/snapshot/" (:revn file))]
+        ;; For avoid unnecesary overhead of creating multiple
+        ;; pointers and handly internally with objects map in their
+        ;; worst case (when probably all shapes and all pointers
+        ;; will be readed in any case), we just realize/resolve them
+        ;; before applying the migration to the file
+        file
+        (cond-> file
+          need-migration?
+          (->> (fdata/realize cfg))
 
-                   (l/trc :hint "take snapshot"
-                          :file-id (str (:id file))
-                          :revn (:revn file)
-                          :label label
-                          :elapsed (ct/format-duration elapsed))
+          need-migration?
+          (fmg/migrate-file libs)
 
-                   (-> file
-                       (assoc ::snapshot-data snapshot)
-                       (assoc ::snapshot-label label)))
-                 file)]
-
-      (bfc/encode-file cfg file))))
+          take-snapshot?
+          (attach-snapshot need-migration? cfg))]
 
+    (apply update-fn cfg file args)))
 
 (defn- soft-validate-file-schema!
   [file]
@@ -401,6 +393,7 @@
                    (not skip-validate))
           (bfc/get-resolved-file-libraries cfg file))
 
+
         ;; The main purpose of this atom is provide a contextual state
         ;; for the changes subsystem where optionally some hints can
         ;; be provided for the changes processing. Right now we are
@@ -462,8 +455,9 @@
 (defn- get-lagged-changes
   [conn {:keys [id revn] :as params}]
   (->> (db/exec! conn [sql:lagged-changes id revn])
-       (map files/decode-row)
-       (vec)))
+       (filter :changes)
+       (mapv (fn [row]
+               (update row :changes blob/decode)))))
 
 (defn- send-notifications!
   [cfg {:keys [team changes session-id] :as params} file]

backend/src/app/rpc/commands/fonts.clj

@@ -26,7 +26,9 @@
    [app.rpc.helpers :as rph]
    [app.rpc.quotes :as quotes]
    [app.storage :as sto]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 (def valid-weight #{100 200 300 400 500 600 700 800 900 950})
 (def valid-style #{"normal" "italic"})
@@ -35,13 +37,14 @@
 
 (def ^:private
   schema:get-font-variants
-  [:and
-   [:map {:title "get-font-variants"}
-    [:team-id {:optional true} ::sm/uuid]
-    [:file-id {:optional true} ::sm/uuid]
-    [:project-id {:optional true} ::sm/uuid]
-    [:share-id {:optional true} ::sm/uuid]]
-   [::sm/contains-any #{:team-id :file-id :project-id}]])
+  [:schema {:title "get-font-variants"}
+   [:and
+    [:map
+     [:team-id {:optional true} ::sm/uuid]
+     [:file-id {:optional true} ::sm/uuid]
+     [:project-id {:optional true} ::sm/uuid]
+     [:share-id {:optional true} ::sm/uuid]]
+    [::sm/contains-any #{:team-id :file-id :project-id}]]])
 
 (sv/defmethod ::get-font-variants
   {::doc/added "1.18"
@@ -103,7 +106,7 @@
                 (create-font-variant cfg (assoc params :profile-id profile-id)))))
 
 (defn create-font-variant
-  [{:keys [::sto/storage ::db/conn]} {:keys [data] :as params}]
+  [{:keys [::sto/storage ::db/conn ::wrk/executor]} {:keys [data] :as params}]
   (letfn [(generate-missing! [data]
             (let [data (media/run {:cmd :generate-fonts :input data})]
               (when (and (not (contains? data "font/otf"))
@@ -155,7 +158,7 @@
                          :otf-file-id (:id otf)
                          :ttf-file-id (:id ttf)}))]
 
-    (let [data   (generate-missing! data)
+    (let [data   (px/invoke! executor (partial generate-missing! data))
           assets (persist-fonts-files! data)
           result (insert-font-variant! assets)]
       (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))

backend/src/app/rpc/commands/management.clj

@@ -28,7 +28,9 @@
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
    [app.storage.tmp :as tmp]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 ;; --- COMMAND: Duplicate File
 
@@ -311,14 +313,15 @@
 
     ;; Update the modification date of the all affected projects
     ;; ensuring that the destination project is the most recent one.
-    (loop [project-ids (into (list project-id) source)
-           modified-at (ct/now)]
-      (when-let [project-id (first project-ids)]
-        (db/update! conn :project
-                    {:modified-at modified-at}
-                    {:id project-id})
-        (recur (rest project-ids)
-               (ct/plus modified-at 10))))
+    (doseq [project-id (into (list project-id) source)]
+
+      ;; NOTE: as this is executed on virtual thread, sleeping does
+      ;; not causes major issues, and allows an easy way to set a
+      ;; trully different modification date to each file.
+      (px/sleep 10)
+      (db/update! conn :project
+                  {:modified-at (ct/now)}
+                  {:id project-id}))
 
     nil))
 
@@ -393,7 +396,12 @@
 ;; --- COMMAND: Clone Template
 
 (defn clone-template
-  [{:keys [::db/pool] :as cfg} {:keys [project-id profile-id] :as params} template]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [project-id profile-id] :as params} template]
+
+  ;; NOTE: the importation process performs some operations
+  ;; that are not very friendly with virtual threads, and for
+  ;; avoid unexpected blocking of other concurrent operations
+  ;; we dispatch that operation to a dedicated executor.
   (let [template (tmp/tempfile-from template
                                     :prefix "penpot.template."
                                     :suffix ""
@@ -411,8 +419,8 @@
                      (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
 
         result   (if (= format :binfile-v3)
-                   (bf.v3/import-files! cfg)
-                   (bf.v1/import-files! cfg))]
+                   (px/invoke! executor (partial bf.v3/import-files! cfg))
+                   (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
     (db/tx-run! cfg
                 (fn [{:keys [::db/conn] :as cfg}]

backend/src/app/rpc/commands/media.clj

@@ -24,8 +24,10 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.services :as sv]
+   [app.worker :as-alias wrk]
    [cuerdas.core :as str]
-   [datoteka.io :as io]))
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
 
 (def default-max-file-size
   (* 1024 1024 10)) ; 10 MiB
@@ -46,7 +48,7 @@
    [:file-id ::sm/uuid]
    [:is-local ::sm/boolean]
    [:name [:string {:max 250}]]
-   [:content media/schema:upload]])
+   [:content ::media/upload]])
 
 (sv/defmethod ::upload-file-media-object
   {::doc/added "1.17"
@@ -151,9 +153,9 @@
       (assoc ::image (process-main-image info)))))
 
 (defn- create-file-media-object
-  [{:keys [::sto/storage ::db/conn] :as cfg}
+  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
    {:keys [id file-id is-local name content]}]
-  (let [result (process-image content)
+  (let [result (px/invoke! executor (partial process-image content))
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]

backend/src/app/rpc/commands/profile.clj

@@ -30,13 +30,16 @@
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
 
 (declare check-profile-existence!)
 (declare decode-row)
+(declare derive-password)
 (declare filter-props)
 (declare get-profile)
 (declare strip-private-attrs)
+(declare verify-password)
 
 (def schema:props-notifications
   [:map {:title "props-notifications"}
@@ -128,7 +131,9 @@
   ;; NOTE: we need to retrieve the profile independently if we use
   ;; it or not for explicit locking and avoid concurrent updates of
   ;; the same row/object.
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
+                    (decode-row))
+
         ;; Update the profile map with direct params
         profile (-> profile
                     (assoc :fullname fullname)
@@ -138,9 +143,9 @@
     (db/update! conn :profile
                 {:fullname fullname
                  :lang lang
-                 :theme theme}
-                {:id profile-id}
-                {::db/return-keys false})
+                 :theme theme
+                 :props (db/tjson (:props profile))}
+                {:id profile-id})
 
     (-> profile
         (strip-private-attrs)
@@ -189,7 +194,7 @@
   [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
   (let [profile (db/get-by-id conn :profile profile-id ::sql/for-update true)]
     (when (and (not= (:password profile) "!")
-               (not (:valid (auth/verify-password old-password (:password profile)))))
+               (not (:valid (verify-password cfg old-password (:password profile)))))
       (ex/raise :type :validation
                 :code :old-password-not-match))
     profile))
@@ -198,7 +203,7 @@
   [{:keys [::db/conn] :as cfg} {:keys [id password] :as profile}]
   (when-not (db/read-only? conn)
     (db/update! conn :profile
-                {:password (auth/derive-password password)}
+                {:password (derive-password cfg password)}
                 {:id id})
     nil))
 
@@ -223,22 +228,21 @@
 
 (defn- update-notifications!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
-  (let [profile
-        (get-profile conn profile-id ::db/for-update true)
+  (let [profile (get-profile conn profile-id)
 
         notifications
         {:dashboard-comments dashboard-comments
          :email-comments email-comments
-         :email-invites email-invites}
+         :email-invites email-invites}]
 
-        props
-        (-> (get profile :props)
-            (assoc :notifications notifications))]
+    (db/update!
+     conn :profile
+     {:props
+      (-> (:props profile)
+          (assoc :notifications notifications)
+          (db/tjson))}
+     {:id (:id profile)})
 
-    (db/update! conn :profile
-                {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
     nil))
 
 ;; --- MUTATION: Update Photo
@@ -249,7 +253,7 @@
 (def ^:private
   schema:update-profile-photo
   [:map {:title "update-profile-photo"}
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-profile-photo
   {:doc/added "1.1"
@@ -300,11 +304,12 @@
      :content-type (:mtype thumb)}))
 
 (defn upload-photo
-  [{:keys [::sto/storage] :as cfg} {:keys [file] :as params}]
+  [{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file] :as params}]
   (let [params (-> cfg
                    (assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
                                        [:process-image/global]])
                    (assoc ::climit/label "upload-photo")
+                   (assoc ::climit/executor executor)
                    (climit/invoke! generate-thumbnail! file))]
     (sto/put-object! storage params)))
 
@@ -406,7 +411,7 @@
 
 (defn update-profile-props
   [{:keys [::db/conn] :as cfg} profile-id props]
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (get-profile conn profile-id ::sql/for-update true)
         props   (reduce-kv (fn [props k v]
                              ;; We don't accept namespaced keys
                              (if (simple-ident? k)
@@ -419,17 +424,16 @@
 
     (db/update! conn :profile
                 {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
+                {:id profile-id})
 
     (filter-props props)))
 
 (sv/defmethod ::update-profile-props
   {::doc/added "1.0"
-   ::sm/params schema:update-profile-props
-   ::db/transaction true}
+   ::sm/params schema:update-profile-props}
   [cfg {:keys [::rpc/profile-id props]}]
-  (update-profile-props cfg profile-id props))
+  (db/tx-run! cfg (fn [cfg]
+                    (update-profile-props cfg profile-id props))))
 
 ;; --- MUTATION: Delete Profile
 
@@ -467,26 +471,6 @@
     (-> (rph/wrap nil)
         (rph/with-transform (session/delete-fn cfg)))))
 
-(def sql:get-subscription-editors
-  "SELECT DISTINCT
-          p.id,
-          p.fullname AS name,
-          p.email AS email
-     FROM team_profile_rel AS tpr1
-     JOIN team_profile_rel AS tpr2
-       ON (tpr1.team_id = tpr2.team_id)
-     JOIN profile AS p
-       ON (tpr2.profile_id = p.id)
-    WHERE tpr1.profile_id = ?
-      AND tpr1.is_owner IS true
-      AND tpr2.can_edit IS true")
-
-(sv/defmethod ::get-subscription-usage
-  {::doc/added "2.9"}
-  [cfg {:keys [::rpc/profile-id]}]
-  (let [editors (db/exec! cfg [sql:get-subscription-editors profile-id])]
-    {:editors editors}))
-
 ;; --- HELPERS
 
 (def sql:owned-teams
@@ -544,6 +528,15 @@
   [props]
   (into {} (filter (fn [[k _]] (simple-ident? k))) props))
 
+(defn derive-password
+  [{:keys [::wrk/executor]} password]
+  (when password
+    (px/invoke! executor (partial auth/derive-password password))))
+
+(defn verify-password
+  [{:keys [::wrk/executor]} password password-data]
+  (px/invoke! executor (partial auth/verify-password password password-data)))
+
 (defn decode-row
   [{:keys [props] :as row}]
   (cond-> row

backend/src/app/rpc/commands/teams.clj

@@ -12,7 +12,7 @@
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
    [app.common.time :as ct]
-   [app.common.types.team :as types.team]
+   [app.common.types.team :as tt]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -503,7 +503,7 @@
 
   (let [features (-> (cfeat/get-enabled-features cf/flags)
                      (set/difference cfeat/frontend-only-features)
-                     (set/difference cfeat/no-team-inheritable-features))
+                     (cfeat/check-client-features! (:features params)))
         params   (-> params
                      (assoc :profile-id profile-id)
                      (assoc :features features))
@@ -629,7 +629,7 @@
 
         ;; assign owner role to new profile
         (db/update! conn :team-profile-rel
-                    (get types.team/permissions-for-role :owner)
+                    (get tt/permissions-for-role :owner)
                     {:team-id id :profile-id reassign-to}))
 
       ;; and finally, if all other conditions does not match and the
@@ -742,7 +742,7 @@
                          :team-id team-id
                          :role role})
 
-    (let [params (get types.team/permissions-for-role role)]
+    (let [params (get tt/permissions-for-role role)]
       ;; Only allow single owner on team
       (when (= role :owner)
         (db/update! conn :team-profile-rel
@@ -760,7 +760,7 @@
   [:map {:title "update-team-member-role"}
    [:team-id ::sm/uuid]
    [:member-id ::sm/uuid]
-   [:role types.team/schema:role]])
+   [:role ::tt/role]])
 
 (sv/defmethod ::update-team-member-role
   {::doc/added "1.17"
@@ -810,7 +810,7 @@
 (def ^:private schema:update-team-photo
   [:map {:title "update-team-photo"}
    [:team-id ::sm/uuid]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-team-photo
   {::doc/added "1.17"

backend/src/app/rpc/commands/teams_invitations.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.teams-invitations
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -21,7 +22,6 @@
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :as files]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
@@ -75,7 +75,7 @@
     [:map
      [:id ::sm/uuid]
      [:fullname :string]]]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:email ::sm/email]])
 
 (def ^:private check-create-invitation-params
@@ -224,112 +224,62 @@
 (def ^:private xf:map-email (map :email))
 
 (defn- create-team-invitations
-  "Unified function to handle both create and resend team invitations.
-  Accepts either:
-  - emails (set) + role (single role for all emails)
-  - invitations (vector of {:email :role} maps)"
-  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails invitations] :as params}]
-  (let [;; Normalize input to a consistent format: [{:email :role}]
-        invitation-data  (cond
-                           ;; Case 1: emails + single role (create invitations style)
-                           (and emails role)
-                           (map (fn [email] {:email email :role role}) emails)
+  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails] :as params}]
+  (let [emails           (set emails)
 
-                           ;; Case 2: invitations with individual roles (resend invitations style)
-                           (some? invitations)
-                           invitations
+        join-requests    (->> (get-valid-access-request-profiles conn (:id team))
+                              (d/index-by :email))
 
-                           :else
-                           (throw (ex-info "Invalid parameters: must provide either emails+role or invitations" {})))
+        team-members     (into #{} xf:map-email
+                               (teams/get-team-members conn (:id team)))
 
-        invitation-emails (into #{} (map :email) invitation-data)
-
-        join-requests     (->> (get-valid-access-request-profiles conn (:id team))
-                               (d/index-by :email))
-
-        team-members      (into #{} xf:map-email
-                                (teams/get-team-members conn (:id team)))
-
-        invitations       (into #{}
-                                (comp
-                                ;; We don't re-send invitations to
-                                ;; already existing members
-                                 (remove #(contains? team-members (:email %)))
+        invitations      (into #{}
+                               (comp
+                                ;;  We don't re-send inviation to
+                                ;;  already existing members
+                                (remove team-members)
                                 ;; We don't send invitations to
                                 ;; join-requested members
-                                 (remove #(contains? join-requests (:email %)))
-                                 (map (fn [{:keys [email role]}]
-                                        (create-invitation cfg
-                                                           (-> params
-                                                               (assoc :email email)
-                                                               (assoc :role role)))))
-                                 (remove nil?))
-                                invitation-data)]
+                                (remove join-requests)
+                                (map (fn [email] (assoc params :email email)))
+                                (keep (partial create-invitation cfg)))
+                               emails)]
 
     ;; For requested invitations, do not send invitation emails, add
     ;; the user directly to the team
     (->> join-requests
-         (filter #(contains? invitation-emails (key %)))
-         (map (fn [[email member]]
-                (let [role (:role (first (filter #(= (:email %) email) invitation-data)))]
-                  (add-member-to-team conn profile team role member))))
-         (doall))
+         (filter #(contains? emails (key %)))
+         (map val)
+         (run! (partial add-member-to-team conn profile team role)))
 
     invitations))
 
 (def ^:private schema:create-team-invitations
-  [:and
-   [:map {:title "create-team-invitations"}
-    [:team-id ::sm/uuid]
-    ;; Support both formats:
-    ;; 1. emails (set) + role (single role for all)
-    ;; 2. invitations (vector of {:email :role} maps)
-    [:emails {:optional true} [::sm/set ::sm/email]]
-    [:role {:optional true} types.team/schema:role]
-    [:invitations {:optional true} [:vector [:map
-                                             [:email ::sm/email]
-                                             [:role types.team/schema:role]]]]]
-
-   ;; Ensure exactly one format is provided
-   [:fn (fn [params]
-          (let [has-emails-role (and (contains? params :emails)
-                                     (contains? params :role))
-                has-invitations (contains? params :invitations)]
-            (and (or has-emails-role has-invitations)
-                 (not (and has-emails-role has-invitations)))))]])
+  [:map {:title "create-team-invitations"}
+   [:team-id ::sm/uuid]
+   [:role ::types.team/role]
+   [:emails [::sm/set ::sm/email]]])
 
 (def ^:private max-invitations-by-request-threshold
   "The number of invitations can be sent in a single rpc request"
   25)
 
 (sv/defmethod ::create-team-invitations
-  "A rpc call that allows to send single or multiple invitations to join the team.
-
-  Supports two parameter formats:
-  1. emails (set) + role (single role for all emails)
-  2. invitations (vector of {:email :role} maps for individual roles)"
+  "A rpc call that allow to send a single or multiple invitations to
+  join the team."
   {::doc/added "1.17"
    ::doc/module :teams
    ::sm/params schema:create-team-invitations}
-  [cfg {:keys [::rpc/profile-id team-id role emails] :as params}]
+  [cfg {:keys [::rpc/profile-id team-id emails] :as params}]
   (let [perms    (teams/get-permissions cfg profile-id team-id)
         profile  (db/get-by-id cfg :profile profile-id)
-        ;; Determine which format is being used
-        using-emails-format? (and emails role)
-        ;; Handle both parameter formats
-        emails   (if using-emails-format?
-                   (into #{} (map profile/clean-email) emails)
-                   #{})
-        ;; Calculate total invitation count for both formats
-        invitation-count (if using-emails-format?
-                           (count emails)
-                           (count (:invitations params)))]
+        emails   (into #{} (map profile/clean-email) emails)]
 
     (when-not (:is-admin perms)
       (ex/raise :type :validation
                 :code :insufficient-permissions))
 
-    (when (> invitation-count max-invitations-by-request-threshold)
+    (when (> (count emails) max-invitations-by-request-threshold)
       (ex/raise :type :validation
                 :code :max-invitations-by-request
                 :hint "the maximum of invitation on single request is reached"
@@ -338,7 +288,7 @@
     (-> cfg
         (assoc ::quotes/profile-id profile-id)
         (assoc ::quotes/team-id team-id)
-        (assoc ::quotes/incr invitation-count)
+        (assoc ::quotes/incr (count emails))
         (quotes/check! {::quotes/id ::quotes/invitations-per-team}
                        {::quotes/id ::quotes/profiles-per-team}))
 
@@ -354,12 +304,7 @@
                                   (-> params
                                       (assoc :profile profile)
                                       (assoc :team team)
-                                      ;; Pass parameters in the correct format for the unified function
-                                      (cond-> using-emails-format?
-                                        ;; If using emails+role format, ensure both are present
-                                        (assoc :emails emails :role role)
-                                        ;; If using invitations format, the :invitations key is already in params
-                                        (not using-emails-format?) identity)))]
+                                      (assoc :emails emails)))]
 
       (with-meta {:total (count invitations)
                   :invitations invitations}
@@ -373,7 +318,7 @@
    [:features {:optional true} ::cfeat/features]
    [:id {:optional true} ::sm/uuid]
    [:emails [::sm/set ::sm/email]]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::create-team-with-invitations
   {::doc/added "1.17"
@@ -458,7 +403,7 @@
   [:map {:title "update-team-invitation-role"}
    [:team-id ::sm/uuid]
    [:email ::sm/email]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::update-team-invitation-role
   {::doc/added "1.17"
@@ -554,7 +499,7 @@
   "A specific method for obtain a file with name and page-id used for
   team request access procediment"
   [cfg file-id]
-  (let [file (files/get-file cfg file-id :migrate? false)]
+  (let [file (bfc/get-file cfg file-id :migrate? false)]
     (-> file
         (dissoc :data)
         (dissoc :deleted-at)

backend/src/app/rpc/commands/verify_token.clj

@@ -128,7 +128,7 @@
    [:iss :keyword]
    [:exp ::ct/inst]
    [:profile-id ::sm/uuid]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:team-id ::sm/uuid]
    [:member-email ::sm/email]
    [:member-id {:optional true} ::sm/uuid]])

backend/src/app/rpc/commands/viewer.clj

@@ -51,7 +51,7 @@
 
 (defn- get-view-only-bundle
   [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
-  (let [file    (files/get-file cfg file-id)
+  (let [file    (bfc/get-file cfg file-id)
 
         project (db/get conn :project
                         {:id (:project-id file)}
@@ -81,7 +81,7 @@
 
         libs    (->> (bfc/get-file-libraries conn file-id)
                      (mapv (fn [{:keys [id] :as lib}]
-                             (merge lib (files/get-file cfg id)))))
+                             (merge lib (bfc/get-file cfg id)))))
 
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]

backend/src/app/rpc/doc.clj

@@ -166,6 +166,9 @@
      :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
                 ;; :description "penpot backend"
                 }]
+     :security
+     {:api_key []}
+
      :paths paths
      :components {:schemas @definitions}}))
 

backend/src/app/rpc/permissions.clj

@@ -10,14 +10,15 @@
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]))
 
-(def schema:permissions
-  [:map {:title "Permissions"}
-   [:type {:gen/elements [:membership :share-link]} :keyword]
-   [:is-owner ::sm/boolean]
-   [:is-admin ::sm/boolean]
-   [:can-edit ::sm/boolean]
-   [:can-read ::sm/boolean]
-   [:is-logged ::sm/boolean]])
+(sm/register!
+ ^{::sm/type ::permissions}
+ [:map {:title "Permissions"}
+  [:type {:gen/elements [:membership :share-link]} :keyword]
+  [:is-owner ::sm/boolean]
+  [:is-admin ::sm/boolean]
+  [:can-edit ::sm/boolean]
+  [:can-read ::sm/boolean]
+  [:is-logged ::sm/boolean]])
 
 (def valid-roles
   #{:admin :owner :editor :viewer})

backend/src/app/srepl/cli.clj

@@ -7,7 +7,7 @@
 (ns app.srepl.cli
   "PREPL API for external usage (CLI or ADMIN)"
   (:require
-   [app.auth :refer [derive-password]]
+   [app.auth :as auth]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.common.schema.generators :as sg]
@@ -54,7 +54,7 @@
   (some-> (get-current-system)
           (db/tx-run!
            (fn [{:keys [::db/conn] :as system}]
-             (let [password (derive-password password)
+             (let [password (cmd.profile/derive-password system password)
                    params   {:id (uuid/next)
                              :email email
                              :fullname fullname
@@ -74,7 +74,7 @@
                             (assoc :fullname fullname)
 
                             (some? password)
-                            (assoc :password (derive-password password))
+                            (assoc :password (auth/derive-password password))
 
                             (some? is-active)
                             (assoc :is-active is-active))]
@@ -124,7 +124,7 @@
 
 (defmethod exec-command "derive-password"
   [{:keys [password]}]
-  (derive-password password))
+  (auth/derive-password password))
 
 (defmethod exec-command "authenticate"
   [{:keys [token]}]

backend/src/app/srepl/helpers.clj

@@ -14,9 +14,8 @@
    [app.common.files.validate :as cfv]
    [app.common.time :as ct]
    [app.db :as db]
-   [app.main :as main]
-   [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-snapshot :as fsnap]))
+   [app.features.file-snapshots :as fsnap]
+   [app.main :as main]))
 
 (def ^:dynamic *system* nil)
 
@@ -48,7 +47,7 @@
   ([system id]
    (db/run! system
             (fn [system]
-              (files/get-file system id :migrate? false)))))
+              (bfc/get-file system id :decode? false)))))
 
 (defn update-team!
   [system {:keys [id] :as team}]
@@ -118,10 +117,10 @@
   (let [conn (db/get-connection system)]
     (->> (get-and-lock-team-files conn team-id)
          (reduce (fn [result file-id]
-                   (let [file (fsnap/get-file-snapshots system file-id)]
-                     (fsnap/create-file-snapshot! system file
-                                                  {:label label
-                                                   :created-by :admin})
+                   (let [file (bfc/get-file system file-id :realize? true :lock-for-update? true)]
+                     (fsnap/create! system file
+                                    {:label label
+                                     :created-by "admin"})
                      (inc result)))
                  0))))
 
@@ -132,21 +131,23 @@
                   (into #{}))
 
         snap (search-file-snapshots conn ids label)
-
         ids' (into #{} (map :file-id) snap)]
 
     (when (not= ids ids')
       (throw (RuntimeException. "no uniform snapshot available")))
 
     (reduce (fn [result {:keys [file-id id]}]
-              (fsnap/restore-file-snapshot! system file-id id)
+              (fsnap/restore! system file-id id)
               (inc result))
             0
             snap)))
 
 (defn process-file!
   [system file-id update-fn & {:keys [label validate? with-libraries?] :or {validate? true} :as opts}]
-  (let [file  (bfc/get-file system file-id ::db/for-update true)
+  (let [file  (bfc/get-file system file-id
+                            :lock-for-update? true
+                            :realize? true)
+
         libs  (when with-libraries?
                 (bfc/get-resolved-file-libraries system file))
 
@@ -163,10 +164,10 @@
         (cfv/validate-file-schema! file'))
 
       (when (string? label)
-        (fsnap/create-file-snapshot! system file
-                                     {:label label
-                                      :deleted-at (ct/in-future {:days 30})
-                                      :created-by :admin}))
+        (fsnap/create! system file
+                       {:label label
+                        :deleted-at (ct/in-future {:days 30})
+                        :created-by "admin"}))
 
       (let [file' (update file' :revn inc)]
         (bfc/update-file! system file')

backend/src/app/srepl/main.clj

@@ -24,13 +24,13 @@
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.fdata :as feat.fdata]
+   [app.features.fdata :as fdata]
+   [app.features.file-snapshots :as fsnap]
    [app.loggers.audit :as audit]
    [app.main :as main]
    [app.msgbus :as mbus]
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.files-snapshot :as fsnap]
    [app.rpc.commands.management :as mgmt]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.projects :as projects]
@@ -40,7 +40,6 @@
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.worker :as wrk]
-   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [print-table]]
    [clojure.stacktrace :as strace]
@@ -151,15 +150,15 @@
 
 (defn enable-objects-map-feature-on-file!
   [file-id & {:as opts}]
-  (process-file! file-id feat.fdata/enable-objects-map opts))
+  (process-file! file-id fdata/enable-objects-map opts))
 
 (defn enable-pointer-map-feature-on-file!
   [file-id & {:as opts}]
-  (process-file! file-id feat.fdata/enable-pointer-map opts))
+  (process-file! file-id fdata/enable-pointer-map opts))
 
 (defn enable-path-data-feature-on-file!
   [file-id & {:as opts}]
-  (process-file! file-id feat.fdata/enable-path-data opts))
+  (process-file! file-id fdata/enable-path-data opts))
 
 (defn enable-storage-features-on-file!
   [file-id & {:as opts}]
@@ -339,7 +338,10 @@
   collectable file-changes entry."
   [& {:keys [file-id label]}]
   (let [file-id (h/parse-uuid file-id)]
-    (db/tx-run! main/system fsnap/create-file-snapshot! {:file-id file-id :label label})))
+    (db/tx-run! main/system
+                (fn [cfg]
+                  (let [file (bfc/get-file cfg file-id :realize? true)]
+                    (fsnap/create! cfg file {:label label :created-by "admin"}))))))
 
 (defn restore-file-snapshot!
   [file-id & {:keys [label id]}]
@@ -349,13 +351,13 @@
                 (fn [{:keys [::db/conn] :as system}]
                   (cond
                     (uuid? snapshot-id)
-                    (fsnap/restore-file-snapshot! system file-id snapshot-id)
+                    (fsnap/restore! system file-id snapshot-id)
 
                     (string? label)
                     (->> (h/search-file-snapshots conn #{file-id} label)
                          (map :id)
                          (first)
-                         (fsnap/restore-file-snapshot! system file-id))
+                         (fsnap/restore! system file-id))
 
                     :else
                     (throw (ex-info "snapshot id or label should be provided" {})))))))
@@ -364,9 +366,9 @@
   [file-id & {:as _}]
   (let [file-id (h/parse-uuid file-id)]
     (db/tx-run! main/system
-                (fn [{:keys [::db/conn]}]
-                  (->> (fsnap/get-file-snapshots conn file-id)
-                       (print-table [:label :id :revn :created-at]))))))
+                (fn [cfg]
+                  (->> (fsnap/get-visible-snapshots cfg file-id)
+                       (print-table [:label :id :revn :created-at :created-by]))))))
 
 (defn take-team-snapshot!
   [team-id & {:keys [label rollback?] :or {rollback? true}}]
@@ -548,6 +550,68 @@
                  :rollback rollback?
                  :elapsed elapsed))))))
 
+(defn process!
+  "Apply a function to all files in the database"
+  [& {:keys [max-jobs
+             rollback?
+             max-items
+             chunk-size
+             proc-fn]
+      :or {max-items Long/MAX_VALUE
+           chunk-size 100
+           rollback? true}
+      :as opts}]
+
+  (let [tpoint     (ct/tpoint)
+        max-jobs   (or max-jobs (px/get-available-processors))
+        processed  (atom 0)
+        opts       (-> opts
+                       (assoc :chunk-size chunk-size)
+                       (dissoc :rollback?)
+                       (dissoc :proc-fn)
+                       (dissoc :max-jobs)
+                       (dissoc :max-items))
+
+        start-job
+        (fn [jid]
+          (l/dbg :hint "start job thread" :jid jid)
+          (px/sleep 1000)
+
+          (loop []
+            (let [result (-> main/system
+                             (assoc ::db/rollback rollback?)
+                             (proc-fn opts))]
+              (let [total (swap! processed + result)]
+                (l/dbg :hint "chunk processed" :jid jid :total total :chunk result ::l/sync? true)
+
+                (when (and (pos? result)
+                           (< total max-items))
+                  (recur))))))]
+
+    (l/dbg :hint "process:start"
+           :rollback rollback?
+           :max-jobs max-jobs
+           :max-items max-items)
+
+    (try
+      (let [jobs (->> (range max-jobs)
+                      (map (fn [jid] (px/fn->thread (partial start-job jid))))
+                      (doall))]
+        (doseq [job jobs]
+          (.join ^java.lang.Thread job)))
+
+      (catch Throwable cause
+        (l/dbg :hint "process:error" :cause cause))
+
+      (finally
+        (let [elapsed (ct/format-duration (tpoint))]
+          (l/dbg :hint "process:end"
+                 :processed @processed
+                 :rollback rollback?
+                 :elapsed elapsed))))))
+
+
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DELETE/RESTORE OBJECTS (WITH CASCADE, SOFT)
@@ -607,11 +671,10 @@
   (let [file-id (h/parse-uuid file-id)]
     (db/tx-run! main/system
                 (fn [system]
-                  (when-let [file (some-> (db/get* system :file
-                                                   {:id file-id}
-                                                   {::db/remove-deleted false
-                                                    ::sql/columns [:id :name]})
-                                          (files/decode-row))]
+                  (when-let [file (db/get* system :file
+                                           {:id file-id}
+                                           {::db/remove-deleted false
+                                            ::sql/columns [:id :name]})]
                     (audit/insert! system
                                    {::audit/name "restore-file"
                                     ::audit/type "action"
@@ -832,6 +895,19 @@
                     (with-open [reader (io/reader path)]
                       (process-data! system deleted-at (line-seq reader))))))))
 
+
+(defn process-chunks
+  "A generic function that executes the specified proc iterativelly
+  until 0 results is returned"
+  [cfg proc-fn & params]
+  (loop [total 0]
+    (let [result (apply proc-fn cfg params)]
+      (if (pos? result)
+        (do
+          (l/trc :hint "chunk processed" :size result :total total)
+          (recur (+ total result)))
+        total))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; CASCADE FIXING
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/storage.clj

@@ -27,9 +27,7 @@
 
 (defn get-legacy-backend
   []
-  (when-let [name (cf/get :assets-storage-backend)]
-    (l/wrn :hint "using deprecated configuration, please read 2.11 release notes"
-           :href "https://github.com/penpot/penpot/releases/tag/2.11.0")
+  (let [name (cf/get :assets-storage-backend)]
     (case name
       :assets-fs :fs
       :assets-s3 :s3
@@ -115,13 +113,10 @@
 
 (defn- create-database-object
   [{:keys [::backend ::db/connectable]} {:keys [::content ::expired-at ::touched-at ::touch] :as params}]
-  (let [id     (or (:id params) (uuid/random))
+  (let [id     (or (::id params) (uuid/random))
         mdata  (cond-> (get-metadata params)
                  (satisfies? impl/IContentHash content)
-                 (assoc :hash (impl/get-hash content))
-
-                 :always
-                 (dissoc :id))
+                 (assoc :hash (impl/get-hash content)))
 
         touched-at (if touch
                      (or touched-at (ct/now))

backend/src/app/storage/gc_touched.clj

@@ -34,7 +34,7 @@
            (SELECT EXISTS (SELECT 1 FROM team_font_variant WHERE ttf_file_id = ?))) AS has_refs")
 
 (defn- has-team-font-variant-refs?
-  [conn id]
+  [conn {:keys [id]}]
   (-> (db/exec-one! conn [sql:has-team-font-variant-refs id id id id])
       (get :has-refs)))
 
@@ -44,7 +44,7 @@
            (SELECT EXISTS (SELECT 1 FROM file_media_object WHERE thumbnail_id = ?))) AS has_refs")
 
 (defn- has-file-media-object-refs?
-  [conn id]
+  [conn {:keys [id]}]
   (-> (db/exec-one! conn [sql:has-file-media-object-refs id id])
       (get :has-refs)))
 
@@ -53,7 +53,7 @@
            (SELECT EXISTS (SELECT 1 FROM team WHERE photo_id = ?))) AS has_refs")
 
 (defn- has-profile-refs?
-  [conn id]
+  [conn {:keys [id]}]
   (-> (db/exec-one! conn [sql:has-profile-refs id id])
       (get :has-refs)))
 
@@ -62,7 +62,7 @@
   "SELECT EXISTS (SELECT 1 FROM file_tagged_object_thumbnail WHERE media_id = ?) AS has_refs")
 
 (defn- has-file-object-thumbnails-refs?
-  [conn id]
+  [conn {:keys [id]}]
   (-> (db/exec-one! conn [sql:has-file-object-thumbnail-refs id])
       (get :has-refs)))
 
@@ -71,36 +71,23 @@
   "SELECT EXISTS (SELECT 1 FROM file_thumbnail WHERE media_id = ?) AS has_refs")
 
 (defn- has-file-thumbnails-refs?
-  [conn id]
+  [conn {:keys [id]}]
   (-> (db/exec-one! conn [sql:has-file-thumbnail-refs id])
       (get :has-refs)))
 
-(def ^:private
-  sql:has-file-data-refs
-  "SELECT EXISTS (SELECT 1 FROM file WHERE data_ref_id = ?) AS has_refs")
+(def sql:exists-file-data-refs
+  "SELECT EXISTS (
+     SELECT 1 FROM file_data
+      WHERE file_id = ?
+        AND id = ?
+        AND metadata->>'storage-ref-id' = ?::text
+   ) AS has_refs")
 
 (defn- has-file-data-refs?
-  [conn id]
-  (-> (db/exec-one! conn [sql:has-file-data-refs id])
-      (get :has-refs)))
-
-(def ^:private
-  sql:has-file-data-fragment-refs
-  "SELECT EXISTS (SELECT 1 FROM file_data_fragment WHERE data_ref_id = ?) AS has_refs")
-
-(defn- has-file-data-fragment-refs?
-  [conn id]
-  (-> (db/exec-one! conn [sql:has-file-data-fragment-refs id])
-      (get :has-refs)))
-
-(def ^:private
-  sql:has-file-change-refs
-  "SELECT EXISTS (SELECT 1 FROM file_change WHERE data_ref_id = ?) AS has_refs")
-
-(defn- has-file-change-refs?
-  [conn id]
-  (-> (db/exec-one! conn [sql:has-file-change-refs id])
-      (get :has-refs)))
+  [conn sobject]
+  (let [{:keys [file-id id]} (:metadata sobject)]
+    (-> (db/exec-one! conn [sql:exists-file-data-refs file-id id (:id sobject)])
+        (get :has-refs))))
 
 (def ^:private sql:mark-freeze-in-bulk
   "UPDATE storage_object
@@ -143,52 +130,50 @@
       "file-media-object"))
 
 (defn- process-objects!
-  [conn has-refs? ids bucket]
+  [conn has-refs? bucket objects]
   (loop [to-freeze #{}
          to-delete #{}
-         ids       (seq ids)]
-    (if-let [id (first ids)]
-      (if (has-refs? conn id)
+         objects   (seq objects)]
+    (if-let [{:keys [id] :as object} (first objects)]
+      (if (has-refs? conn object)
         (do
           (l/debug :hint "processing object"
                    :id (str id)
                    :status "freeze"
                    :bucket bucket)
-          (recur (conj to-freeze id) to-delete (rest ids)))
+          (recur (conj to-freeze id) to-delete (rest objects)))
         (do
           (l/debug :hint "processing object"
                    :id (str id)
                    :status "delete"
                    :bucket bucket)
-          (recur to-freeze (conj to-delete id) (rest ids))))
+          (recur to-freeze (conj to-delete id) (rest objects))))
       (do
         (some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
         (some->> (seq to-delete) (mark-delete-in-bulk! conn))
         [(count to-freeze) (count to-delete)]))))
 
 (defn- process-bucket!
-  [conn bucket ids]
+  [conn bucket objects]
   (case bucket
-    "file-media-object"     (process-objects! conn has-file-media-object-refs? ids bucket)
-    "team-font-variant"     (process-objects! conn has-team-font-variant-refs? ids bucket)
-    "file-object-thumbnail" (process-objects! conn has-file-object-thumbnails-refs? ids bucket)
-    "file-thumbnail"        (process-objects! conn has-file-thumbnails-refs? ids bucket)
-    "profile"               (process-objects! conn has-profile-refs? ids bucket)
-    "file-data"             (process-objects! conn has-file-data-refs? ids bucket)
-    "file-data-fragment"    (process-objects! conn has-file-data-fragment-refs? ids bucket)
-    "file-change"           (process-objects! conn has-file-change-refs? ids bucket)
+    "file-media-object"     (process-objects! conn has-file-media-object-refs? bucket objects)
+    "team-font-variant"     (process-objects! conn has-team-font-variant-refs? bucket objects)
+    "file-object-thumbnail" (process-objects! conn has-file-object-thumbnails-refs? bucket objects)
+    "file-thumbnail"        (process-objects! conn has-file-thumbnails-refs? bucket objects)
+    "profile"               (process-objects! conn has-profile-refs? bucket objects)
+    "file-data"             (process-objects! conn has-file-data-refs? bucket objects)
     (ex/raise :type :internal
               :code :unexpected-unknown-reference
               :hint (dm/fmt "unknown reference '%'" bucket))))
 
 (defn process-chunk!
   [{:keys [::db/conn]} chunk]
-  (reduce-kv (fn [[nfo ndo] bucket ids]
-               (let [[nfo' ndo'] (process-bucket! conn bucket ids)]
+  (reduce-kv (fn [[nfo ndo] bucket objects]
+               (let [[nfo' ndo'] (process-bucket! conn bucket objects)]
                  [(+ nfo nfo')
                   (+ ndo ndo')]))
              [0 0]
-             (d/group-by lookup-bucket :id #{} chunk)))
+             (d/group-by lookup-bucket identity #{} chunk)))
 
 (def ^:private
   sql:get-touched-storage-objects
@@ -212,8 +197,8 @@
          deleted 0]
     (if-let [chunk (get-chunk pool)]
       (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
-        (recur (long (+ freezed nfo))
-               (long (+ deleted ndo))))
+        (recur (+ freezed nfo)
+               (+ deleted ndo)))
       (do
         (l/inf :hint "task finished"
                :to-freeze freezed

backend/src/app/storage/s3.clj

@@ -31,13 +31,13 @@
    java.time.Duration
    java.util.Collection
    java.util.Optional
-   java.util.concurrent.atomic.AtomicLong
    org.reactivestreams.Subscriber
    software.amazon.awssdk.core.ResponseBytes
    software.amazon.awssdk.core.async.AsyncRequestBody
    software.amazon.awssdk.core.async.AsyncResponseTransformer
    software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody
    software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
+   software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
    software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
    software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
    software.amazon.awssdk.regions.Region
@@ -87,11 +87,12 @@
 
 (def ^:private schema:config
   [:map {:title "s3-backend-config"}
-   ::wrk/netty-io-executor
+   ::wrk/executor
    [::region {:optional true} :keyword]
    [::bucket {:optional true} ::sm/text]
    [::prefix {:optional true} ::sm/text]
-   [::endpoint {:optional true} ::sm/uri]])
+   [::endpoint {:optional true} ::sm/uri]
+   [::io-threads {:optional true} ::sm/int]])
 
 (defmethod ig/expand-key ::backend
   [k v]
@@ -109,7 +110,6 @@
           presigner (build-s3-presigner params)]
       (assoc params
              ::sto/type :s3
-             ::counter (AtomicLong. 0)
              ::client @client
              ::presigner presigner
              ::close-fn #(.close ^java.lang.AutoCloseable client)))))
@@ -121,7 +121,7 @@
 (defmethod ig/halt-key! ::backend
   [_ {:keys [::close-fn]}]
   (when (fn? close-fn)
-    (close-fn)))
+    (px/run! close-fn)))
 
 (def ^:private schema:backend
   [:map {:title "s3-backend"}
@@ -198,16 +198,19 @@
   (Region/of (name region)))
 
 (defn- build-s3-client
-  [{:keys [::region ::endpoint ::wrk/netty-io-executor]}]
+  [{:keys [::region ::endpoint ::io-threads ::wrk/executor]}]
   (let [aconfig  (-> (ClientAsyncConfiguration/builder)
+                     (.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
                      (.build))
 
         sconfig  (-> (S3Configuration/builder)
                      (cond-> (some? endpoint) (.pathStyleAccessEnabled true))
                      (.build))
 
+        thr-num  (or io-threads (min 16 (px/get-available-processors)))
         hclient  (-> (NettyNioAsyncHttpClient/builder)
-                     (.eventLoopGroup (SdkEventLoopGroup/create netty-io-executor))
+                     (.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
+                                                 (.numberOfThreads (int thr-num))))
                      (.connectionAcquisitionTimeout default-timeout)
                      (.connectionTimeout default-timeout)
                      (.readTimeout default-timeout)
@@ -259,7 +262,7 @@
       (.close ^InputStream input))))
 
 (defn- make-request-body
-  [counter content]
+  [executor content]
   (let [size (impl/get-size content)]
     (reify
       AsyncRequestBody
@@ -269,19 +272,16 @@
       (^void subscribe [_ ^Subscriber subscriber]
         (let [delegate (AsyncRequestBody/forBlockingInputStream (long size))
               input    (io/input-stream content)]
-
-          (px/thread-call (partial write-input-stream delegate input)
-                          {:name (str "penpot/storage/" (.getAndIncrement ^AtomicLong counter))})
-
+          (px/run! executor (partial write-input-stream delegate input))
           (.subscribe ^BlockingInputStreamAsyncRequestBody delegate
                       ^Subscriber subscriber))))))
 
 (defn- put-object
-  [{:keys [::client ::bucket ::prefix ::counter]} {:keys [id] :as object} content]
+  [{:keys [::client ::bucket ::prefix ::wrk/executor]} {:keys [id] :as object} content]
   (let [path    (dm/str prefix (impl/id->path id))
         mdata   (meta object)
         mtype   (:content-type mdata "application/octet-stream")
-        rbody   (make-request-body counter content)
+        rbody   (make-request-body executor content)
         request (.. (PutObjectRequest/builder)
                     (bucket bucket)
                     (contentType mtype)

backend/src/app/storage/tmp.clj

@@ -44,7 +44,7 @@
   [_ cfg]
   (fs/create-dir default-tmp-dir)
   (px/fn->thread (partial io-loop cfg)
-                 {:name "penpot/storage/tmp-cleaner"}))
+                 {:name "penpot/storage/tmp-cleaner" :virtual true}))
 
 (defmethod ig/halt-key! ::cleaner
   [_ thread]

backend/src/app/tasks/delete_object.clj

@@ -45,6 +45,11 @@
                 {:deleted-at deleted-at}
                 {:file-id id})
 
+    ;; Mark file data fragment to be deleted
+    (db/update! conn :file-data-fragment
+                {:deleted-at deleted-at}
+                {:file-id id})
+
     ;; Mark file media objects to be deleted
     (db/update! conn :file-media-object
                 {:deleted-at deleted-at}

backend/src/app/tasks/file_gc.clj

@@ -23,26 +23,13 @@
    [app.config :as cf]
    [app.db :as db]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-snapshots :as fsnap]
    [app.storage :as sto]
    [app.worker :as wrk]
    [integrant.core :as ig]))
 
 (declare get-file)
 
-(def sql:get-snapshots
-  "SELECT fc.file_id AS id,
-          fc.id AS snapshot_id,
-          fc.data,
-          fc.revn,
-          fc.version,
-          fc.features,
-          fc.data_backend,
-          fc.data_ref_id
-     FROM file_change AS fc
-    WHERE fc.file_id = ?
-      AND fc.data IS NOT NULL
-    ORDER BY fc.created_at ASC")
-
 (def ^:private sql:mark-file-media-object-deleted
   "UPDATE file_media_object
       SET deleted_at = now()
@@ -57,21 +44,22 @@
 (defn- clean-file-media!
   "Performs the garbage collection of file media objects."
   [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [xform  (comp
-                (map (partial bfc/decode-file cfg))
-                xf:collect-used-media)
+  (let [used-media
+        (fsnap/reduce-snapshots cfg id xf:collect-used-media conj #{})
 
-        used   (->> (db/plan conn [sql:get-snapshots id] {:fetch-size 1})
-                    (transduce xform conj #{}))
-        used   (into used xf:collect-used-media [file])
+        used-media
+        (into used-media xf:collect-used-media [file])
 
-        ids    (db/create-array conn "uuid" used)
-        unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids])
-                    (into #{} (map :id)))]
+        used-media
+        (db/create-array conn "uuid" used-media)
 
-    (l/dbg :hint "clean" :rel "file-media-object" :file-id (str id) :total (count unused))
+        unused-media
+        (->> (db/exec! conn [sql:mark-file-media-object-deleted id used-media])
+             (into #{} (map :id)))]
 
-    (doseq [id unused]
+    (l/dbg :hint "clean" :rel "file-media-object" :file-id (str id) :total (count unused-media))
+
+    (doseq [id unused-media]
       (l/trc :hint "mark deleted"
              :rel "file-media-object"
              :id (str id)
@@ -98,7 +86,7 @@
                                            (thc/fmt-object-id file-id page-id id "frame")
                                            (thc/fmt-object-id file-id page-id id "component")))))))
 
-        ids    (db/create-array conn "text" using)
+        ids    (db/create-array conn "uuid" using)
         unused (->> (db/exec! conn [sql:mark-file-object-thumbnails-deleted file-id ids])
                     (into #{} (map :object-id)))]
 
@@ -134,13 +122,7 @@
     file))
 
 (def ^:private sql:get-files-for-library
-  "SELECT f.id,
-          f.data,
-          f.modified_at,
-          f.features,
-          f.version,
-          f.data_backend,
-          f.data_ref_id
+  "SELECT f.id
      FROM file AS f
      LEFT JOIN file_library_rel AS fl ON (fl.file_id = f.id)
     WHERE fl.library_file_id = ?
@@ -161,15 +143,21 @@
         deleted-components
         (ctkl/deleted-components-seq data)
 
-        xform
+        file-xform
         (mapcat (partial get-used-components deleted-components file-id))
 
+        library-xform
+        (comp
+         (map :id)
+         (map #(bfc/get-file cfg % :realize? true :read-only? true))
+         file-xform)
+
         used-remote
         (->> (db/plan conn [sql:get-files-for-library file-id] {:fetch-size 1})
-             (transduce (comp (map (partial bfc/decode-file cfg)) xform) conj #{}))
+             (transduce library-xform conj #{}))
 
         used-local
-        (into #{} xform [file])
+        (into #{} file-xform [file])
 
         unused
         (transduce bfc/xf-map-id disj
@@ -229,34 +217,22 @@
     (cfv/validate-file-schema! file)
     file))
 
-(def ^:private sql:get-file
-  "SELECT f.id,
-          f.data,
-          f.revn,
-          f.version,
-          f.features,
-          f.modified_at,
-          f.data_backend,
-          f.data_ref_id
-     FROM file AS f
-    WHERE f.has_media_trimmed IS false
-      AND f.modified_at < now() - ?::interval
-      AND f.deleted_at IS NULL
-      AND f.id = ?
-      FOR UPDATE
-     SKIP LOCKED")
-
 (defn get-file
-  [{:keys [::db/conn ::min-age]} file-id]
-  (let [min-age (if min-age
-                  (db/interval min-age)
-                  (db/interval 0))]
-    (->> (db/exec! conn [sql:get-file min-age file-id])
-         (first))))
+  [cfg {:keys [file-id revn]}]
+  (let [file (bfc/get-file cfg file-id
+                           :realize? true
+                           :skip-locked? true
+                           :lock-for-update? true)]
+
+    ;; We should ensure that the scheduled file and the procesing file
+    ;; has not changed since schedule, for this reason we check the
+    ;; revn from props with the revn from retrieved file from database
+    (when (= revn (:revn file))
+      file)))
 
 (defn- process-file!
-  [cfg file-id]
-  (if-let [file (get-file cfg file-id)]
+  [cfg {:keys [file-id] :as props}]
+  (if-let [file (get-file cfg props)]
     (let [file (->> file
                     (bfc/decode-file cfg)
                     (bfl/clean-file)
@@ -267,7 +243,7 @@
       true)
 
     (do
-      (l/dbg :hint "skip" :file-id (str file-id))
+      (l/dbg :hint "skip cleaning, criteria does not match" :file-id (str file-id))
       false)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -282,26 +258,20 @@
 (defmethod ig/init-key ::handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (let [min-age (ct/duration (or (:min-age props)
-                                   (cf/get-deletion-delay)))
-          file-id (get props :file-id)
-          cfg     (-> cfg
-                      (assoc ::db/rollback (:rollback? props))
-                      (assoc ::min-age min-age))]
-
-      (try
-        (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                          (let [cfg        (update cfg ::sto/storage sto/configure conn)
-                                processed? (process-file! cfg file-id)]
-                            (when (and processed? (contains? cf/flags :tiered-file-data-storage))
-                              (wrk/submit! (-> cfg
-                                               (assoc ::wrk/task :offload-file-data)
-                                               (assoc ::wrk/params props)
-                                               (assoc ::wrk/priority 10)
-                                               (assoc ::wrk/delay 1000))))
-                            processed?)))
-
-        (catch Throwable cause
-          (l/err :hint "error on cleaning file"
-                 :file-id (str (:file-id props))
-                 :cause cause))))))
+    (try
+      (-> cfg
+          (assoc ::db/rollback (:rollback? props))
+          (db/tx-run! (fn [{:keys [::db/conn] :as cfg}]
+                        (let [cfg        (update cfg ::sto/storage sto/configure conn)
+                              processed? (process-file! cfg props)]
+                          (when (and processed? (contains? cf/flags :tiered-file-data-storage))
+                            (wrk/submit! (-> cfg
+                                             (assoc ::wrk/task :offload-file-data)
+                                             (assoc ::wrk/params props)
+                                             (assoc ::wrk/priority 10)
+                                             (assoc ::wrk/delay 1000))))
+                          processed?))))
+      (catch Throwable cause
+        (l/err :hint "error on cleaning file"
+               :file-id (str (:file-id props))
+               :cause cause)))))

backend/src/app/tasks/file_gc_scheduler.clj

@@ -17,29 +17,29 @@
 (def ^:private
   sql:get-candidates
   "SELECT f.id,
+          f.revn,
           f.modified_at
      FROM file AS f
     WHERE f.has_media_trimmed IS false
       AND f.modified_at < now() - ?::interval
       AND f.deleted_at IS NULL
     ORDER BY f.modified_at DESC
-      FOR UPDATE
+      FOR UPDATE OF f
      SKIP LOCKED")
 
 (defn- get-candidates
   [{:keys [::db/conn ::min-age] :as cfg}]
   (let [min-age (db/interval min-age)]
-    (db/cursor conn [sql:get-candidates min-age] {:chunk-size 10})))
+    (db/plan conn [sql:get-candidates min-age] {:fetch-size 10})))
 
 (defn- schedule!
-  [{:keys [::min-age] :as cfg}]
-  (let [total (reduce (fn [total {:keys [id]}]
-                        (let [params {:file-id id :min-age min-age}]
+  [cfg]
+  (let [total (reduce (fn [total {:keys [id modified-at revn]}]
+                        (let [params {:file-id id :modified-at modified-at :revn revn}]
                           (wrk/submit! (assoc cfg ::wrk/params params))
                           (inc total)))
                       0
                       (get-candidates cfg))]
-
     {:processed total}))
 
 (defmethod ig/assert-key ::handler
@@ -48,7 +48,7 @@
 
 (defmethod ig/expand-key ::handler
   [k v]
-  {k (assoc v ::min-age (cf/get-deletion-delay))})
+  {k (assoc v ::min-age (cf/get-file-clean-delay))})
 
 (defmethod ig/init-key ::handler
   [_ cfg]

backend/src/app/tasks/objects_gc.clj

@@ -11,6 +11,7 @@
    [app.common.logging :as l]
    [app.common.time :as ct]
    [app.db :as db]
+   [app.features.fdata :as fdata]
    [app.storage :as sto]
    [integrant.core :as ig]))
 
@@ -123,17 +124,19 @@
                0)))
 
 (def ^:private sql:get-files
-  "SELECT id, deleted_at, project_id, data_backend, data_ref_id
-     FROM file
-    WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
-    ORDER BY deleted_at ASC
+  "SELECT f.id,
+          f.deleted_at,
+          f.project_id
+     FROM file AS f
+    WHERE f.deleted_at IS NOT NULL
+      AND f.deleted_at < now() + ?::interval
+    ORDER BY f.deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-files!
-  [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
   (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id deleted-at project-id] :as file}]
                  (l/trc :hint "permanently delete"
@@ -142,8 +145,8 @@
                         :project-id (str project-id)
                         :deleted-at (ct/format-inst deleted-at))
 
-                 (when (= "objects-storage" (:data-backend file))
-                   (sto/touch-object! storage (:data-ref-id file)))
+                 ;; Delete associated file data
+                 (fdata/delete! cfg {:file-id id :id id :type "main"})
 
                  ;; And finally, permanently delete the file.
                  (db/delete! conn :file {:id id})
@@ -209,32 +212,6 @@
                  (inc total))
                0)))
 
-(def ^:private sql:get-file-data-fragments
-  "SELECT file_id, id, deleted_at, data_ref_id
-     FROM file_data_fragment
-    WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
-    ORDER BY deleted_at ASC
-    LIMIT ?
-      FOR UPDATE
-     SKIP LOCKED")
-
-(defn- delete-file-data-fragments!
-  [{:keys [::db/conn ::sto/storage ::deletion-threshold ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
-       (reduce (fn [total {:keys [file-id id deleted-at data-ref-id]}]
-                 (l/trc :hint "permanently delete"
-                        :rel "file-data-fragment"
-                        :id (str id)
-                        :file-id (str file-id)
-                        :deleted-at (ct/format-inst deleted-at))
-
-                 (some->> data-ref-id (sto/touch-object! storage))
-                 (db/delete! conn :file-data-fragment {:file-id file-id :id id})
-
-                 (inc total))
-               0)))
-
 (def ^:private sql:get-file-media-objects
   "SELECT id, file_id, media_id, thumbnail_id, deleted_at
      FROM file_media_object
@@ -264,8 +241,35 @@
                  (inc total))
                0)))
 
+(def ^:private sql:get-file-data-fragments
+  "SELECT file_id, id, deleted_at
+     FROM file_data_fragment
+    WHERE deleted_at IS NOT NULL
+      AND deleted_at < now() + ?::interval
+    ORDER BY deleted_at ASC
+    LIMIT ?
+      FOR UPDATE
+     SKIP LOCKED")
+
+(defn- delete-file-data-fragments!
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  (->> (db/plan conn [sql:get-file-data-fragments deletion-threshold chunk-size] {:fetch-size 5})
+       (reduce (fn [total {:keys [file-id id deleted-at]}]
+                 (l/trc :hint "permanently delete"
+                        :rel "file-data-fragment"
+                        :id (str id)
+                        :file-id (str file-id)
+                        :deleted-at (ct/format-inst deleted-at))
+
+                 ;; Delete associated file data
+                 (fdata/delete! cfg {:file-id file-id :id id :type "fragment"})
+                 (db/delete! conn :file-data-fragment {:file-id file-id :id id})
+
+                 (inc total))
+               0)))
+
 (def ^:private sql:get-file-change
-  "SELECT id, file_id, deleted_at, data_backend, data_ref_id
+  "SELECT id, file_id, deleted_at
      FROM file_change
     WHERE deleted_at IS NOT NULL
       AND deleted_at < now() + ?::interval
@@ -275,7 +279,7 @@
      SKIP LOCKED")
 
 (defn- delete-file-changes!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
   (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
                  (l/trc :hint "permanently delete"
@@ -284,8 +288,8 @@
                         :file-id (str file-id)
                         :deleted-at (ct/format-inst deleted-at))
 
-                 (when (= "objects-storage" (:data-backend xlog))
-                   (sto/touch-object! storage (:data-ref-id xlog)))
+                 ;; Delete associated file data, if it exists
+                 (fdata/delete! cfg {:file-id file-id :id id :type "snapshot"})
 
                  (db/delete! conn :file-change {:id id})
 
@@ -295,10 +299,10 @@
 (def ^:private deletion-proc-vars
   [#'delete-profiles!
    #'delete-file-media-objects!
-   #'delete-file-data-fragments!
    #'delete-file-object-thumbnails!
    #'delete-file-thumbnails!
    #'delete-file-changes!
+   #'delete-file-data-fragments!
    #'delete-files!
    #'delete-projects!
    #'delete-fonts!
@@ -313,7 +317,7 @@
                                    (db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
                                    (proc-fn cfg)))]
       (if (pos? result)
-        (recur (long (+ total result)))
+        (recur (+ total result))
         total))))
 
 (defmethod ig/assert-key ::handler
@@ -335,7 +339,7 @@
         (if-let [proc-fn (first procs)]
           (let [result (execute-proc! cfg proc-fn)]
             (recur (rest procs)
-                   (long (+ total result))))
+                   (+ total result)))
           (do
             (l/inf :hint "task finished" :deleted total)
             {:processed total}))))))

backend/src/app/tasks/offload_file_data.clj

@@ -8,101 +8,73 @@
   "A maintenance task responsible of moving file data from hot
   storage (the database row) to a cold storage (fs or s3)."
   (:require
-   [app.common.exceptions :as ex]
+   [app.binfile.common :as bfc]
    [app.common.logging :as l]
    [app.db :as db]
-   [app.db.sql :as-alias sql]
+   [app.features.fdata :as fdata]
+   [app.features.file-snapshots :as fsnap]
    [app.storage :as sto]
+   [app.util.blob :as blob]
    [integrant.core :as ig]))
 
-(defn- offload-file-data!
-  [{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
-  (let [file (db/get conn :file {:id file-id}
-                     {::sql/for-update true})]
-    (when (nil? (:data file))
-      (ex/raise :hint "file already offloaded"
-                :type :internal
-                :code :file-already-offloaded
-                :file-id file-id))
+(defn- offload-file-data
+  [{:keys [::db/conn ::file-id] :as cfg}]
+  (let [file (bfc/get-file cfg file-id :realize? true :lock-for-update? true)]
+    (cond
+      (not= "db" (:backend file))
+      (l/wrn :hint (str "skiping file offload (file offloaded or incompatible with offloading) for " file-id)
+             :file-id (str file-id))
 
-    (let [data (sto/content (:data file))
-          sobj (sto/put-object! storage
-                                {::sto/content data
-                                 ::sto/touch true
-                                 :bucket "file-data"
-                                 :content-type "application/octet-stream"
-                                 :file-id file-id})]
+      (nil? (:data file))
+      (l/err :hint (str "skiping file offload (missing data) for " file-id)
+             :file-id (str file-id))
 
-      (l/trc :hint "offload file data"
-             :file-id (str file-id)
-             :storage-id (str (:id sobj)))
+      :else
+      (do
+        (fdata/update! cfg {:id file-id
+                            :file-id file-id
+                            :type "main"
+                            :backend "storage"
+                            :data (blob/encode (:data file))})
 
-      (db/update! conn :file
-                  {:data-backend "objects-storage"
-                   :data-ref-id (:id sobj)
-                   :data nil}
-                  {:id file-id}
-                  {::db/return-keys false}))))
+        (db/update! conn :file
+                    {:data nil}
+                    {:id file-id}
+                    {::db/return-keys false})
 
-(defn- offload-file-data-fragments!
-  [{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
-  (doseq [fragment (db/query conn :file-data-fragment
-                             {:file-id file-id
-                              :deleted-at nil
-                              :data-backend nil}
-                             {::db/for-update true})]
-    (let [data (sto/content (:data fragment))
-          sobj (sto/put-object! storage
-                                {::sto/content data
-                                 ::sto/touch true
-                                 :bucket "file-data-fragment"
-                                 :content-type "application/octet-stream"
-                                 :file-id file-id
-                                 :file-fragment-id (:id fragment)})]
-
-      (l/trc :hint "offload file data fragment"
-             :file-id (str file-id)
-             :file-fragment-id (str (:id fragment))
-             :storage-id (str (:id sobj)))
-
-      (db/update! conn :file-data-fragment
-                  {:data-backend "objects-storage"
-                   :data-ref-id (:id sobj)
-                   :data nil}
-                  {:id (:id fragment)}
-                  {::db/return-keys false}))))
+        (l/trc :hint "offload file data"
+               :file-id (str file-id))))))
 
 (def sql:get-snapshots
-  "SELECT fc.*
-     FROM file_change AS fc
-    WHERE fc.file_id = ?
-      AND fc.label IS NOT NULL
-      AND fc.data IS NOT NULL
-      AND fc.data_backend IS NULL")
+  (str "WITH snapshots AS (" fsnap/sql:snapshots ")"
+       "SELECT s.*
+          FROM snapshots AS s
+         WHERE s.backend = 'db'
+           AND s.file_id = ?
+         ORDER BY s.created_at"))
 
-(defn- offload-file-snapshots!
-  [{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
-  (doseq [snapshot (db/exec! conn [sql:get-snapshots file-id])]
-    (let [data (sto/content (:data snapshot))
-          sobj (sto/put-object! storage
-                                {::sto/content data
-                                 ::sto/touch true
-                                 :bucket "file-change"
-                                 :content-type "application/octet-stream"
-                                 :file-id file-id
-                                 :file-change-id (:id snapshot)})]
-
-      (l/trc :hint "offload file change"
+(defn- offload-snapshot-data
+  [{:keys [::db/conn ::file-id] :as cfg} snapshot]
+  (let [{:keys [id data] :as snapshot} (fdata/resolve-file-data cfg snapshot)]
+    (if (nil? (:data snapshot))
+      (l/err :hint (str "skiping snapshot offload (missing data) for " file-id)
              :file-id (str file-id)
-             :file-change-id (str (:id snapshot))
-             :storage-id (str (:id sobj)))
+             :snapshot-id id)
+      (do
+        (fsnap/create! cfg {:id id
+                            :file-id file-id
+                            :type "snapshot"
+                            :backend "storage"
+                            :data data})
 
-      (db/update! conn :file-change
-                  {:data-backend "objects-storage"
-                   :data-ref-id (:id sobj)
-                   :data nil}
-                  {:id (:id snapshot)}
-                  {::db/return-keys false}))))
+        (l/trc :hint "offload snapshot data"
+               :file-id (str file-id)
+               :snapshot-id (str id))
+
+        (db/update! conn :file-change
+                    {:data nil}
+                    {:id id :file-id file-id}
+                    {::db/return-keys false})))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HANDLER
@@ -116,10 +88,12 @@
 (defmethod ig/init-key ::handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (-> cfg
-        (assoc ::db/rollback (:rollback? props))
-        (assoc ::file-id (:file-id props))
-        (db/tx-run! (fn [cfg]
-                      (offload-file-data! cfg)
-                      (offload-file-data-fragments! cfg)
-                      (offload-file-snapshots! cfg))))))
+    (let [file-id (:file-id props)]
+      (-> cfg
+          (assoc ::db/rollback (:rollback? props))
+          (assoc ::file-id (:file-id props))
+          (db/tx-run! (fn [{:keys [::db/conn] :as cfg}]
+                        (offload-file-data cfg)
+
+                        (run! (partial offload-snapshot-data cfg)
+                              (db/plan conn [sql:get-snapshots file-id]))))))))

backend/src/app/util/events.clj

@@ -27,7 +27,7 @@
      (sp/put! channel [type data])
      nil)))
 
-(defn spawn-listener
+(defn start-listener
   [channel on-event on-close]
   (assert (sp/chan? channel) "expected active events channel")
 
@@ -51,7 +51,7 @@
   [f on-event]
 
   (binding [*channel* (sp/chan :buf 32)]
-    (let [listener (spawn-listener *channel* on-event (constantly nil))]
+    (let [listener (start-listener *channel* on-event (constantly nil))]
       (try
         (f)
         (finally

backend/src/app/worker/dispatcher.clj

@@ -112,7 +112,7 @@
 
     (if (db/read-only? pool)
       (l/wrn :hint "not started (db is read-only)")
-      (px/fn->thread dispatcher :name "penpot/worker-dispatcher"))))
+      (px/fn->thread dispatcher :name "penpot/worker/dispatcher" :virtual false))))
 
 (defmethod ig/halt-key! ::wrk/dispatcher
   [_ thread]

backend/src/app/worker/executor.clj

@@ -7,79 +7,97 @@
 (ns app.worker.executor
   "Async tasks abstraction (impl)."
   (:require
+   [app.common.data :as d]
    [app.common.logging :as l]
-   [app.common.math :as mth]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.metrics :as mtx]
    [app.worker :as-alias wrk]
    [integrant.core :as ig]
    [promesa.exec :as px])
   (:import
-   io.netty.channel.nio.NioEventLoopGroup
-   io.netty.util.concurrent.DefaultEventExecutorGroup
-   java.util.concurrent.ExecutorService
-   java.util.concurrent.ThreadFactory))
+   java.util.concurrent.ThreadPoolExecutor))
 
 (set! *warn-on-reflection* true)
 
 (sm/register!
  {:type ::wrk/executor
-  :pred #(instance? ExecutorService %)
+  :pred #(instance? ThreadPoolExecutor %)
   :type-properties
   {:title "executor"
-   :description "Instance of ExecutorService"}})
-
-(sm/register!
- {:type ::wrk/netty-io-executor
-  :pred #(instance? NioEventLoopGroup %)
-  :type-properties
-  {:title "executor"
-   :description "Instance of NioEventLoopGroup"}})
-
-(sm/register!
- {:type ::wrk/netty-executor
-  :pred #(instance? DefaultEventExecutorGroup %)
-  :type-properties
-  {:title "executor"
-   :description "Instance of DefaultEventExecutorGroup"}})
+   :description "Instance of ThreadPoolExecutor"}})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; IO Executor
+;; EXECUTOR
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defmethod ig/assert-key ::wrk/netty-io-executor
-  [_ {:keys [threads]}]
-  (assert (or (nil? threads) (int? threads))
-          "expected valid threads value, revisit PENPOT_NETTY_IO_THREADS environment variable"))
+(defmethod ig/init-key ::wrk/executor
+  [_ _]
+  (let [factory  (px/thread-factory :prefix "penpot/default/")
+        executor (px/cached-executor :factory factory :keepalive 60000)]
+    (l/inf :hint "executor started")
+    executor))
 
-(defmethod ig/init-key ::wrk/netty-io-executor
-  [_ {:keys [threads]}]
-  (let [factory  (px/thread-factory :prefix "penpot/netty-io/")
-        nthreads (or threads (mth/round (/ (px/get-available-processors) 2)))
-        nthreads (max 2 nthreads)]
-    (l/inf :hint "start netty io executor" :threads nthreads)
-    (NioEventLoopGroup. (int nthreads) ^ThreadFactory factory)))
-
-(defmethod ig/halt-key! ::wrk/netty-io-executor
+(defmethod ig/halt-key! ::wrk/executor
   [_ instance]
   (px/shutdown! instance))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; IO Offload Executor
+;; MONITOR
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defmethod ig/assert-key ::wrk/netty-executor
-  [_ {:keys [threads]}]
-  (assert (or (nil? threads) (int? threads))
-          "expected valid threads value, revisit PENPOT_EXEC_THREADS environment variable"))
+(defn- get-stats
+  [^ThreadPoolExecutor executor]
+  {:active (.getPoolSize ^ThreadPoolExecutor executor)
+   :running (.getActiveCount ^ThreadPoolExecutor executor)
+   :completed (.getCompletedTaskCount ^ThreadPoolExecutor executor)})
 
-(defmethod ig/init-key ::wrk/netty-executor
-  [_ {:keys [threads]}]
-  (let [factory  (px/thread-factory :prefix "penpot/exec/")
-        nthreads (or threads (mth/round (/ (px/get-available-processors) 2)))
-        nthreads (max 2 nthreads)]
-    (l/inf :hint "start default executor" :threads nthreads)
-    (DefaultEventExecutorGroup. (int nthreads) ^ThreadFactory factory)))
+(defmethod ig/expand-key ::wrk/monitor
+  [k v]
+  {k (-> (d/without-nils v)
+         (assoc ::interval (ct/duration "2s")))})
 
-(defmethod ig/halt-key! ::wrk/netty-executor
-  [_ instance]
-  (px/shutdown! instance))
+(defmethod ig/init-key ::wrk/monitor
+  [_ {:keys [::wrk/executor ::mtx/metrics ::interval ::wrk/name]}]
+  (letfn [(monitor! [executor prev-completed]
+            (let [labels        (into-array String [(d/name name)])
+                  stats         (get-stats executor)
+
+                  completed     (:completed stats)
+                  completed-inc (- completed prev-completed)
+                  completed-inc (if (neg? completed-inc) 0 completed-inc)]
+
+              (mtx/run! metrics
+                        :id :executor-active-threads
+                        :labels labels
+                        :val (:active stats))
+
+              (mtx/run! metrics
+                        :id :executor-running-threads
+                        :labels labels
+                        :val (:running stats))
+
+              (mtx/run! metrics
+                        :id :executors-completed-tasks
+                        :labels labels
+                        :inc completed-inc)
+
+              completed-inc))]
+
+    (px/thread
+      {:name "penpot/executors-monitor" :virtual true}
+      (l/inf :hint "monitor started" :name name)
+      (try
+        (loop [completed 0]
+          (px/sleep interval)
+          (recur (long (monitor! executor completed))))
+        (catch InterruptedException _cause
+          (l/trc :hint "monitor: interrupted" :name name))
+        (catch Throwable cause
+          (l/err :hint "monitor: unexpected error" :name name :cause cause))
+        (finally
+          (l/inf :hint "monitor: terminated" :name name))))))
+
+(defmethod ig/halt-key! ::wrk/monitor
+  [_ thread]
+  (px/interrupt! thread))

backend/src/app/worker/runner.clj

@@ -248,7 +248,7 @@
 (defn- start-thread!
   [{:keys [::rds/redis ::id ::queue ::wrk/tenant] :as cfg}]
   (px/thread
-    {:name (str "penpot/worker-runner/" id)}
+    {:name (format "penpot/worker/runner:%s" id)}
     (l/inf :hint "started" :id id :queue queue)
     (try
       (dm/with-open [rconn (rds/connect redis)]
@@ -303,7 +303,7 @@
       (l/wrn :hint "not started (db is read-only)" :queue queue :parallelism parallelism)
       (doall
        (->> (range parallelism)
-            (map #(assoc cfg ::id (str queue "/" %)))
+            (map #(assoc cfg ::id %))
             (map start-thread!))))))
 
 (defmethod ig/halt-key! ::wrk/runner

backend/test/backend_tests/helpers.clj

@@ -113,6 +113,7 @@
                              :app.auth.oidc.providers/generic
                              :app.setup/templates
                              :app.auth.oidc/routes
+                             :app.worker/monitor
                              :app.http.oauth/handler
                              :app.notifications/handler
                              :app.loggers.mattermost/reporter

backend/test/backend_tests/http_management_test.clj

@@ -1,96 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns backend-tests.http-management-test
-  (:require
-   [app.common.data :as d]
-   [app.common.time :as ct]
-   [app.db :as db]
-   [app.http.access-token]
-   [app.http.management :as mgmt]
-   [app.http.session :as sess]
-   [app.main :as-alias main]
-   [app.rpc :as-alias rpc]
-   [backend-tests.helpers :as th]
-   [clojure.test :as t]
-   [mockery.core :refer [with-mocks]]
-   [yetti.response :as-alias yres]))
-
-(t/use-fixtures :once th/state-init)
-(t/use-fixtures :each th/database-reset)
-
-
-(t/deftest authenticate-method
-  (let [profile  (th/create-profile* 1)
-        props    (get th/*system* :app.setup/props)
-        token    (#'sess/gen-token props {:profile-id (:id profile)})
-        request  {:params {:token token}}
-        response (#'mgmt/authenticate th/*system* request)]
-
-    (t/is (= 200 (::yres/status response)))
-    (t/is (= "authentication" (-> response ::yres/body :iss)))
-    (t/is (= (:id profile) (-> response ::yres/body :uid)))))
-
-(t/deftest get-customer-method
-  (let [profile  (th/create-profile* 1)
-        request  {:params {:id (:id profile)}}
-        response (#'mgmt/get-customer th/*system* request)]
-
-    (t/is (= 200 (::yres/status response)))
-    (t/is (= (:id profile) (-> response ::yres/body :id)))
-    (t/is (= (:fullname profile) (-> response ::yres/body :name)))
-    (t/is (= (:email profile) (-> response ::yres/body :email)))
-    (t/is (= 1 (-> response ::yres/body :num-editors)))
-    (t/is (nil? (-> response ::yres/body :subscription)))))
-
-(t/deftest update-customer-method
-  (let [profile  (th/create-profile* 1)
-
-        subs     {:type "unlimited"
-                  :description nil
-                  :id "foobar"
-                  :customer-id (str (:id profile))
-                  :status "past_due"
-                  :billing-period "week"
-                  :quantity 1
-                  :created-at (ct/truncate (ct/now) :day)
-                  :cancel-at-period-end true
-                  :start-date nil
-                  :ended-at nil
-                  :trial-end nil
-                  :trial-start nil
-                  :cancel-at nil
-                  :canceled-at nil
-                  :current-period-end nil
-                  :current-period-start nil
-
-                  :cancellation-details
-                  {:comment "other"
-                   :reason "other"
-                   :feedback "other"}}
-
-        request  {:params {:id (:id profile)
-                           :subscription subs}}
-        response (#'mgmt/update-customer th/*system* request)]
-
-    (t/is (= 201 (::yres/status response)))
-    (t/is (nil? (::yres/body response)))
-
-    (let [request  {:params {:id (:id profile)}}
-          response (#'mgmt/get-customer th/*system* request)]
-
-      (t/is (= 200 (::yres/status response)))
-      (t/is (= (:id profile) (-> response ::yres/body :id)))
-      (t/is (= (:fullname profile) (-> response ::yres/body :name)))
-      (t/is (= (:email profile) (-> response ::yres/body :email)))
-      (t/is (= 1 (-> response ::yres/body :num-editors)))
-
-      (let [subs' (-> response ::yres/body :subscription)]
-        (t/is (= subs' subs))))))
-
-
-
-

backend/test/backend_tests/rpc_doc_test.clj

@@ -1,36 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns backend-tests.rpc-doc-test
-  "Internal binfile test, no RPC involved"
-  (:require
-   [app.common.json :as json]
-   [app.common.pprint :as pp]
-   [app.common.schema :as sm]
-   [app.common.schema.generators :as sg]
-   [app.common.schema.test :as smt]
-   [app.rpc :as-alias rpc]
-   [app.rpc.doc :as rpc.doc]
-   [backend-tests.helpers :as th]
-   [clojure.test :as t]))
-
-(t/use-fixtures :once th/state-init)
-
-(t/deftest openapi-context-json-encode
-  (smt/check!
-   (smt/for [context (->> sg/int
-                          (sg/fmap (fn [_]
-                                     (rpc.doc/prepare-openapi-context (::rpc/methods th/*system*)))))]
-     (try
-       (json/encode context)
-       true
-       (catch Throwable _cause
-         false)))
-   {:num 30}))
-
-
-
-

backend/test/backend_tests/util_objects_map_test.clj

@@ -86,7 +86,7 @@
 
 (t/deftest internal-encode-decode
   (smt/check!
-   (smt/for [data (->> (cg/map cg/uuid (sg/generator cts/schema:shape))
+   (smt/for [data (->> (cg/map cg/uuid (sg/generator ::cts/shape))
                        (cg/not-empty))]
      (let [obj1 (omap/wrap data)
            obj2 (omap/create (deref obj1))
@@ -103,7 +103,7 @@
 
 (t/deftest fressian-encode-decode
   (smt/check!
-   (smt/for [data (->> (cg/map cg/uuid (sg/generator cts/schema:shape))
+   (smt/for [data (->> (cg/map cg/uuid (sg/generator ::cts/shape))
                        (cg/not-empty)
                        (cg/fmap omap/wrap)
                        (cg/fmap (fn [o] {:objects o})))]
@@ -119,7 +119,7 @@
 
 (t/deftest transit-encode-decode
   (smt/check!
-   (smt/for [data (->> (cg/map cg/uuid (sg/generator cts/schema:shape))
+   (smt/for [data (->> (cg/map cg/uuid (sg/generator ::cts/shape))
                        (cg/not-empty)
                        (cg/fmap omap/wrap)
                        (cg/fmap (fn [o] {:objects o})))]

common/deps.edn

@@ -1,5 +1,5 @@
 {:deps
- {org.clojure/clojure {:mvn/version "1.12.2"}
+ {org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/data.json {:mvn/version "2.5.1"}
   org.clojure/tools.cli {:mvn/version "1.1.230"}
   org.clojure/test.check {:mvn/version "1.1.1"}
@@ -7,25 +7,25 @@
   org.clojure/clojurescript {:mvn/version "1.12.42"}
 
   ;; Logging
-  org.apache.logging.log4j/log4j-api {:mvn/version "2.25.1"}
-  org.apache.logging.log4j/log4j-core {:mvn/version "2.25.1"}
-  org.apache.logging.log4j/log4j-web {:mvn/version "2.25.1"}
-  org.apache.logging.log4j/log4j-jul {:mvn/version "2.25.1"}
-  org.apache.logging.log4j/log4j-slf4j2-impl {:mvn/version "2.25.1"}
+  org.apache.logging.log4j/log4j-api {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-core {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-web {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-jul {:mvn/version "2.24.3"}
+  org.apache.logging.log4j/log4j-slf4j2-impl {:mvn/version "2.24.3"}
   org.slf4j/slf4j-api {:mvn/version "2.0.17"}
-  pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.40"}
+  pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.32"}
 
   selmer/selmer {:mvn/version "1.12.62"}
   criterium/criterium {:mvn/version "0.4.6"}
 
   metosin/jsonista {:mvn/version "0.3.13"}
-  metosin/malli {:mvn/version "0.19.1"}
+  metosin/malli {:mvn/version "0.18.0"}
 
   expound/expound {:mvn/version "0.9.0"}
   com.cognitect/transit-clj {:mvn/version "1.0.333"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
-  integrant/integrant {:mvn/version "1.0.0"}
+  integrant/integrant {:mvn/version "0.13.1"}
 
   funcool/tubax {:mvn/version "2021.05.20-0"}
   funcool/cuerdas {:mvn/version "2025.06.16-414"}
@@ -43,11 +43,11 @@
 
   frankiesardo/linked {:mvn/version "1.3.0"}
 
-  com.sun.mail/jakarta.mail {:mvn/version "2.0.2"}
+  com.sun.mail/jakarta.mail {:mvn/version "2.0.1"}
   org.la4j/la4j {:mvn/version "0.6.0"}
 
   ;; exception printing
-  fipp/fipp {:mvn/version "0.6.29"}
+  fipp/fipp {:mvn/version "0.6.27"}
 
   me.flowthing/pp {:mvn/version "2024-11-13.77"}
 
@@ -59,7 +59,7 @@
  {:dev
   {:extra-deps
    {org.clojure/tools.namespace {:mvn/version "RELEASE"}
-    thheller/shadow-cljs {:mvn/version "3.2.0"}
+    thheller/shadow-cljs {:mvn/version "3.1.5"}
     com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
     com.bhauman/rebel-readline {:mvn/version "RELEASE"}
     criterium/criterium {:mvn/version "RELEASE"}
@@ -68,7 +68,7 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:mvn/version "0.10.10"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test

common/src/app/common/buffer.cljc

@@ -5,8 +5,7 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.common.buffer
-  "A collection of helpers and macros for work with byte
-  buffer (ByteBuffer on JVM and DataView on JS)."
+  "A collection of helpers and macros for work with byte buffers"
   (:refer-clojure :exclude [clone])
   (:require
    [app.common.uuid :as uuid])
@@ -20,49 +19,42 @@
   (if (:ns &env)
     `(.getInt8 ~target ~offset true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(long (.get ~target (unchecked-int ~offset))))))
+      `(long (.get ~target ~offset)))))
 
 (defmacro read-unsigned-byte
   [target offset]
   (if (:ns &env)
     `(.getUint8 ~target ~offset true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(bit-and (long (.get ~target (unchecked-int ~offset))) 0xff))))
+      `(bit-and (long (.get ~target ~offset)) 0xff))))
 
 (defmacro read-bool
   [target offset]
   (if (:ns &env)
     `(== 1 (.getInt8 ~target ~offset true))
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(== 1 (.get ~target (unchecked-int ~offset))))))
+      `(== 1 (.get ~target ~offset)))))
 
 (defmacro read-short
   [target offset]
   (if (:ns &env)
     `(.getInt16 ~target ~offset true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.getShort ~target (unchecked-int ~offset)))))
+      `(.getShort ~target ~offset))))
 
 (defmacro read-int
   [target offset]
   (if (:ns &env)
     `(.getInt32 ~target ~offset true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(long (.getInt ~target (unchecked-int ~offset))))))
-
-(defmacro read-long
-  [target offset]
-  (if (:ns &env)
-    `(.getInt64 ~target ~offset true)
-    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.getLong ~target (unchecked-int ~offset)))))
+      `(long (.getInt ~target ~offset)))))
 
 (defmacro read-float
   [target offset]
   (if (:ns &env)
     `(.getFloat32 ~target ~offset true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(double (.getFloat ~target (unchecked-int ~offset))))))
+      `(double (.getFloat ~target ~offset)))))
 
 (defmacro read-uuid
   [target offset]
@@ -76,57 +68,16 @@
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
       `(try
          (.order ~target ByteOrder/BIG_ENDIAN)
-         (let [msb# (.getLong ~target (unchecked-int (+ ~offset 0)))
-               lsb# (.getLong ~target (unchecked-int (+ ~offset 8)))]
+         (let [msb# (.getLong ~target (+ ~offset 0))
+               lsb# (.getLong ~target (+ ~offset 8))]
            (java.util.UUID. (long msb#) (long lsb#)))
          (finally
            (.order ~target ByteOrder/LITTLE_ENDIAN))))))
 
-(defmacro read-bytes
-  "Get a byte array from buffer. It is potentially unsafe because on
-  JS/CLJS it returns a subarray without doing any copy of data."
-  [target offset size]
-  (if (:ns &env)
-    `(new js/Uint8Array
-          (.-buffer ~target)
-          (+ (.-byteOffset ~target) ~offset)
-          ~size)
-    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
-          bbuf   (with-meta (gensym "bbuf") {:tag bytes})]
-      `(let [~bbuf (byte-array ~size)]
-         (.get ~target
-               (unchecked-int ~offset)
-               ~bbuf
-               0
-               ~size)
-         ~bbuf))))
-
-;; FIXME: implement in cljs
-(defmacro write-bytes
-  ([target offset src size]
-   `(write-bytes ~target ~offset ~src 0 ~size))
-  ([target offset src src-offset size]
-   (if (:ns &env)
-     (throw (ex-info "not implemented" {}))
-     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
-           src    (with-meta src {:tag 'bytes})]
-       `(.put ~target
-              (unchecked-int ~offset)
-              ~src
-              (unchecked-int ~src-offset)
-              (unchecked-int ~size))))))
-
 (defmacro write-byte
   [target offset value]
   (if (:ns &env)
     `(.setInt8 ~target ~offset ~value true)
-    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.put ~target (unchecked-int ~offset) (unchecked-byte ~value)))))
-
-(defmacro write-u8
-  [target offset value]
-  (if (:ns &env)
-    `(.setUint8 ~target ~offset ~value true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
       `(.put ~target ~offset (unchecked-byte ~value)))))
 
@@ -135,45 +86,28 @@
   (if (:ns &env)
     `(.setInt8 ~target ~offset (if ~value 0x01 0x00) true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.put ~target (unchecked-int ~offset) (unchecked-byte (if ~value 0x01 0x00))))))
+      `(.put ~target ~offset (unchecked-byte (if ~value 0x01 0x00))))))
 
 (defmacro write-short
   [target offset value]
   (if (:ns &env)
     `(.setInt16 ~target ~offset ~value true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.putShort ~target (unchecked-int ~offset) (unchecked-short ~value)))))
+      `(.putShort ~target ~offset (unchecked-short ~value)))))
 
 (defmacro write-int
   [target offset value]
   (if (:ns &env)
     `(.setInt32 ~target ~offset ~value true)
-    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.putInt ~target (unchecked-int ~offset) (unchecked-int ~value)))))
-
-(defmacro write-u32
-  [target offset value]
-  (if (:ns &env)
-    `(.setUint32 ~target ~offset ~value true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
       `(.putInt ~target ~offset (unchecked-int ~value)))))
 
-(defmacro write-i32
-  "Idiomatic alias for `write-int`"
-  [target offset value]
-  `(write-int ~target ~offset ~value))
-
 (defmacro write-float
   [target offset value]
   (if (:ns &env)
     `(.setFloat32 ~target ~offset ~value true)
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
-      `(.putFloat ~target (unchecked-int ~offset) (unchecked-float ~value)))))
-
-(defmacro write-f32
-  "Idiomatic alias for `write-float`."
-  [target offset value]
-  `(write-float ~target ~offset ~value))
+      `(.putFloat ~target ~offset (unchecked-float ~value)))))
 
 (defmacro write-uuid
   [target offset value]
@@ -185,15 +119,13 @@
        (.setUint32 ~target (+ ~offset 12) (aget barray# 3) true))
 
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
-          value  (with-meta value {:tag 'java.util.UUID})
-          prev   (with-meta (gensym "prev-") {:tag 'java.nio.ByteOrder})]
-      `(let [~prev (.order ~target)]
-         (try
-           (.order ~target ByteOrder/BIG_ENDIAN)
-           (.putLong ~target (unchecked-int (+ ~offset 0)) (.getMostSignificantBits ~value))
-           (.putLong ~target (unchecked-int (+ ~offset 8)) (.getLeastSignificantBits ~value))
-           (finally
-             (.order ~target ~prev)))))))
+          value  (with-meta value {:tag 'java.util.UUID})]
+      `(try
+         (.order ~target ByteOrder/BIG_ENDIAN)
+         (.putLong ~target (+ ~offset 0) (.getMostSignificantBits ~value))
+         (.putLong ~target (+ ~offset 8) (.getLeastSignificantBits ~value))
+         (finally
+           (.order ~target ByteOrder/LITTLE_ENDIAN))))))
 
 (defn wrap
   [data]
@@ -203,7 +135,7 @@
 
 (defn allocate
   [size]
-  #?(:clj (let [buffer (ByteBuffer/allocate (unchecked-int size))]
+  #?(:clj (let [buffer (ByteBuffer/allocate (int size))]
             (.order buffer ByteOrder/LITTLE_ENDIAN))
      :cljs (new js/DataView (new js/ArrayBuffer size))))
 
@@ -224,14 +156,6 @@
        (.set dst-view src-view)
        (js/DataView. dst-buff))))
 
-;; FIXME: cljs impl
-#?(:clj
-   (defn copy-bytes
-     [src src-offset size dst dst-offset]
-     (let [tmp (byte-array size)]
-       (.get ^ByteBuffer src src-offset tmp 0 size)
-       (.put ^ByteBuffer dst dst-offset tmp 0 size))))
-
 (defn equals?
   [buffer-a buffer-b]
   #?(:clj
@@ -259,18 +183,3 @@
   [o]
   #?(:clj (instance? ByteBuffer o)
      :cljs (instance? js/DataView o)))
-
-(defn slice
-  [buffer offset size]
-  #?(:cljs
-     (let [offset (+ (.-byteOffset buffer) offset)]
-       (new js/DataView (.-buffer buffer) offset size))
-
-     :clj
-     (-> (.slice ^ByteBuffer buffer (unchecked-int offset) (unchecked-int size))
-         (.order ByteOrder/LITTLE_ENDIAN))))
-
-(defn size
-  [o]
-  #?(:cljs (.-byteLength ^js o)
-     :clj  (.capacity ^ByteBuffer o)))

common/src/app/common/features.cljc

@@ -51,7 +51,6 @@
     "styles/v2"
     "layout/grid"
     "plugins/runtime"
-    "tokens/numeric-input"
     "design-tokens/v1"
     "text-editor/v2"
     "render-wasm/v1"
@@ -65,8 +64,12 @@
     "layout/grid"
     "components/v2"
     "plugins/runtime"
-    "design-tokens/v1"
-    "variants/v1"})
+    "design-tokens/v1"})
+
+;; A set of features that should not be propagated to team on creating
+;; or modifying a file
+(def no-team-inheritable-features
+  #{"fdata/path-data"})
 
 ;; A set of features which only affects on frontend and can be enabled
 ;; and disabled freely by the user any time. This features does not
@@ -76,20 +79,13 @@
   #{"styles/v2"
     "plugins/runtime"
     "text-editor/v2"
-    "tokens/numeric-input"
     "render-wasm/v1"})
 
 ;; Features that are mainly backend only or there are a proper
 ;; fallback when frontend reports no support for it
 (def backend-only-features
-  #{"fdata/pointer-map"
-    "fdata/objects-map"})
-
-;; A set of features that should not be propagated to team on creating
-;; or modifying a file or creating or modifying a team
-(def no-team-inheritable-features
-  #{"fdata/path-data"
-    "fdata/shape-data-type"})
+  #{"fdata/objects-map"
+    "fdata/pointer-map"})
 
 ;; This is a set of features that does not require an explicit
 ;; migration like components/v2 or the migration is not mandatory to
@@ -99,19 +95,18 @@
   (-> #{"layout/grid"
         "design-tokens/v1"
         "fdata/shape-data-type"
-        "fdata/path-data"
-        "tokens/numeric-input"
-        "variants/v1"}
+        "fdata/path-data"}
       (into frontend-only-features)
       (into backend-only-features)))
 
-(sm/register!
- ^{::sm/type ::features}
- [:schema
-  {:title "FileFeatures"
-   ::smdj/inline true
-   :gen/gen (smg/subseq supported-features)}
-  [::sm/set :string]])
+(def schema:features
+  (sm/register!
+   ^{::sm/type ::features}
+   [:schema
+    {:title "FileFeatures"
+     ::smdj/inline true
+     :gen/gen (smg/subseq supported-features)}
+    [::sm/set :string]]))
 
 (defn- flag->feature
   "Translate a flag to a feature name"
@@ -125,7 +120,6 @@
     :feature-text-editor-v2 "text-editor/v2"
     :feature-render-wasm "render-wasm/v1"
     :feature-variants "variants/v1"
-    :feature-token-input "tokens/numeric-input"
     nil))
 
 (defn migrate-legacy-features
@@ -227,6 +221,8 @@
                 :hint (str/ffmt "enabled feature '%' not present in file (missing migration)"
                                 not-supported)))
 
+    (check-supported-features! file-features)
+
     ;; Components v1 is deprecated
     (when-not (contains? file-features "components/v2")
       (ex/raise :type :restriction

common/src/app/common/files/changes.cljc

@@ -83,25 +83,24 @@
 
     [:multi {:decode/json #(update % :grid-type keyword)
              :gen/gen gen
-             :title "SetDefaultGridChange"
              :dispatch :grid-type
              ::smd/simplified true}
      [:square
-      [:map {:title "SetDefautSquareGridAttrs"}
+      [:map
        [:type [:= :set-default-grid]]
        [:page-id ::sm/uuid]
        [:grid-type [:= :square]]
        [:params [:maybe ctg/schema:square-params]]]]
 
      [:column
-      [:map {:title "SetDefaultColumnGridAttrs"}
+      [:map
        [:type [:= :set-default-grid]]
        [:page-id ::sm/uuid]
        [:grid-type [:= :column]]
        [:params [:maybe ctg/schema:column-params]]]]
 
      [:row
-      [:map {:title "SetDefaultRowGridAttrs"}
+      [:map
        [:type [:= :set-default-grid]]
        [:page-id ::sm/uuid]
        [:grid-type [:= :row]]
@@ -112,20 +111,20 @@
                 [:type [:= :set-guide]]
                 [:page-id ::sm/uuid]
                 [:id ::sm/uuid]
-                [:params [:maybe ctp/schema:guide]]]
+                [:params [:maybe ::ctp/guide]]]
         gen    (->> (sg/generator schema)
                     (sg/fmap (fn [change]
                                (if (some? (:params change))
                                  (update change :params assoc :id (:id change))
                                  change))))]
-    (sm/update-properties schema assoc :gen/gen gen)))
+    [:schema {:gen/gen gen} schema]))
 
 (def schema:set-flow-change
   (let [schema [:map {:title "SetFlowChange"}
                 [:type [:= :set-flow]]
                 [:page-id ::sm/uuid]
                 [:id ::sm/uuid]
-                [:params [:maybe ctp/schema:flow]]]
+                [:params [:maybe ::ctp/flow]]]
 
         gen    (->> (sg/generator schema)
                     (sg/fmap (fn [change]
@@ -133,7 +132,7 @@
                                  (update change :params assoc :id (:id change))
                                  change))))]
 
-    (sm/update-properties schema assoc :gen/gen gen)))
+    [:schema {:gen/gen gen} schema]))
 
 (def schema:set-plugin-data-change
   (let [types  #{:file :page :shape :color :typography :component}
@@ -170,265 +169,287 @@
 
                                  :else
                                  (dissoc change :page-id)))))]
-    [:and (sm/update-properties schema assoc :gen/gen gen) check1]))
+
+    [:and {:gen/gen gen} schema check1]))
 
 (def schema:change
-  [:multi {:dispatch :type
-           :title "Change"
-           :decode/json #(update % :type keyword)
-           ::smd/simplified true}
+  [:schema
+   [:multi {:dispatch :type
+            :title "Change"
+            :decode/json #(update % :type keyword)
+            ::smd/simplified true}
+    [:set-option
 
-   [:set-comment-thread-position
-    [:map {:title "SetCommentThreadPositionChange"}
-     [:comment-thread-id ::sm/uuid]
-     [:page-id ::sm/uuid]
-     [:frame-id [:maybe ::sm/uuid]]
-     [:position [:maybe ::gpt/point]]]]
+     ;; DEPRECATED: remove before 2.3 release
+     ;;
+     ;; Is still there for not cause error when event is received
+     [:map {:title "SetOptionChange"}]]
 
-   [:add-obj
-    [:map {:title "AddObjChange"}
-     [:type [:= :add-obj]]
-     [:id ::sm/uuid]
-     [:obj cts/schema:shape]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:frame-id ::sm/uuid]
-     [:parent-id {:optional true} [:maybe ::sm/uuid]]
-     [:index {:optional true} [:maybe :int]]
-     [:ignore-touched {:optional true} :boolean]]]
+    [:set-comment-thread-position
+     [:map
+      [:comment-thread-id ::sm/uuid]
+      [:page-id ::sm/uuid]
+      [:frame-id [:maybe ::sm/uuid]]
+      [:position [:maybe ::gpt/point]]]]
 
-   [:mod-obj
-    [:map {:title "ModObjChange"}
-     [:type [:= :mod-obj]]
-     [:id ::sm/uuid]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:operations [:vector {:gen/max 5} schema:operation]]]]
+    [:add-obj
+     [:map {:title "AddObjChange"}
+      [:type [:= :add-obj]]
+      [:id ::sm/uuid]
+      [:obj :map]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:frame-id ::sm/uuid]
+      [:parent-id {:optional true} [:maybe ::sm/uuid]]
+      [:index {:optional true} [:maybe :int]]
+      [:ignore-touched {:optional true} :boolean]]]
 
-   [:del-obj
-    [:map {:title "DelObjChange"}
-     [:type [:= :del-obj]]
-     [:id ::sm/uuid]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:ignore-touched {:optional true} :boolean]]]
+    [:mod-obj
+     [:map {:title "ModObjChange"}
+      [:type [:= :mod-obj]]
+      [:id ::sm/uuid]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:operations [:vector {:gen/max 5} schema:operation]]]]
 
-   [:set-guide schema:set-guide-change]
-   [:set-flow schema:set-flow-change]
-   [:set-default-grid schema:set-default-grid-change]
+    [:del-obj
+     [:map {:title "DelObjChange"}
+      [:type [:= :del-obj]]
+      [:id ::sm/uuid]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:ignore-touched {:optional true} :boolean]]]
 
-   [:fix-obj
-    [:map {:title "FixObjChange"}
-     [:type [:= :fix-obj]]
-     [:id ::sm/uuid]
-     [:fix {:optional true} :keyword]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]]]
+    [:set-guide schema:set-guide-change]
+    [:set-flow schema:set-flow-change]
+    [:set-default-grid schema:set-default-grid-change]
 
-   [:mov-objects
-    [:map {:title "MovObjectsChange"}
-     [:type [:= :mov-objects]]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:ignore-touched {:optional true} :boolean]
-     [:parent-id ::sm/uuid]
-     [:shapes ::sm/any]
-     [:index {:optional true} [:maybe :int]]
-     [:after-shape {:optional true} ::sm/any]
-     [:allow-altering-copies {:optional true} :boolean]]]
+    [:fix-obj
+     [:map {:title "FixObjChange"}
+      [:type [:= :fix-obj]]
+      [:id ::sm/uuid]
+      [:fix {:optional true} :keyword]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]]]
 
-   [:reorder-children
-    [:map {:title "ReorderChildrenChange"}
-     [:type [:= :reorder-children]]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:ignore-touched {:optional true} :boolean]
-     [:parent-id ::sm/uuid]
-     [:shapes ::sm/any]]]
+    [:mov-objects
+     [:map {:title "MovObjectsChange"}
+      [:type [:= :mov-objects]]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:ignore-touched {:optional true} :boolean]
+      [:parent-id ::sm/uuid]
+      [:shapes ::sm/any]
+      [:index {:optional true} [:maybe :int]]
+      [:after-shape {:optional true} ::sm/any]
+      [:allow-altering-copies {:optional true} :boolean]]]
 
-   [:add-page
-    [:map {:title "AddPageChange"}
-     [:type [:= :add-page]]
-     [:id {:optional true} ::sm/uuid]
-     [:name {:optional true} :string]
-     [:page {:optional true} ::sm/any]]]
+    [:reorder-children
+     [:map {:title "ReorderChildrenChange"}
+      [:type [:= :reorder-children]]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:ignore-touched {:optional true} :boolean]
+      [:parent-id ::sm/uuid]
+      [:shapes ::sm/any]]]
 
-   [:mod-page
-    [:map {:title "ModPageChange"}
-     [:type [:= :mod-page]]
-     [:id ::sm/uuid]
-     ;; All props are optional, background can be nil because is the
-     ;; way to remove already set background
-     [:background {:optional true} [:maybe ctc/schema:hex-color]]
-     [:name {:optional true} :string]]]
+    [:add-page
+     [:map {:title "AddPageChange"}
+      [:type [:= :add-page]]
+      [:id {:optional true} ::sm/uuid]
+      [:name {:optional true} :string]
+      [:page {:optional true} ::sm/any]]]
 
-   [:set-plugin-data schema:set-plugin-data-change]
+    [:mod-page
+     [:map {:title "ModPageChange"}
+      [:type [:= :mod-page]]
+      [:id ::sm/uuid]
+      ;; All props are optional, background can be nil because is the
+      ;; way to remove already set background
+      [:background {:optional true} [:maybe ctc/schema:hex-color]]
+      [:name {:optional true} :string]]]
 
-   [:del-page
-    [:map {:title "DelPageChange"}
-     [:type [:= :del-page]]
-     [:id ::sm/uuid]]]
+    [:set-plugin-data schema:set-plugin-data-change]
 
-   [:mov-page
-    [:map {:title "MovPageChange"}
-     [:type [:= :mov-page]]
-     [:id ::sm/uuid]
-     [:index :int]]]
+    [:del-page
+     [:map {:title "DelPageChange"}
+      [:type [:= :del-page]]
+      [:id ::sm/uuid]]]
 
-   [:reg-objects
-    [:map {:title "RegObjectsChange"}
-     [:type [:= :reg-objects]]
-     [:page-id {:optional true} ::sm/uuid]
-     [:component-id {:optional true} ::sm/uuid]
-     [:shapes [:vector {:gen/max 5} ::sm/uuid]]]]
+    [:mov-page
+     [:map {:title "MovPageChange"}
+      [:type [:= :mov-page]]
+      [:id ::sm/uuid]
+      [:index :int]]]
 
-   [:add-color
-    [:map {:title "AddColorChange"}
-     [:type [:= :add-color]]
-     [:color ctc/schema:library-color]]]
+    [:reg-objects
+     [:map {:title "RegObjectsChange"}
+      [:type [:= :reg-objects]]
+      [:page-id {:optional true} ::sm/uuid]
+      [:component-id {:optional true} ::sm/uuid]
+      [:shapes [:vector {:gen/max 5} ::sm/uuid]]]]
 
-   [:mod-color
-    [:map {:title "ModColorChange"}
-     [:type [:= :mod-color]]
-     [:color ctc/schema:library-color]]]
+    [:add-color
+     [:map {:title "AddColorChange"}
+      [:type [:= :add-color]]
+      [:color ctc/schema:library-color]]]
 
-   [:del-color
-    [:map {:title "DelColorChange"}
-     [:type [:= :del-color]]
-     [:id ::sm/uuid]]]
+    [:mod-color
+     [:map {:title "ModColorChange"}
+      [:type [:= :mod-color]]
+      [:color ctc/schema:library-color]]]
 
-   [:add-media
-    [:map {:title "AddMediaChange"}
-     [:type [:= :add-media]]
-     [:object ctf/schema:media]]]
+    [:del-color
+     [:map {:title "DelColorChange"}
+      [:type [:= :del-color]]
+      [:id ::sm/uuid]]]
 
-   [:mod-media
-    [:map {:title "ModMediaChange"}
-     [:type [:= :mod-media]]
-     [:object ctf/schema:media]]]
+    ;; DEPRECATED: remove before 2.3
+    [:add-recent-color
+     [:map {:title "AddRecentColorChange"}]]
 
-   [:del-media
-    [:map {:title "DelMediaChange"}
-     [:type [:= :del-media]]
-     [:id ::sm/uuid]]]
+    [:add-media
+     [:map {:title "AddMediaChange"}
+      [:type [:= :add-media]]
+      [:object ctf/schema:media]]]
 
-   [:add-component
-    [:map {:title "AddComponentChange"}
-     [:type [:= :add-component]]
-     [:id ::sm/uuid]
-     [:name :string]
-     [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
-     [:path {:optional true} :string]
-     [:main-instance-id ::sm/uuid]
-     [:main-instance-page ::sm/uuid]]]
+    [:mod-media
+     [:map {:title "ModMediaChange"}
+      [:type [:= :mod-media]]
+      [:object ctf/schema:media]]]
 
-   [:mod-component
-    [:map {:title "ModComponentChange"}
-     [:type [:= :mod-component]]
-     [:id ::sm/uuid]
-     [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
-     [:name {:optional true} :string]
-     [:variant-id {:optional true} ::sm/uuid]
-     [:variant-properties {:optional true} [:vector ctv/schema:variant-property]]]]
+    [:del-media
+     [:map {:title "DelMediaChange"}
+      [:type [:= :del-media]]
+      [:id ::sm/uuid]]]
 
-   [:del-component
-    [:map {:title "DelComponentChange"}
-     [:type [:= :del-component]]
-     [:id ::sm/uuid]
-     ;; when it is an undo of a cut-paste, we need to undo the movement
-     ;; of the shapes so we need to move them delta
-     [:delta {:optional true} ::gpt/point]
-     [:skip-undelete? {:optional true} :boolean]]]
+    [:add-component
+     [:map {:title "AddComponentChange"}
+      [:type [:= :add-component]]
+      [:id ::sm/uuid]
+      [:name :string]
+      [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
+      [:path {:optional true} :string]]]
 
-   [:restore-component
-    [:map {:title "RestoreComponentChange"}
-     [:type [:= :restore-component]]
-     [:id ::sm/uuid]
-     [:page-id ::sm/uuid]]]
+    [:mod-component
+     [:map {:title "ModCompoenentChange"}
+      [:type [:= :mod-component]]
+      [:id ::sm/uuid]
+      [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]]
+      [:name {:optional true} :string]
+      [:variant-id {:optional true} ::sm/uuid]
+      [:variant-properties {:optional true} [:vector ::ctv/variant-property]]]]
 
-   [:purge-component
-    [:map {:title "PurgeComponentChange"}
-     [:type [:= :purge-component]]
-     [:id ::sm/uuid]]]
+    [:del-component
+     [:map {:title "DelComponentChange"}
+      [:type [:= :del-component]]
+      [:id ::sm/uuid]
+      ;; when it is an undo of a cut-paste, we need to undo the movement
+      ;; of the shapes so we need to move them delta
+      [:delta {:optional true} ::gpt/point]
+      [:skip-undelete? {:optional true} :boolean]]]
 
-   [:add-typography
-    [:map {:title "AddTypogrphyChange"}
-     [:type [:= :add-typography]]
-     [:typography ctt/schema:typography]]]
+    [:restore-component
+     [:map {:title "RestoreComponentChange"}
+      [:type [:= :restore-component]]
+      [:id ::sm/uuid]
+      [:page-id ::sm/uuid]]]
 
-   [:mod-typography
-    [:map {:title "ModTypogrphyChange"}
-     [:type [:= :mod-typography]]
-     [:typography ctt/schema:typography]]]
+    [:purge-component
+     [:map {:title "PurgeComponentChange"}
+      [:type [:= :purge-component]]
+      [:id ::sm/uuid]]]
 
-   [:del-typography
-    [:map {:title "DelTypogrphyChange"}
-     [:type [:= :del-typography]]
-     [:id ::sm/uuid]]]
+    [:add-typography
+     [:map {:title "AddTypogrphyChange"}
+      [:type [:= :add-typography]]
+      [:typography ::ctt/typography]]]
 
-   [:set-tokens-lib
-    [:map {:title "SetTokensLib"}
-     [:type [:= :set-tokens-lib]]
-     [:tokens-lib ::sm/any]]]  ;; TODO: we should define a plain object schema for tokens-lib
+    [:mod-typography
+     [:map {:title "ModTypogrphyChange"}
+      [:type [:= :mod-typography]]
+      [:typography ::ctt/typography]]]
 
-   [:set-token
-    [:map {:title "SetTokenChange"}
-     [:type [:= :set-token]]
-     [:set-id ::sm/uuid]
-     [:token-id ::sm/uuid]
-     [:attrs [:maybe ctob/schema:token-attrs]]]]
+    [:del-typography
+     [:map {:title "DelTypogrphyChange"}
+      [:type [:= :del-typography]]
+      [:id ::sm/uuid]]]
 
-   [:set-token-set
-    [:map {:title "SetTokenSetChange"}
-     [:type [:= :set-token-set]]
-     [:id ::sm/uuid]
-     [:attrs [:maybe ctob/schema:token-set-attrs]]]]
+    [:update-active-token-themes
+     [:map {:title "UpdateActiveTokenThemes"}
+      [:type [:= :update-active-token-themes]]
+      [:theme-paths [:set :string]]]]
 
-   [:set-token-theme
-    [:map {:title "SetTokenThemeChange"}
-     [:type [:= :set-token-theme]]
-     [:id ::sm/uuid]
-     [:attrs [:maybe ctob/schema:token-theme-attrs]]]]
+    [:rename-token-set-group
+     [:map {:title "RenameTokenSetGroup"}
+      [:type [:= :rename-token-set-group]]
+      [:set-group-path [:vector :string]]
+      [:set-group-fname :string]]]
 
-   [:set-active-token-themes
-    [:map {:title "SetActiveTokenThemes"}
-     [:type [:= :set-active-token-themes]]
-     [:theme-paths [:set :string]]]]
+    [:move-token-set
+     [:map {:title "MoveTokenSet"}
+      [:type [:= :move-token-set]]
+      [:from-path [:vector :string]]
+      [:to-path [:vector :string]]
+      [:before-path [:maybe [:vector :string]]]
+      [:before-group [:maybe :boolean]]]]
 
-   [:rename-token-set-group
-    [:map {:title "RenameTokenSetGroup"}
-     [:type [:= :rename-token-set-group]]
-     [:set-group-path [:vector :string]]
-     [:set-group-fname :string]]]
+    [:move-token-set-group
+     [:map {:title "MoveTokenSetGroup"}
+      [:type [:= :move-token-set-group]]
+      [:from-path [:vector :string]]
+      [:to-path [:vector :string]]
+      [:before-path [:maybe [:vector :string]]]
+      [:before-group [:maybe :boolean]]]]
 
-   [:move-token-set
-    [:map {:title "MoveTokenSet"}
-     [:type [:= :move-token-set]]
-     [:from-path [:vector :string]]
-     [:to-path [:vector :string]]
-     [:before-path [:maybe [:vector :string]]]
-     [:before-group [:maybe :boolean]]]]
+    [:set-token-theme
+     [:map {:title "SetTokenThemeChange"}
+      [:type [:= :set-token-theme]]
+      [:theme-name :string]
+      [:group :string]
+      [:theme [:maybe ctob/schema:token-theme-attrs]]]]
 
-   [:move-token-set-group
-    [:map {:title "MoveTokenSetGroup"}
-     [:type [:= :move-token-set-group]]
-     [:from-path [:vector :string]]
-     [:to-path [:vector :string]]
-     [:before-path [:maybe [:vector :string]]]
-     [:before-group [:maybe :boolean]]]]
+    [:set-tokens-lib
+     [:map {:title "SetTokensLib"}
+      [:type [:= :set-tokens-lib]]
+      [:tokens-lib ::sm/any]]]
 
-   [:set-base-font-size
-    [:map {:title "ModBaseFontSize"}
-     [:type [:= :set-base-font-size]]
-     [:base-font-size :string]]]])
+    [:set-token-set
+     [:map {:title "SetTokenSetChange"}
+      [:type [:= :set-token-set]]
+      [:set-name :string]
+      [:group? :boolean]
+
+      ;; FIXME: we should not pass private types as part of changes
+      ;; protocol, the changes protocol should reflect a
+      ;; method/protocol for perform surgical operations on file data,
+      ;; this has nothing todo with internal types of a file data
+      ;; structure.
+      [:token-set {:gen/gen (sg/generator ctob/schema:token-set)}
+       [:maybe [:fn ctob/token-set?]]]]]
+
+    [:set-token
+     [:map {:title "SetTokenChange"}
+      [:type [:= :set-token]]
+      [:set-name :string]
+      [:token-id ::sm/uuid]
+      [:token [:maybe ctob/schema:token-attrs]]]]
+
+    [:set-base-font-size
+     [:map {:title "ModBaseFontSize"}
+      [:type [:= :set-base-font-size]]
+      [:base-font-size :string]]]]])
 
 (def schema:changes
   [:sequential {:gen/max 5 :gen/min 1} schema:change])
 
+(sm/register! ::change schema:change)
+(sm/register! ::changes schema:changes)
+
 (def valid-change?
   (sm/lazy-validator schema:change))
 
-(def check-changes
+(def check-changes!
   (sm/check-fn schema:changes))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -513,7 +534,7 @@
    ;; When verify? false we spec the schema validation. Currently used
    ;; to make just 1 validation even if the changes are applied twice
    (when verify?
-     (check-changes items))
+     (check-changes! items))
 
    (binding [*touched-changes* (volatile! #{})
              cts/*wasm-sync* true]
@@ -526,6 +547,11 @@
        #?(:clj (validate-shapes! data result items))
        result))))
 
+;; DEPRECATED: remove after 2.3 release
+(defmethod process-change :set-option
+  [data _]
+  data)
+
 ;; --- Comment Threads
 
 (defmethod process-change :set-comment-thread-position
@@ -919,6 +945,12 @@
   [data {:keys [id]}]
   (ctl/delete-color data id))
 
+;; DEPRECATED: remove before 2.3
+(defmethod process-change :add-recent-color
+  [data _]
+  data)
+
+
 ;; -- Media
 
 (defmethod process-change :add-media
@@ -969,63 +1001,64 @@
   [data {:keys [id]}]
   (ctyl/delete-typography data id))
 
-;; -- Design Tokens
+;; -- Tokens
 
 (defmethod process-change :set-tokens-lib
   [data {:keys [tokens-lib]}]
   (assoc data :tokens-lib tokens-lib))
 
 (defmethod process-change :set-token
-  [data {:keys [set-id token-id attrs]}]
+  [data {:keys [set-name token-id token]}]
   (update data :tokens-lib
           (fn [lib]
             (let [lib' (ctob/ensure-tokens-lib lib)]
               (cond
-                (not attrs)
-                (ctob/delete-token lib' set-id token-id)
+                (not token)
+                (ctob/delete-token-from-set lib' set-name token-id)
 
-                (not (ctob/get-token lib' set-id token-id))
-                (ctob/add-token lib' set-id (ctob/make-token attrs))
+                (not (ctob/get-token-in-set lib' set-name token-id))
+                (ctob/add-token-in-set lib' set-name (ctob/make-token token))
 
                 :else
-                (ctob/update-token lib' set-id token-id
-                                   (fn [prev-token]
-                                     (ctob/make-token (merge prev-token attrs)))))))))
+                (ctob/update-token-in-set lib' set-name token-id (fn [prev-token]
+                                                                   (ctob/make-token (merge prev-token token)))))))))
 
 (defmethod process-change :set-token-set
-  [data {:keys [id attrs]}]
+  [data {:keys [set-name group? token-set]}]
   (update data :tokens-lib
           (fn [lib]
             (let [lib' (ctob/ensure-tokens-lib lib)]
               (cond
-                (not attrs)
-                (ctob/delete-set lib' id)
+                (not token-set)
+                (if group?
+                  (ctob/delete-set-group lib' set-name)
+                  (ctob/delete-set lib' set-name))
 
-                (not (ctob/get-set lib' id))
-                (ctob/add-set lib' (ctob/make-token-set attrs))
+                (not (ctob/get-set lib' set-name))
+                (ctob/add-set lib' token-set)
 
                 :else
-                (ctob/update-set lib' id (fn [_] (ctob/make-token-set attrs))))))))
+                (ctob/update-set lib' set-name (fn [_] token-set)))))))
 
 (defmethod process-change :set-token-theme
-  [data {:keys [id attrs]}]
+  [data {:keys [group theme-name theme]}]
   (update data :tokens-lib
           (fn [lib]
             (let [lib' (ctob/ensure-tokens-lib lib)]
               (cond
-                (not attrs)
-                (ctob/delete-theme lib' id)
+                (not theme)
+                (ctob/delete-theme lib' group theme-name)
 
-                (not (ctob/get-theme lib' id))
-                (ctob/add-theme lib' (ctob/make-token-theme attrs))
+                (not (ctob/get-theme lib' group theme-name))
+                (ctob/add-theme lib' (ctob/make-token-theme theme))
 
                 :else
                 (ctob/update-theme lib'
-                                   id
+                                   group theme-name
                                    (fn [prev-token-theme]
-                                     (ctob/make-token-theme (merge prev-token-theme attrs)))))))))
+                                     (ctob/make-token-theme (merge prev-token-theme theme)))))))))
 
-(defmethod process-change :set-active-token-themes
+(defmethod process-change :update-active-token-themes
   [data {:keys [theme-paths]}]
   (update data :tokens-lib #(-> % (ctob/ensure-tokens-lib)
                                 (ctob/set-active-themes theme-paths))))
@@ -1049,7 +1082,7 @@
                                 (ctob/ensure-tokens-lib)
                                 (ctob/move-set-group from-path to-path before-path before-group))))
 
-;; === Design Tokens configuration
+;; === Base font size
 
 (defmethod process-change :set-base-font-size
   [data {:keys [base-font-size]}]
@@ -1058,23 +1091,21 @@
 
 ;; === Operations
 
-(def  decode-shape-attrs
-  (sm/decoder cts/schema:shape-attrs sm/json-transformer))
+(def ^:private decode-shape
+  (sm/decoder cts/schema:shape sm/json-transformer))
 
 (defmethod process-operation :assign
   [{:keys [type] :as shape} {:keys [value] :as op}]
   (let [modifications (assoc value :type type)
-        modifications (decode-shape-attrs modifications)]
+        modifications (decode-shape modifications)]
     (reduce-kv (fn [shape k v]
-                 (if (not= v (get shape k))
-                   (process-operation shape {:type :set
-                                             :attr k
-                                             :val v
-                                             :ignore-touched (:ignore-touched op)
-                                             :ignore-geometry (:ignore-geometry op)})
-                   shape))
+                 (process-operation shape {:type :set
+                                           :attr k
+                                           :val v
+                                           :ignore-touched (:ignore-touched op)
+                                           :ignore-geometry (:ignore-geometry op)}))
                shape
-               (dissoc modifications :type))))
+               modifications)))
 
 (defmethod process-operation :set
   [shape op]

common/src/app/common/files/changes_builder.cljc

@@ -21,11 +21,10 @@
    [app.common.types.path :as path]
    [app.common.types.shape.layout :as ctl]
    [app.common.types.tokens-lib :as ctob]
-   [app.common.uuid :as uuid]
-   [clojure.datafy :refer [datafy]]))
+   [app.common.uuid :as uuid]))
 
 ;; Auxiliary functions to help create a set of changes (undo + redo)
-;; TODO: this is a duplicate schema
+
 (def schema:changes
   (sm/register!
    ^{::sm/type ::changes}
@@ -37,7 +36,7 @@
     [:stack-undo? {:optional true} boolean?]
     [:undo-group {:optional true} ::sm/any]]))
 
-(def check-changes
+(def check-changes!
   (sm/check-fn schema:changes))
 
 (defn empty-changes
@@ -169,8 +168,9 @@
 
 (defn apply-changes-local
   [changes & {:keys [apply-to-library?]}]
-  (assert (check-changes changes)
-          "expected valid changes")
+  (assert
+   (check-changes! changes)
+   "expected valid changes")
 
   (if-let [file-data (::file-data (meta changes))]
     (let [library-data  (::library-data (meta changes))
@@ -718,7 +718,6 @@
     (reduce resize-parent changes all-parents)))
 
 ;; Library changes
-
 (defn add-color
   [changes color]
   (-> changes
@@ -800,6 +799,160 @@
         (update :undo-changes conj {:type :add-typography :typography prev-typography})
         (apply-changes-local))))
 
+(defn update-active-token-themes
+  [changes active-theme-paths prev-active-theme-paths]
+  (-> changes
+      (update :redo-changes conj {:type :update-active-token-themes :theme-paths active-theme-paths})
+      (update :undo-changes conj {:type :update-active-token-themes :theme-paths prev-active-theme-paths})
+      (apply-changes-local)))
+
+(defn set-token-theme [changes group theme-name theme]
+  (assert-library! changes)
+  (let [library-data (::library-data (meta changes))
+        prev-theme (some-> (get library-data :tokens-lib)
+                           (ctob/get-theme group theme-name))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token-theme
+                                    :theme-name theme-name
+                                    :group group
+                                    :theme theme})
+        (update :undo-changes conj (if prev-theme
+                                     {:type :set-token-theme
+                                      :group group
+                                      :theme-name (or
+                                                   ;; Undo of edit
+                                                   (:name theme)
+                                                   ;; Undo of delete
+                                                   theme-name)
+                                      :theme prev-theme}
+                                     ;; Undo of create
+                                     {:type :set-token-theme
+                                      :group group
+                                      :theme-name theme-name
+                                      :theme nil}))
+        (apply-changes-local))))
+
+(defn rename-token-set-group
+  [changes set-group-path set-group-fname]
+  (let [undo-path (ctob/replace-last-path-name set-group-path set-group-fname)
+        undo-fname (last set-group-path)]
+    (-> changes
+        (update :redo-changes conj {:type :rename-token-set-group :set-group-path set-group-path :set-group-fname set-group-fname})
+        (update :undo-changes conj {:type :rename-token-set-group :set-group-path undo-path :set-group-fname undo-fname})
+        (apply-changes-local))))
+
+(defn move-token-set
+  [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?] :as opts}]
+  (-> changes
+      (update :redo-changes conj {:type :move-token-set
+                                  :from-path from-path
+                                  :to-path to-path
+                                  :before-path before-path
+                                  :before-group before-group?})
+      (update :undo-changes conj {:type :move-token-set
+                                  :from-path to-path
+                                  :to-path from-path
+                                  :before-path prev-before-path
+                                  :before-group prev-before-group?})
+      (apply-changes-local)))
+
+(defn move-token-set-group
+  [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?]}]
+  (-> changes
+      (update :redo-changes conj {:type :move-token-set-group
+                                  :from-path from-path
+                                  :to-path to-path
+                                  :before-path before-path
+                                  :before-group before-group?})
+      (update :undo-changes conj {:type :move-token-set-group
+                                  :from-path to-path
+                                  :to-path from-path
+                                  :before-path prev-before-path
+                                  :before-group prev-before-group?})
+      (apply-changes-local)))
+
+(defn set-tokens-lib
+  [changes tokens-lib]
+  (assert-library! changes)
+  (let [library-data (::library-data (meta changes))
+        prev-tokens-lib (get library-data :tokens-lib)]
+    (-> changes
+        (update :redo-changes conj {:type :set-tokens-lib :tokens-lib tokens-lib})
+        (update :undo-changes conj {:type :set-tokens-lib :tokens-lib prev-tokens-lib})
+        (apply-changes-local))))
+
+(defn set-token [changes set-name token-id token]
+  (assert-library! changes)
+  (let [library-data (::library-data (meta changes))
+        prev-token (some-> (get library-data :tokens-lib)
+                           (ctob/get-set set-name)
+                           (ctob/get-token token-id))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token
+                                    :set-name set-name
+                                    :token-id token-id
+                                    :token token})
+        (update :undo-changes conj (if prev-token
+                                     {:type :set-token
+                                      :set-name set-name
+                                      :token-id (or
+                                                   ;; Undo of edit
+                                                 (:id token)
+                                                   ;; Undo of delete
+                                                 token-id)
+                                      :token prev-token}
+                                     ;; Undo of create token
+                                     {:type :set-token
+                                      :set-name set-name
+                                      :token-id token-id
+                                      :token nil}))
+        (apply-changes-local))))
+
+(defn rename-token-set
+  [changes name new-name]
+
+  (assert-library! changes)
+  (let [library-data   (::library-data (meta changes))
+        prev-token-set (some-> (get library-data :tokens-lib)
+                               (ctob/get-set name))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token-set
+                                    :set-name name
+                                    :token-set (ctob/rename prev-token-set new-name)
+                                    :group? false})
+        (update :undo-changes conj {:type :set-token-set
+                                    :set-name new-name
+                                    :token-set prev-token-set
+                                    :group? false})
+        (apply-changes-local))))
+
+(defn set-token-set
+  [changes set-name group? token-set]
+  (assert-library! changes)
+  (let [library-data   (::library-data (meta changes))
+        prev-token-set (some-> (get library-data :tokens-lib)
+                               (ctob/get-set set-name))]
+    (-> changes
+        (update :redo-changes conj {:type :set-token-set
+                                    :set-name set-name
+                                    :token-set token-set
+                                    :group? group?})
+        (update :undo-changes conj (if prev-token-set
+                                     {:type :set-token-set
+                                      :set-name (if token-set
+                                                  ;; Undo of edit
+                                                  (ctob/get-name token-set)
+                                                  ;; Undo of delete
+                                                  set-name)
+                                      :token-set prev-token-set
+                                      :group? group?}
+                                     ;; Undo of create
+                                     {:type :set-token-set
+                                      :set-name set-name
+                                      :token-set nil
+                                      :group? group?}))
+        (apply-changes-local))))
+
 (defn add-component
   ([changes id path name updated-shapes main-instance-id main-instance-page]
    (add-component changes id path name updated-shapes main-instance-id main-instance-page nil nil nil))
@@ -929,144 +1082,6 @@
                                   :id id
                                   :delta delta})))
 
-;; Design Tokens changes
-
-(defn set-tokens-lib
-  [changes tokens-lib]
-  (assert-library! changes)
-  (let [library-data (::library-data (meta changes))
-        prev-tokens-lib (get library-data :tokens-lib)]
-    (-> changes
-        (update :redo-changes conj {:type :set-tokens-lib :tokens-lib tokens-lib})
-        (update :undo-changes conj {:type :set-tokens-lib :tokens-lib prev-tokens-lib})
-        (apply-changes-local))))
-
-(defn set-token [changes set-id token-id token]
-  (assert-library! changes)
-  (let [library-data (::library-data (meta changes))
-        prev-token (some-> (get library-data :tokens-lib)
-                           (ctob/get-token set-id token-id))]
-    (-> changes
-        (update :redo-changes conj {:type :set-token
-                                    :set-id set-id
-                                    :token-id token-id
-                                    :attrs (datafy token)})
-        (update :undo-changes conj {:type :set-token
-                                    :set-id set-id
-                                    :token-id token-id
-                                    :attrs (datafy prev-token)})
-        (apply-changes-local))))
-
-(defn set-token-set
-  [changes id token-set]
-  (assert-library! changes)
-  (let [library-data   (::library-data (meta changes))
-        prev-token-set (some-> (get library-data :tokens-lib)
-                               (ctob/get-set id))]
-    (-> changes
-        (update :redo-changes conj {:type :set-token-set
-                                    :id id
-                                    :attrs (datafy token-set)})
-        (update :undo-changes conj {:type :set-token-set
-                                    :id id
-                                    :attrs (datafy prev-token-set)})
-        (apply-changes-local))))
-
-(defn rename-token-set
-  [changes id new-name]
-  (assert-library! changes)
-  (let [library-data   (::library-data (meta changes))
-        prev-token-set (some-> (get library-data :tokens-lib)
-                               (ctob/get-set id))]
-    (-> changes
-        (update :redo-changes conj {:type :set-token-set
-                                    :id id
-                                    :attrs (datafy (ctob/rename prev-token-set new-name))})
-        (update :undo-changes conj {:type :set-token-set
-                                    :id id
-                                    :attrs (datafy prev-token-set)})
-        (apply-changes-local))))
-
-(defn set-token-theme [changes id theme]
-  (assert-library! changes)
-  (let [library-data (::library-data (meta changes))
-        prev-theme (some-> (get library-data :tokens-lib)
-                           (ctob/get-theme id))]
-    (-> changes
-        (update :redo-changes conj {:type :set-token-theme
-                                    :id id
-                                    :attrs (datafy theme)})
-        (update :undo-changes conj {:type :set-token-theme
-                                    :id id
-                                    :attrs (datafy prev-theme)})
-        (apply-changes-local))))
-
-(defn set-active-token-themes
-  [changes active-theme-paths]
-  (assert-library! changes)
-  (let [library-data (::library-data (meta changes))
-        prev-active-theme-paths (d/nilv (some-> (get library-data :tokens-lib)
-                                                (ctob/get-active-theme-paths))
-                                        #{})]
-    (-> changes
-        (update :redo-changes conj {:type :set-active-token-themes :theme-paths active-theme-paths})
-        (update :undo-changes conj {:type :set-active-token-themes :theme-paths prev-active-theme-paths})
-        (apply-changes-local))))
-
-(defn rename-token-set-group
-  [changes set-group-path set-group-fname]
-  (let [undo-path (ctob/replace-last-path-name set-group-path set-group-fname)
-        undo-fname (last set-group-path)]
-    (-> changes
-        (update :redo-changes conj {:type :rename-token-set-group :set-group-path set-group-path :set-group-fname set-group-fname})
-        (update :undo-changes conj {:type :rename-token-set-group :set-group-path undo-path :set-group-fname undo-fname})
-        (apply-changes-local))))
-
-(defn move-token-set
-  [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?] :as opts}]
-  (-> changes
-      (update :redo-changes conj {:type :move-token-set
-                                  :from-path from-path
-                                  :to-path to-path
-                                  :before-path before-path
-                                  :before-group before-group?})
-      (update :undo-changes conj {:type :move-token-set
-                                  :from-path to-path
-                                  :to-path from-path
-                                  :before-path prev-before-path
-                                  :before-group prev-before-group?})
-      (apply-changes-local)))
-
-(defn move-token-set-group
-  [changes {:keys [from-path to-path before-path before-group? prev-before-path prev-before-group?]}]
-  (-> changes
-      (update :redo-changes conj {:type :move-token-set-group
-                                  :from-path from-path
-                                  :to-path to-path
-                                  :before-path before-path
-                                  :before-group before-group?})
-      (update :undo-changes conj {:type :move-token-set-group
-                                  :from-path to-path
-                                  :to-path from-path
-                                  :before-path prev-before-path
-                                  :before-group prev-before-group?})
-      (apply-changes-local)))
-
-(defn set-base-font-size
-  [changes new-base-font-size]
-  (assert-file-data! changes)
-  (let [file-data  (::file-data (meta changes))
-        previous-font-size (ctf/get-base-font-size file-data)]
-    (-> changes
-        (update :redo-changes conj {:type :set-base-font-size
-                                    :base-font-size new-base-font-size})
-
-        (update :undo-changes conj {:type :set-base-font-size
-                                    :base-font-size previous-font-size})
-        (apply-changes-local))))
-
-;; Misc changes
-
 (defn reorder-children
   [changes id children]
   (assert-page-id! changes)
@@ -1149,3 +1164,15 @@
   [changes]
   (::page-id (meta changes)))
 
+(defn set-base-font-size
+  [changes new-base-font-size]
+  (assert-file-data! changes)
+  (let [file-data  (::file-data (meta changes))
+        previous-font-size (ctf/get-base-font-size file-data)]
+    (-> changes
+        (update :redo-changes conj {:type :set-base-font-size
+                                    :base-font-size new-base-font-size})
+
+        (update :undo-changes conj {:type :set-base-font-size
+                                    :base-font-size previous-font-size})
+        (apply-changes-local))))

common/src/app/common/files/helpers.cljc

@@ -426,15 +426,15 @@
 
 (defn components-nesting-loop?
   "Check if a nesting loop would be created if the given shape is moved below the given parent"
-  ([objects shape-id parent-id]
-   (let [children (get-children-with-self objects shape-id)
-         parents  (get-parents-with-self objects parent-id)]
-     (components-nesting-loop? children parents)))
-  ([children parents]
-   (let [xf-get-component-id (keep :component-id)
-         child-components    (into #{} xf-get-component-id children)
-         parent-components   (into #{} xf-get-component-id parents)]
-     (seq (set/intersection child-components parent-components)))))
+  [objects shape-id parent-id]
+  (let [xf-get-component-id (keep :component-id)
+
+        children            (get-children-with-self objects shape-id)
+        child-components    (into #{} xf-get-component-id children)
+
+        parents             (get-parents-with-self objects parent-id)
+        parent-components   (into #{} xf-get-component-id parents)]
+    (seq (set/intersection child-components parent-components))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; ALGORITHMS & TRANSFORMATIONS FOR SHAPES
@@ -692,9 +692,122 @@
     (walk/postwalk process-form data)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; SHAPES ORGANIZATION
+;; SHAPES ORGANIZATION (PATH MANAGEMENT)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn split-path
+  "Decompose a string in the form 'one / two / three' into
+  a vector of strings, normalizing spaces."
+  [path]
+  (let [xf (comp (map str/trim)
+                 (remove str/empty?))]
+    (->> (str/split path "/")
+         (into [] xf))))
+
+(defn join-path
+  "Regenerate a path as a string, from a vector."
+  [path-vec]
+  (str/join " / " path-vec))
+
+(defn join-path-with-dot
+  "Regenerate a path as a string, from a vector."
+  [path-vec]
+  (str/join "\u00A0\u2022\u00A0" path-vec))
+
+(defn clean-path
+  "Remove empty items from the path."
+  [path]
+  (->> (split-path path)
+       (join-path)))
+
+(defn parse-path-name
+  "Parse a string in the form 'group / subgroup / name'.
+  Retrieve the path and the name in separated values, normalizing spaces."
+  [path-name]
+  (let [path-name-split (split-path path-name)
+        path (str/join " / " (butlast path-name-split))
+        name (or (last path-name-split) "")]
+    [path name]))
+
+(defn merge-path-item
+  "Put the item at the end of the path."
+  [path name]
+  (if-not (empty? path)
+    (if-not (empty? name)
+      (str path " / " name)
+      path)
+    name))
+
+(defn merge-path-item-with-dot
+  "Put the item at the end of the path."
+  [path name]
+  (if-not (empty? path)
+    (if-not (empty? name)
+      (str path "\u00A0\u2022\u00A0" name)
+      path)
+    name))
+
+(defn compact-path
+  "Separate last item of the path, and truncate the others if too long:
+    'one'                          ->  ['' 'one' false]
+    'one / two / three'            ->  ['one / two' 'three' false]
+    'one / two / three / four'     ->  ['one / two / ...' 'four' true]
+    'one-item-but-very-long / two' ->  ['...' 'two' true] "
+  [path max-length dot?]
+  (let [path-split (split-path path)
+        last-item  (last path-split)
+        merge-path (if dot?
+                     merge-path-item-with-dot
+                     merge-path-item)]
+    (loop [other-items (seq (butlast path-split))
+           other-path  ""]
+      (if-let [item (first other-items)]
+        (let [full-path (-> other-path
+                            (merge-path item)
+                            (merge-path last-item))]
+          (if (> (count full-path) max-length)
+            [(merge-path other-path "...") last-item true]
+            (recur (next other-items)
+                   (merge-path other-path item))))
+        [other-path last-item false]))))
+
+(defn butlast-path
+  "Remove the last item of the path."
+  [path]
+  (let [split (split-path path)]
+    (if (= 1 (count split))
+      ""
+      (join-path (butlast split)))))
+
+(defn butlast-path-with-dots
+  "Remove the last item of the path."
+  [path]
+  (let [split (split-path path)]
+    (if (= 1 (count split))
+      ""
+      (join-path-with-dot (butlast split)))))
+
+(defn last-path
+  "Returns the last item of the path."
+  [path]
+  (last (split-path path)))
+
+(defn compact-name
+  "Append the first item of the path and the name."
+  [path name]
+  (let [path-split (split-path path)]
+    (merge-path-item (first path-split) name)))
+
+
+(defn split-by-last-period
+  "Splits a string into two parts:
+   the text before and including the last period,
+   and the text after the last period."
+  [s]
+  (if-let [last-period (str/last-index-of s ".")]
+    [(subs s 0 (inc last-period)) (subs s (inc last-period))]
+    [s ""]))
+
 (defn get-frame-objects
   "Retrieves a new objects map only with the objects under frame-id (with frame-id)"
   [objects frame-id]

common/src/app/common/files/indices.cljc

@@ -6,29 +6,14 @@
 
 (ns app.common.files.indices
   (:require
-   [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.files.helpers :as cfh]
    [app.common.uuid :as uuid]))
 
-(defn- generate-index
-  "An optimized algorithm for calculate parents index that walk from top
-  to down starting from a provided shape-id. Usefull when you want to
-  create an index for the whole objects or subpart of the tree."
-  [index objects shape-id parents]
-  (let [shape   (get objects shape-id)
-        index   (assoc index shape-id parents)
-        parents (cons shape-id parents)]
-    (reduce (fn [index shape-id]
-              (generate-index index objects shape-id parents))
-            index
-            (:shapes shape))))
-
 (defn generate-child-all-parents-index
   "Creates an index where the key is the shape id and the value is a set
   with all the parents"
   ([objects]
-   (generate-index {} objects uuid/zero []))
+   (generate-child-all-parents-index objects (vals objects)))
 
   ([objects shapes]
    (let [shape->entry
@@ -39,25 +24,24 @@
 (defn create-clip-index
   "Retrieves the mask information for an object"
   [objects parents-index]
-  (let [get-clip-parents
-        (fn [shape]
-          (let [shape-id (dm/get-prop shape :id)]
-            (cond-> []
-              (or (and (cfh/frame-shape? shape)
-                       (not (:show-content shape))
-                       (not= uuid/zero shape-id))
-                  (cfh/bool-shape? shape))
-              (conj shape)
-
-              (:masked-group shape)
-              (conj (get objects (->> shape :shapes first))))))
-
-        xform
-        (comp (map (d/getf objects))
-              (mapcat get-clip-parents))
-
-        populate-with-clips
+  (let [retrieve-clips
         (fn [parents]
-          (into [] xform parents))]
+          (let [lookup-object (fn [id] (get objects id))
+                get-clip-parents
+                (fn [shape]
+                  (cond-> []
+                    (or (and (= :frame (:type shape))
+                             (not (:show-content shape))
+                             (not= uuid/zero (:id shape)))
+                        (cfh/bool-shape? shape))
+                    (conj shape)
 
-    (d/update-vals parents-index populate-with-clips)))
+                    (:masked-group shape)
+                    (conj (get objects (->> shape :shapes first)))))]
+
+            (into []
+                  (comp (map lookup-object)
+                        (mapcat get-clip-parents))
+                  parents)))]
+    (-> parents-index
+        (update-vals retrieve-clips))))

common/src/app/common/files/migrations.cljc

@@ -31,7 +31,6 @@
    [app.common.types.shape :as cts]
    [app.common.types.shape.interactions :as ctsi]
    [app.common.types.shape.shadow :as ctss]
-   [app.common.types.shape.text :as ctst]
    [app.common.types.text :as types.text]
    [app.common.uuid :as uuid]
    [clojure.set :as set]
@@ -1569,41 +1568,6 @@
     (-> data
         (update :pages-index d/update-vals update-page))))
 
-(defmethod migrate-data "0011-fix-invalid-text-touched-flags"
-  [data _]
-  (letfn [(fix-shape [shape]
-            (let [touched-groups (ctk/normal-touched-groups shape)
-                  content-touched? (touched-groups :content-group)
-                  text-touched?    (or (touched-groups :text-content-text)
-                                       (touched-groups :text-content-attribute)
-                                       (touched-groups :text-content-structure))]
-              (if (and text-touched? (not content-touched?))
-                (update shape :touched ctk/set-touched-group :content-group)
-                shape)))
-
-          (update-page [page]
-            (d/update-when page :objects d/update-vals fix-shape))]
-    (-> data
-        (update :pages-index d/update-vals update-page))))
-
-(defmethod migrate-data "0012-fix-position-data"
-  [data _]
-  (let [decode-fn
-        (sm/decoder ctst/schema:position-data sm/json-transformer)
-
-        update-object
-        (fn [object]
-          (if (cfh/text-shape? object)
-            (d/update-when object :position-data decode-fn)
-            object))
-
-        update-container
-        (fn [container]
-          (d/update-when container :objects d/update-vals update-object))]
-
-    (-> data
-        (update :pages-index d/update-vals update-container)
-        (d/update-when :components d/update-vals update-container))))
 
 (def available-migrations
   (into (d/ordered-set)
@@ -1671,6 +1635,4 @@
          "0008-fix-library-colors-v4"
          "0009-clean-library-colors"
          "0009-add-partial-text-touched-flags"
-         "0010-fix-swap-slots-pointing-non-existent-shapes"
-         "0011-fix-invalid-text-touched-flags"
-         "0012-fix-position-data"]))
+         "0010-fix-swap-slots-pointing-non-existent-shapes"]))

common/src/app/common/files/repair.cljc

@@ -320,31 +320,6 @@
             (pcb/with-file-data file-data)
             (pcb/update-shapes shape-ids detach-shape))))))
 
-(defmethod repair-error :ref-shape-is-not-head
-  [_ {:keys [shape page-id] :as error} file-data _]
-  (let [repair-shape
-        (fn [shape]
-          ; Convert shape in a normal copy, removing nested copy status
-          (log/debug :hint "  -> unhead shape")
-          (ctk/unhead-shape shape))]
-
-    (log/dbg :hint "repairing shape :shape-ref-is-not-head" :id (:id shape) :name (:name shape) :page-id page-id)
-    (-> (pcb/empty-changes nil page-id)
-        (pcb/with-file-data file-data)
-        (pcb/update-shapes [(:id shape)] repair-shape))))
-
-(defmethod repair-error :ref-shape-is-head
-  [_ {:keys [shape page-id args] :as error} file-data _]
-  (let [repair-shape
-        (fn [shape]
-          ; Convert shape in a nested head, adding component info
-          (log/debug :hint "  -> reroot shape")
-          (ctk/rehead-shape shape (:component-file args) (:component-id args)))]
-
-    (log/dbg :hint "repairing shape :shape-ref-is-head" :id (:id shape) :name (:name shape) :page-id page-id)
-    (-> (pcb/empty-changes nil page-id)
-        (pcb/with-file-data file-data)
-        (pcb/update-shapes [(:id shape)] repair-shape))))
 
 (defmethod repair-error :shape-ref-cycle
   [_ {:keys [shape args] :as error} file-data _]
@@ -516,19 +491,6 @@
         (pcb/with-library-data file-data)
         (pcb/update-component (:id shape) repair-component))))
 
-(defmethod repair-error :invalid-text-touched
-  [_ {:keys [shape page-id] :as error} file-data _]
-  (let [repair-shape
-        (fn [shape]
-          ;; Add content group
-          (log/debug :hint "  -> add :content-group to :touched-groups")
-          (update shape :touched ctk/set-touched-group :content-group))]
-
-    (log/dbg :hint "repairing shape :invalid-text-touched" :id (:id shape) :name (:name shape) :page-id page-id)
-    (-> (pcb/empty-changes nil page-id)
-        (pcb/with-file-data file-data)
-        (pcb/update-shapes [(:id shape)] repair-shape))))
-
 (defmethod repair-error :misplaced-slot
   [_ {:keys [shape page-id] :as error} file-data _]
   (let [repair-shape

common/src/app/common/files/shapes_builder.cljc

@@ -269,22 +269,6 @@
    (d/parse-double width 1)
    (d/parse-double height 1)))
 
-(defn- parse-radius-attrs
-  [attrs]
-  (if (or (contains? attrs :rx) (contains? attrs :ry))
-    (let [rx-val (d/parse-double (:rx attrs) 0)
-          ry-val (d/parse-double (:ry attrs) 0)
-          radius (cond
-                   (and (contains? attrs :rx) (contains? attrs :ry))
-                   (min rx-val ry-val)
-                   (contains? attrs :rx)
-                   rx-val
-                   (contains? attrs :ry)
-                   ry-val
-                   :else 0)]
-      {:r1 radius :r2 radius :r3 radius :r4 radius})
-    {}))
-
 (defn create-rect-shape [name frame-id svg-data {:keys [attrs] :as data}]
   (let [transform (->> (csvg/parse-transform (:transform attrs))
                        (gmt/transform-in (gpt/point svg-data)))
@@ -296,9 +280,7 @@
                       (update :y - (:y origin)))
 
         props     (-> (dissoc attrs :x :y :width :height :rx :ry :transform)
-                      (csvg/attrs->props))
-
-        radius-attrs (parse-radius-attrs attrs)]
+                      (csvg/attrs->props))]
     (cts/setup-shape
      (-> (calculate-rect-metadata rect transform)
          (assoc :type :rect)
@@ -306,10 +288,13 @@
          (assoc :frame-id frame-id)
          (assoc :svg-viewbox vbox)
          (assoc :svg-attrs props)
-        ;; We need to ensure fills are empty on import process
-        ;; because setup-shape assings one by default.
+         ;; We need to ensure fills are empty on import process
+         ;; because setup-shape assings one by default.
          (assoc :fills [])
-         (merge radius-attrs)))))
+         (cond-> (contains? attrs :rx)
+           (assoc :rx (d/parse-double (:rx attrs) 0)))
+         (cond-> (contains? attrs :ry)
+           (assoc :ry (d/parse-double (:ry attrs) 0)))))))
 
 (defn- parse-circle-attrs
   [attrs]
@@ -523,7 +508,6 @@
                  :else (dm/str tag))]
     (dm/str "svg-" suffix)))
 
-
 (defn parse-svg-element
   [frame-id svg-data {:keys [tag attrs hidden] :as element} unames]
 

common/src/app/common/files/validate.cljc

@@ -11,7 +11,6 @@
    [app.common.exceptions :as ex]
    [app.common.files.helpers :as cfh]
    [app.common.files.variant :as cfv]
-   [app.common.path-names :as cpn]
    [app.common.schema :as sm]
    [app.common.types.component :as ctk]
    [app.common.types.components-list :as ctkl]
@@ -48,8 +47,6 @@
     :should-be-component-root
     :should-not-be-component-root
     :ref-shape-not-found
-    :ref-shape-is-head
-    :ref-shape-is-not-head
     :shape-ref-in-main
     :root-main-not-allowed
     :nested-main-not-allowed
@@ -60,7 +57,6 @@
     :not-component-not-allowed
     :component-nil-objects-not-allowed
     :instance-head-not-frame
-    :invalid-text-touched
     :misplaced-slot
     :missing-slot
     :shape-ref-cycle
@@ -308,28 +304,6 @@
                   "Shape inside main instance should not have shape-ref"
                   shape file page)))
 
-(defn- check-ref-is-not-head
-  "Validate that the referenced shape is not a nested copy root."
-  [shape file page libraries]
-  (let [ref-shape (ctf/find-ref-shape file page libraries shape :include-deleted? true)]
-    (when (and (some? ref-shape)
-               (ctk/instance-head? ref-shape))
-      (report-error :ref-shape-is-head
-                    (str/ffmt "Referenced shape % is a component, so the copy must also be" (:shape-ref shape))
-                    shape file page))))
-
-(defn- check-ref-is-head
-  "Validate that the referenced shape is a nested copy root."
-  [shape file page libraries]
-  (let [ref-shape (ctf/find-ref-shape file page libraries shape :include-deleted? true)]
-    (when (and (some? ref-shape)
-               (not (ctk/instance-head? ref-shape)))
-      (report-error :ref-shape-is-not-head
-                    (str/ffmt "Referenced shape % of a head copy must also be a head" (:shape-ref shape))
-                    shape file page
-                    :component-file (:component-file ref-shape)
-                    :component-id (:component-id ref-shape)))))
-
 (defn- check-empty-swap-slot
   "Validate that this shape does not have any swap slot."
   [shape file page]
@@ -354,20 +328,6 @@
                   "This shape has children with the same swap slot"
                   shape file page)))
 
-(defn- check-valid-touched
-  "Validate that the text touched flags are coherent."
-  [shape file page]
-  (let [touched-groups (ctk/normal-touched-groups shape)
-        content-touched? (touched-groups :content-group)
-        text-touched?    (or (touched-groups :text-content-text)
-                             (touched-groups :text-content-attribute)
-                             (touched-groups :text-content-structure))]
-    ;; For now we only check this combination, that has been reported in some bugs
-    (when (and text-touched? (not content-touched?))
-      (report-error :invalid-text-touched
-                    "This thape has text type touched but not content touched"
-                    shape file page))))
-
 (defn- check-shape-main-root-top
   "Root shape of a top main instance:
 
@@ -407,10 +367,8 @@
     (check-component-not-main-head shape file page libraries)
     (check-component-root shape file page)
     (check-component-ref shape file page libraries)
-    (check-ref-is-head shape file page libraries)
     (check-empty-swap-slot shape file page)
     (check-duplicate-swap-slot shape file page)
-    (check-valid-touched shape file page)
     (run! #(check-shape % file page libraries :context :copy-top :library-exists library-exists) (:shapes shape))))
 
 (defn- check-shape-copy-root-nested
@@ -421,12 +379,10 @@
   [shape file page libraries library-exists]
   (check-component-not-main-head shape file page libraries)
   (check-component-not-root shape file page)
-  (check-valid-touched shape file page)
   ;; We can have situations where the nested copy and the ancestor copy come from different libraries and some of them have been dettached
   ;; so we only validate the shape-ref if the ancestor is from a valid library
   (when library-exists
-    (check-component-ref shape file page libraries)
-    (check-ref-is-head shape file page libraries))
+    (check-component-ref shape file page libraries))
   (run! #(check-shape % file page libraries :context :copy-nested) (:shapes shape)))
 
 (defn- check-shape-main-not-root
@@ -444,9 +400,7 @@
   (check-component-not-main-not-head shape file page)
   (check-component-not-root shape file page)
   (check-component-ref shape file page libraries)
-  (check-ref-is-not-head shape file page libraries)
   (check-empty-swap-slot shape file page)
-  (check-valid-touched shape file page)
   (run! #(check-shape % file page libraries :context :copy-any) (:shapes shape)))
 
 (defn- check-shape-not-component
@@ -512,7 +466,7 @@
       (report-error :variant-bad-name
                     (str/ffmt "Variant % has an invalid name" (:id shape))
                     shape file page))
-    (when-not (= (:name parent) (cpn/merge-path-item (:path component) (:name component)))
+    (when-not (= (:name parent) (cfh/merge-path-item (:path component) (:name component)))
       (report-error :variant-component-bad-name
                     (str/ffmt "Component % has an invalid name" (:id shape))
                     shape file page))
@@ -571,7 +525,7 @@
               ;; mains can't be nested into mains
               (if (or (= context :not-component) (= context :main-top))
                 (report-error :nested-main-not-allowed
-                              "Component main not allowed inside other component"
+                              "Nested main component only allowed inside other component"
                               shape file page)
                 (check-shape-main-root-nested shape file page libraries))
 
@@ -636,20 +590,6 @@
                     (str/ffmt "Shape % should be a variant" (:id main-component))
                     main-component file component-page))))
 
-(defn- check-main-inside-main
-  [component file]
-  (let [component-page (ctf/get-component-page (:data file) component)
-        main-instance  (ctst/get-shape component-page (:main-instance-id component))
-        main-parents?  (->> main-instance
-                            :id
-                            (cfh/get-parents (:objects component-page))
-                            (some ctk/main-instance?)
-                            boolean)]
-    (when main-parents?
-      (report-error :nested-main-not-allowed
-                    "Component main not allowed inside other component"
-                    main-instance file component-page))))
-
 (defn- check-component
   "Validate semantic coherence of a component. Report all errors found."
   [component file]
@@ -657,8 +597,6 @@
     (report-error :component-nil-objects-not-allowed
                   "Objects list cannot be nil"
                   component file nil))
-  (when-not (:deleted component)
-    (check-main-inside-main component file))
   (when (:deleted component)
     (check-component-duplicate-swap-slot component file)
     (check-ref-cycles component file))

common/src/app/common/flags.cljc

@@ -119,10 +119,7 @@
     ;; Only for developtment.
     :tiered-file-data-storage
     :token-units
-    :token-base-font-size
-    :token-color
     :token-typography-types
-    :token-typography-composite
     :transit-readable-response
     :user-feedback
     ;; TODO: remove this flag.
@@ -134,8 +131,7 @@
     :hide-release-modal
     :subscriptions
     :subscriptions-old
-    :frontend-binary-fills
-    :inspect-styles})
+    :frontend-binary-fills})
 
 (def all-flags
   (set/union email login varia))
@@ -157,9 +153,7 @@
    :enable-dashboard-templates-section
    :enable-google-fonts-provider
    :enable-component-thumbnails
-   :enable-render-wasm-dpr
-   :enable-token-units
-   :enable-token-typography-types])
+   :enable-render-wasm-dpr])
 
 (defn parse
   [& flags]

common/src/app/common/geom/matrix.cljc

@@ -25,7 +25,16 @@
 
 ;; --- Matrix Impl
 
-(declare format-precision)
+(defn format-precision
+  [mtx precision]
+  (when mtx
+    (dm/fmt "matrix(%, %, %, %, %, %)"
+            (mth/to-fixed (.-a mtx) precision)
+            (mth/to-fixed (.-b mtx) precision)
+            (mth/to-fixed (.-c mtx) precision)
+            (mth/to-fixed (.-d mtx) precision)
+            (mth/to-fixed (.-e mtx) precision)
+            (mth/to-fixed (.-f mtx) precision))))
 
 (cr/defrecord Matrix [^double a
                       ^double b
@@ -37,17 +46,6 @@
   (toString [this]
     (format-precision this precision)))
 
-(defn format-precision
-  [mtx precision]
-  (when mtx
-    (dm/fmt "matrix(%, %, %, %, %, %)"
-            (mth/to-fixed (.-a ^Matrix mtx) precision)
-            (mth/to-fixed (.-b ^Matrix mtx) precision)
-            (mth/to-fixed (.-c ^Matrix mtx) precision)
-            (mth/to-fixed (.-d ^Matrix mtx) precision)
-            (mth/to-fixed (.-e ^Matrix mtx) precision)
-            (mth/to-fixed (.-f ^Matrix mtx) precision))))
-
 (defn matrix?
   "Return true if `v` is Matrix instance."
   [v]

common/src/app/common/geom/modifiers.cljc

@@ -96,6 +96,7 @@
         (if (and (some? layout-line) (<= from-idx max-idx))
           (let [to-idx   (+ from-idx (:num-children layout-line))
                 children (subvec children from-idx to-idx)
+
                 [_ modif-tree]
                 (reduce set-child-modifiers [layout-line modif-tree] children)]
             (recur modif-tree (first pending) (rest pending) to-idx))

common/src/app/common/geom/rect.cljc

@@ -255,10 +255,10 @@
       (if-let [pt (first pts)]
         (let [x (dm/get-prop pt :x)
               y (dm/get-prop pt :y)]
-          (recur (double (mth/min minx x))
-                 (double (mth/min miny y))
-                 (double (mth/max maxx x))
-                 (double (mth/max maxy y))
+          (recur (mth/min minx x)
+                 (mth/min miny y)
+                 (mth/max maxx x)
+                 (mth/max maxy y)
                  (rest pts)))
         (when (d/num? minx miny maxx maxy)
           (make-rect minx miny (- maxx minx) (- maxy miny)))))))

common/src/app/common/geom/shapes/bounds.cljc

@@ -88,11 +88,8 @@
   ([shape]
    (get-shape-filter-bounds shape false))
   ([shape ignore-shadow-margin?]
-   (if (or (and (cfh/svg-raw-shape? shape)
-                (not= :svg (dm/get-in shape [:content :tag])))
-           ;; If no shadows or blur, we return the selrect as is
-           (and (empty? (-> shape :shadow))
-                (zero? (-> shape :blur :value (or 0)))))
+   (if (and (cfh/svg-raw-shape? shape)
+            (not= :svg (dm/get-in shape [:content :tag])))
      (dm/get-prop shape :selrect)
      (let [filters    (shape->filters shape)
            blur-value (or (-> shape :blur :value) 0)

common/src/app/common/geom/shapes/grid_layout/layout_data.cljc

@@ -393,7 +393,7 @@
       min-fr
       (let [{:keys [size type value]} (first tracks)
             min-fr (if (= type :flex) (max min-fr (/ size value)) min-fr)]
-        (recur (rest tracks) (double min-fr))))))
+        (recur (rest tracks) min-fr)))))
 
 (defn calc-layout-data
   ([parent transformed-parent-bounds children bounds objects]

common/src/app/common/logging.cljc

@@ -49,7 +49,6 @@
    [app.common.exceptions :as ex]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [cuerdas.core :as str]
    [promesa.exec :as px]
@@ -222,42 +221,36 @@
   #?(:clj (inst-ms (java.time.Instant/now))
      :cljs (js/Date.now)))
 
-(defn emit-log
-  [props cause context logger level sync?]
-  (let [props    (cond-> props sync? deref)
-        ts       (current-timestamp)
-        gcontext *context*
-        logfn    (fn []
-                   (let [props   (if sync? props (deref props))
-                         props   (into (d/ordered-map) props)
-                         context (if (and (empty? gcontext)
-                                          (empty? context))
-                                   {}
-                                   (d/without-nils (merge gcontext context)))
-
-                         lrecord {::id (uuid/next)
-                                  ::timestamp ts
-                                  ::message (delay (build-message props))
-                                  ::props props
-                                  ::context context
-                                  ::level level
-                                  ::logger logger}
-                         lrecord (cond-> lrecord
-                                   (some? cause)
-                                   (assoc ::cause cause
-                                          ::trace (delay (build-stack-trace cause))))]
-                     (swap! log-record (constantly lrecord))))]
-    (if sync?
-      (logfn)
-      (px/exec! *default-executor* logfn))))
-
 (defmacro log!
   "Emit a new log record to the global log-record state (asynchronously). "
   [& props]
   (let [{:keys [::level ::logger ::context ::sync? cause] :or {sync? false}} props
         props (into [] msg-props-xf props)]
     `(when (enabled? ~logger ~level)
-       (emit-log (delay ~props) ~cause ~context ~logger ~level ~sync?))))
+       (let [props#   (cond-> (delay ~props) ~sync? deref)
+             ts#      (current-timestamp)
+             context# *context*
+             logfn#   (fn []
+                        (let [props#   (if ~sync? props# (deref props#))
+                              props#   (into (d/ordered-map) props#)
+                              cause#   ~cause
+                              context# (d/without-nils
+                                        (merge context# ~context))
+                              lrecord# {::id (uuid/next)
+                                        ::timestamp ts#
+                                        ::message (delay (build-message props#))
+                                        ::props props#
+                                        ::context context#
+                                        ::level ~level
+                                        ::logger ~logger}
+                              lrecord# (cond-> lrecord#
+                                         (some? cause#)
+                                         (assoc ::cause cause#
+                                                ::trace (delay (build-stack-trace cause#))))]
+                          (swap! log-record (constantly lrecord#))))]
+         (if ~sync?
+           (logfn#)
+           (px/exec! *default-executor* logfn#))))))
 
 #?(:clj
    (defn slf4j-log-handler
@@ -283,8 +276,7 @@
      (when (enabled? logger level)
        (let [hstyles (str/ffmt "font-weight: 600; color: %" (level->color level))
              mstyles (str/ffmt "font-weight: 300; color: %" (level->color level))
-             ts      (ct/format-inst (ct/now) "kk:mm:ss.SSSS")
-             header  (str/concat "%c" (level->name level) " " ts  " [" logger "] ")
+             header  (str/concat "%c" (level->name level) " [" logger "] ")
              message (str/concat header "%c" @message)]
 
          (js/console.group message hstyles mstyles)

common/src/app/common/logic/libraries.cljc

@@ -11,13 +11,11 @@
    [app.common.data.macros :as dm]
    [app.common.files.changes-builder :as pcb]
    [app.common.files.helpers :as cfh]
-   [app.common.files.variant :as cfv]
    [app.common.geom.point :as gpt]
    [app.common.geom.shapes :as gsh]
    [app.common.logging :as log]
    [app.common.logic.shapes :as cls]
    [app.common.logic.variant-properties :as clvp]
-   [app.common.path-names :as cpn]
    [app.common.spec :as us]
    [app.common.types.component :as ctk]
    [app.common.types.components-list :as ctkl]
@@ -987,7 +985,7 @@
 (defn generate-rename-component
   "Generate the changes for rename the component with the given id, in the current file library."
   [changes id new-name library-data]
-  (let [[path name]   (cpn/split-group-name new-name)]
+  (let [[path name]   (cfh/parse-path-name new-name)]
     (-> changes
         (pcb/with-library-data library-data)
         (pcb/update-component id #(assoc % :path path :name name)))))
@@ -1735,17 +1733,6 @@
     [(conj roperations roperation)
      (conj uoperations uoperation)]))
 
-(defn- check-detached-main
-  [changes dest-shape origin-shape]
-  ;; Only for direct updates (from main to copy). Check if the main shape
-  ;; has been detached. If so, the copy shape must be unheaded (i.e. converted
-  ;; into a normal copy and not a nested instance).
-  (if (and (= (:shape-ref dest-shape) (:id origin-shape))
-           (ctk/subcopy-head? dest-shape)
-           (not (ctk/instance-head? origin-shape)))
-    (pcb/update-shapes changes [(:id dest-shape)] ctk/unhead-shape {:ignore-touched true})
-    changes))
-
 (defn- update-attrs
   "The main function that implements the attribute sync algorithm. Copy
   attributes that have changed in the origin shape to the dest shape.
@@ -1786,8 +1773,6 @@
             (seq roperations)
             (add-update-attr-changes dest-shape container roperations uoperations)
             :always
-            (check-detached-main dest-shape origin-shape)
-            :always
             (generate-update-tokens container dest-shape origin-shape touched omit-touched?))
 
           (let [attr-group        (get ctk/sync-attrs attr)
@@ -1811,6 +1796,7 @@
                  (= :content attr)
                  (touched attr-group))
 
+
                 skip-operations?
                 (or (= (get origin-shape attr) (get dest-shape attr))
                     (and (touched attr-group)
@@ -2034,6 +2020,7 @@
               skip-operations? (or skip-operations?
                                    (= attr-val (get current-shape attr)))
 
+
               ;; On a text-change, we want to force a position-data reset
               ;; so it's calculated again
               [roperations uoperations]
@@ -2041,16 +2028,6 @@
                 (add-update-attr-operations :position-data current-shape roperations uoperations nil)
                 [roperations uoperations])
 
-              ;; On a rotation operation we need to keep also the transformation matrixes
-              [roperations uoperations]
-              (if (and (not skip-operations?) (= attr :rotation))
-                (let [[roperations uoperations]
-                      (add-update-attr-operations
-                       :transform current-shape roperations uoperations (:transform previous-shape))]
-                  (add-update-attr-operations
-                   :transform-inverse current-shape roperations uoperations (:transform-inverse previous-shape)))
-                [roperations uoperations])
-
               [roperations' uoperations']
               (if skip-operations?
                 [roperations uoperations]
@@ -2236,7 +2213,7 @@
         variant-id  (when (ctk/is-variant? root) (:parent-id root))
         props       (when (ctk/is-variant? root) (get variant-props (:component-id root)))
 
-        [path name] (cpn/split-group-name name)
+        [path name] (cfh/parse-path-name name)
 
         [root-shape updated-shapes]
         (ctn/convert-shape-in-component root objects file-id)
@@ -2528,10 +2505,9 @@
             frames)))
 
 (defn- duplicate-variant
-  [changes library component base-pos parent page-id into-new-variant?]
+  [changes library component base-pos parent-id page-id]
   (let [component-page   (ctpl/get-page (:data library) (:main-instance-page component))
-        objects          (:objects component-page)
-        component-shape  (get objects (:main-instance-id component))
+        component-shape  (dm/get-in component-page [:objects (:main-instance-id component)])
         orig-pos         (gpt/point (:x component-shape) (:y component-shape))
         delta            (gpt/subtract base-pos orig-pos)
         new-component-id (uuid/next)
@@ -2541,27 +2517,11 @@
                                                        new-component-id
                                                        {:apply-changes-local-library? true
                                                         :delta delta
-                                                        :new-variant-id (if into-new-variant? nil (:id parent))
-                                                        :page-id page-id})
-        value             (when into-new-variant?
-                            (str ctv/value-prefix
-                                 (-> (cfv/extract-properties-values (:data library) objects (:id parent))
-                                     last
-                                     :value
-                                     count
-                                     inc)))]
-
+                                                        :new-variant-id parent-id
+                                                        :page-id page-id})]
     [shape
-     (cond-> changes
-       into-new-variant?
-       (clvp/generate-make-shapes-variant [shape] parent)
-
-       ;; If it has the same parent, update the value of the last property
-       (and into-new-variant? (= (:variant-id component) (:id parent)))
-       (clvp/generate-update-property-value new-component-id (-> component :variant-properties count dec) value)
-
-       :always
-       (pcb/change-parent (:id parent) [shape] 0))]))
+     (-> changes
+         (pcb/change-parent parent-id [shape]))]))
 
 
 (defn generate-duplicate-component-change
@@ -2573,13 +2533,11 @@
         pos          (as-> (gsh/move main delta) $
                        (gpt/point (:x $) (:y $)))
 
-        parent       (get objects parent-id)
-
-
         ;; When we duplicate a variant alone, we will instanciate it
         ;; When we duplicate a variant along with its variant-container, we will duplicate it
         in-variant-container? (contains? ids-map (:variant-id main))
 
+
         restore-component
         #(let [{:keys [shape changes]}
                (prepare-restore-component changes
@@ -2592,42 +2550,29 @@
                                           frame-id)]
            [shape changes])
 
-
         [_shape changes]
-        (cond
-          (nil? component)
+        (if (nil? component)
           (restore-component)
+          (if (and (ctk/is-variant? main) in-variant-container?)
+            (duplicate-variant changes
+                               (get libraries file-id)
+                               component
+                               pos
+                               parent-id
+                               (:id page))
 
-          (and (ctk/is-variant? main) in-variant-container?)
-          (duplicate-variant changes
-                             (get libraries file-id)
-                             component
-                             pos
-                             parent
-                             (:id page)
-                             false)
-
-          (ctk/is-variant-container? parent)
-          (duplicate-variant changes
-                             (get libraries file-id)
-                             component
-                             pos
-                             parent
-                             (:id page)
-                             true)
-          :else
-          (generate-instantiate-component changes
-                                          objects
-                                          file-id
-                                          component-id
-                                          pos
-                                          page
-                                          libraries
-                                          main-id
-                                          parent-id
-                                          frame-id
-                                          ids-map
-                                          {}))]
+            (generate-instantiate-component changes
+                                            objects
+                                            file-id
+                                            component-id
+                                            pos
+                                            page
+                                            libraries
+                                            main-id
+                                            parent-id
+                                            frame-id
+                                            ids-map
+                                            {})))]
     changes))
 
 (defn generate-duplicate-shape-change
@@ -2786,8 +2731,7 @@
 
         changes (-> changes
                     (pcb/with-page page)
-                    (pcb/with-objects all-objects)
-                    (pcb/with-library-data library-data))
+                    (pcb/with-objects all-objects))
         changes
         (->> shapes
              (reduce #(generate-duplicate-shape-change %1

common/src/app/common/logic/shapes.cljc

@@ -17,13 +17,12 @@
    [app.common.types.pages-list :as ctpl]
    [app.common.types.shape.interactions :as ctsi]
    [app.common.types.shape.layout :as ctl]
-   [app.common.types.shape.token :as ctst]
    [app.common.types.text :as ctt]
    [app.common.types.token :as cto]
    [app.common.uuid :as uuid]
    [clojure.set :as set]))
 
-(def text-typography-style-attrs (set ctt/text-typography-attrs))
+(def text-typography-attrs (set ctt/text-typography-attrs))
 
 (defn- generate-unapply-tokens
   "When updating attributes that have a token applied, we must unapply it, because the value
@@ -39,14 +38,10 @@
           (let [new-shape (get new-objects (:id shape))
                 attrs     (ctt/get-diff-attrs (:content shape) (:content new-shape))
 
-                attrs     (cond-> attrs
-                            ;; Unapply token when applying typography asset style
-                            (seq (set/intersection text-typography-style-attrs attrs))
-                            (into cto/typography-keys)
-
-                            ;; Unapply font-weight when changing the font-family attribute
-                            (and (:font-id attrs) (ctst/font-weight-applied? shape))
-                            (conj :font-weight))]
+                ;; Unapply token when applying typography asset style
+                attrs     (if (seq (set/intersection text-typography-attrs attrs))
+                            (into attrs cto/typography-keys)
+                            attrs)]
             (apply set/union (map cto/shape-attr->token-attrs attrs))))
 
         check-attr
@@ -185,17 +180,15 @@
                            interactions)))
                  (vals objects))
 
-         id-to-delete? (set ids-to-delete)
          changes
-         (->> (:flows page)
-              (reduce
-               (fn [changes [id flow]]
-                 (if (id-to-delete? (:starting-frame flow))
-                   (-> changes
-                       (pcb/with-page page)
-                       (pcb/set-flow id nil))
-                   changes))
-               changes))
+         (reduce (fn [changes {:keys [id] :as flow}]
+                   (if (contains? ids-to-delete (:starting-frame flow))
+                     (-> changes
+                         (pcb/with-page page)
+                         (pcb/set-flow id nil))
+                     changes))
+                 changes
+                 (:flows page))
 
 
          all-parents
@@ -407,10 +400,9 @@
                             (remove #(= % parent-id) all-parents))]
 
     (-> changes
-        ;; Remove layout-item properties and tokens when moving a shape outside a layout
+        ;; Remove layout-item properties when moving a shape outside a layout
         (cond-> (not (ctl/any-layout? parent))
-          (-> (pcb/update-shapes ids ctl/remove-layout-item-data)
-              (pcb/update-shapes ids cto/unapply-layout-item-tokens)))
+          (pcb/update-shapes ids ctl/remove-layout-item-data))
 
         ;; Remove the hide in viewer flag
         (cond-> (and (not= uuid/zero parent-id) (cfh/frame-shape? parent))