WIP

.circleci/config.yml

@@ -0,0 +1,342 @@
+version: 2.1
+
+jobs:
+  lint:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    steps:
+      - checkout
+
+      - run:
+          name: "fmt check"
+          working_directory: "."
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+
+      - run:
+          name: "lint clj common"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:common
+
+      - run:
+          name: "lint clj frontend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:frontend
+
+      - run:
+          name: "lint clj backend"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:backend
+
+      - run:
+          name: "lint clj exporter"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:exporter
+
+      - run:
+          name: "lint clj library"
+          working_directory: "."
+          command: |
+            yarn run lint:clj:library
+
+  test-common:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
+
+      - run:
+          name: "JVM tests"
+          working_directory: "./common"
+          command: |
+            clojure -M:dev:test
+
+      - run:
+          name: "NODE tests"
+          working_directory: "./common"
+          command: |
+            yarn install
+            yarn run test
+
+      - save_cache:
+          paths:
+            - ~/.m2
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
+
+  test-frontend:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: "install dependencies"
+          working_directory: "./frontend"
+          # We install playwright here because the dependent tasks
+          # uses the same cache as this task so we prepopulate it
+          command: |
+            yarn install
+            yarn run playwright install chromium
+
+      - run:
+          name: "lint scss on frontend"
+          working_directory: "./frontend"
+          command: |
+            yarn run lint:scss
+
+      - run:
+          name: "unit tests"
+          working_directory: "./frontend"
+          command: |
+            yarn run test
+
+      - save_cache:
+          paths:
+            - ~/.m2
+            - ~/.yarn
+            - ~/.gitlibs
+            - ~/.cache/ms-playwright
+          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+  test-library:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: Install dependencies and build
+          working_directory: "./library"
+          command: |
+            yarn install
+
+      - run:
+          name: Build and Test
+          working_directory: "./library"
+          command: |
+            ./scripts/build
+            yarn run test
+
+  test-components:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx6g -Xms2g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: Install dependencies
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run playwright install chromium
+
+      - run:
+          name: Build Storybook
+          working_directory: "./frontend"
+          command: yarn run build:storybook
+
+      - run:
+          name: Serve Storybook and run tests
+          working_directory: "./frontend"
+          command: |
+            npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
+              "npx http-server storybook-static --port 6006 --silent" \
+              "npx wait-on tcp:6006 && yarn test:storybook"
+
+  test-integration:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: large
+
+    environment:
+      JAVA_OPTS: -Xmx6g -Xms2g
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+
+      - run:
+          name: "integration tests"
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run build:app:assets
+            yarn run build:app
+            yarn run build:app:libs
+            yarn run playwright install chromium
+            yarn run test:e2e -x --workers=4
+
+  test-backend:
+    docker:
+      - image: penpotapp/devenv:latest
+      - image: cimg/postgres:14.5
+        environment:
+          POSTGRES_USER: penpot_test
+          POSTGRES_PASSWORD: penpot_test
+          POSTGRES_DB: penpot_test
+      - image: cimg/redis:7.0.5
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "backend/deps.edn" }}
+
+      - run:
+          name: "tests"
+          working_directory: "./backend"
+          command: |
+            clojure -M:dev:test --reporter kaocha.report/documentation
+
+          environment:
+            PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
+            PENPOT_TEST_DATABASE_USERNAME: penpot_test
+            PENPOT_TEST_DATABASE_PASSWORD: penpot_test
+            PENPOT_TEST_REDIS_URI: "redis://localhost/1"
+
+      - save_cache:
+          paths:
+            - ~/.m2
+            - ~/.gitlibs
+          key: v1-dependencies-{{ checksum "backend/deps.edn" }}
+
+  test-render-wasm:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+    environment:
+
+    steps:
+      - checkout
+
+      - run:
+          name: "fmt check"
+          working_directory: "./render-wasm"
+          command: |
+            cargo fmt --check
+
+      - run:
+          name: "lint"
+          working_directory: "./render-wasm"
+          command: |
+            ./lint
+
+      - run:
+          name: "cargo tests"
+          working_directory: "./render-wasm"
+          command: |
+            ./test
+
+workflows:
+  penpot:
+    jobs:
+      - lint
+      - test-frontend:
+          requires:
+            - lint: success
+
+      - test-library:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-components:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-integration:
+          requires:
+            - test-frontend: success
+            - lint: success
+
+      - test-backend:
+          requires:
+            - lint: success
+
+      - test-common:
+          requires:
+            - lint: success
+
+      - test-render-wasm

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,7 +13,6 @@
 - [ ] Add a detailed explanation of how to reproduce the issue and/or verify the fix, if applicable.
 - [ ] Include screenshots or videos, if applicable.
 - [ ] Add or modify existing integration tests in case of bugs or new features, if applicable.
-- [ ] Refactor any modified SCSS files following the refactor guide.
 - [ ] Check CI passes successfully.
 - [ ] Update the `CHANGES.md` file, referencing the related GitHub issue, if applicable.
 

.github/workflows/build-bundle.yml

@@ -1,93 +0,0 @@
-name: Bundles Builder
-
-on:
-  # Create bundle from manual action
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-
-jobs:
-  build-bundle:
-    name: Build and Upload Penpot Bundle
-    runs-on: ubuntu-24.04
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.gh_ref }}
-
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-          echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT
-
-      - name: Build bundle
-        env:
-          BUILD_WASM: ${{ inputs.build_wasm }}
-          BUILD_STORYBOOK: ${{ inputs.build_storybook }}
-        run: ./manage.sh build-bundle
-
-      - name: Prepare directories for zipping
-        run: |
-          mkdir zips
-          mv bundles penpot
-
-      - name: Create zip bundle
-        run: |
-          echo "📦 Packaging Penpot bundle..."
-          zip -r zips/penpot.zip penpot
-
-      - name: Upload Penpot bundle to S3
-        run: |
-          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip --metadata bundle-version=${{ steps.vars.outputs.bundle_version }}
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            ❌ 📦 *[PENPOT] Error building penpot bundles.*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-               Bundle version: `${{ steps.vars.outputs.bundle_version }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra

.github/workflows/build-bundles.yml

@@ -0,0 +1,64 @@
+name: Build and Upload Penpot Bundles
+
+on:
+  # Create bundle from manual action
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: string
+        required: true
+
+jobs:
+  build-bundles:
+    name: Build and Upload Penpot Bundles
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gh_ref }}
+
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "gh_branch=${{ github.base_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx for multi-arch build
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run manage.sh build-bundle from host
+        run: ./manage.sh build-bundle
+
+      - name: Prepare directories for zipping
+        run: |
+          mkdir zips
+          mv bundles penpot
+
+      - name: Create zip bundles
+        run: |
+          echo "📦 Packaging Penpot bundles..."
+          zip -r zips/penpot.zip penpot
+
+      - name: Upload Penpot bundle to S3
+        run: |
+          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_branch}}-latest.zip
+          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.commit_hash }}.zip
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          TEXT: |
+            ❌ *[PENPOT] Error during the execution of the job*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_branch}}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/build-develop.yml

@@ -1,21 +1,12 @@
-name: _DEVELOP
+name: Build and Upload Penpot DEVELOP Bundles
 
 on:
   schedule:
     - cron: '16 5-20 * * 1-5'
 
 jobs:
-  build-bundle:
+  build-develop-bundle:
-    uses: ./.github/workflows/build-bundle.yml
+    uses: ./.github/workflows/build-bundles.yml
-    secrets: inherit
-    with:
-      gh_ref: "develop"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
     secrets: inherit
     with:
       gh_ref: "develop"

.github/workflows/build-docker-devenv.yml

@@ -1,36 +0,0 @@
-name: DevEnv Docker Image Builder
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build-and-push:
-    name: Build and push DevEnv Docker image
-    environment: release-admins
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.PUB_DOCKER_USERNAME }}
-          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      - name: Build and push DevEnv Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'penpotapp/devenv'
-        with:
-          context: ./docker/devenv/
-          file: ./docker/devenv/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ env.DOCKER_IMAGE }}:latest
-          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache,mode=max

.github/workflows/build-docker.yml

@@ -1,152 +0,0 @@
-name: Docker Images Builder
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-
-jobs:
-  build-and-push:
-    name: Build and Push Penpot Docker Images
-    runs-on: ubuntu-24.04-arm
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.gh_ref }}
-
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Download Penpot Bundles
-        id: bundles
-        env:
-          FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        run: |
-          tmp=$(aws s3api head-object \
-            --bucket ${{ secrets.S3_BUCKET }} \
-            --key "$FILE_NAME" \
-            --query 'Metadata."bundle-version"' \
-            --output text)
-          echo "bundle_version=$tmp" >> $GITHUB_OUTPUT
-          pushd docker/images
-          aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
-          unzip $FILE_NAME > /dev/null
-          mv penpot/backend bundle-backend
-          mv penpot/frontend bundle-frontend
-          mv penpot/exporter bundle-exporter
-          mv penpot/storybook bundle-storybook
-          popd
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.DOCKER_REGISTRY }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Extract metadata (tags, labels)
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images:
-            frontend
-            backend
-            exporter
-            storybook
-          labels: |
-            bundle_version=${{ steps.bundles.outputs.bundle_version }}
-
-      - name: Build and push Backend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'backend'
-          BUNDLE_PATH: './bundle-backend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.backend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Frontend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'frontend'
-          BUNDLE_PATH: './bundle-frontend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.frontend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Exporter Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'exporter'
-          BUNDLE_PATH: './bundle-exporter'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.exporter
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Storybook Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'storybook'
-          BUNDLE_PATH: './bundle-storybook'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.storybook
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            ❌ 🐳 *[PENPOT] Error building penpot docker images.*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-            📦 Bundle: `${{ steps.bundles.outputs.bundle_version }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra

.github/workflows/build-staging.yml

@@ -1,21 +1,12 @@
-name: _STAGING
+name: Build and Upload Penpot STAGING Bundles
 
 on:
   schedule:
-    - cron: '36 5-20 * * 1-5'
+    - cron: '0 5 * * 1-5'
 
 jobs:
-  build-bundle:
+  build-staging-bundle:
-    uses: ./.github/workflows/build-bundle.yml
+    uses: ./.github/workflows/build-bundles.yml
-    secrets: inherit
-    with:
-      gh_ref: "staging"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
     secrets: inherit
     with:
       gh_ref: "staging"

.github/workflows/build-tag.yml

@@ -1,45 +0,0 @@
-name: _TAG
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-      build_wasm: "no"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-
-  notify:
-    name: Notifications
-    needs: build-docker
-
-    steps:
-      - name: Notify Mattermost
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            🐳 *[PENPOT] Docker image available.*
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra
-
-  publish-final-tag:
-    if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta')  && contains(github.ref_name, '.') }}
-    needs: build-docker
-    uses: ./.github/workflows/release.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert).+[^.])$'
+          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/release.yml

@@ -1,114 +0,0 @@
-name: Release Publisher
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-
-permissions:
-  contents: write
-
-jobs:
-  release:
-    environment: release-admins
-    runs-on: ubuntu-24.04
-    outputs:
-      version: ${{ steps.vars.outputs.gh_ref }}
-      release_notes: ${{ steps.extract_release_notes.outputs.release_notes }}
-    steps:
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ steps.vars.outputs.gh_ref }}
-
-      # --- Publicly release the docker images ---
-      - name: Configure ECR credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
-          aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
-          aws-region: ${{ secrets.AWS_REGION }}
-
-      - name: Install Skopeo
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y skopeo
-
-      - name: Copy images from AWS ECR to Docker Hub
-        env:
-          AWS_REGION: ${{ secrets.AWS_REGION }}
-          DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
-          PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
-          PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-        run: |
-          aws ecr get-login-password --region $AWS_REGION | \
-            skopeo login --username AWS --password-stdin \
-            $DOCKER_REGISTRY
-
-          echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io
-
-          IMAGES=("frontend" "backend" "exporter" "storybook")
-
-          for image in "${IMAGES[@]}"; do
-            skopeo copy --all \
-              docker://$DOCKER_REGISTRY/$image:$TAG \
-              docker://docker.io/penpotapp/$image:$TAG
-
-            for alias in main latest; do
-              skopeo copy --all \
-                docker://$DOCKER_REGISTRY/$image:$TAG \
-                docker://docker.io/penpotapp/$image:$alias
-            done
-          done
-
-      # --- Release notes extraction ---
-      - name: Extract release notes from CHANGES.md
-        id: extract_release_notes
-        env:
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-        run: |
-          RELEASE_NOTES=$(awk "/^## $TAG$/{flag=1; next} /^## /{flag=0} flag" CHANGES.md | awk '{$1=$1};1')
-          if [ -z "$RELEASE_NOTES" ]; then
-            RELEASE_NOTES="No changes for $TAG according to CHANGES.md"
-          fi
-          echo "release_notes<<EOF" >> $GITHUB_OUTPUT
-          echo "$RELEASE_NOTES" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-
-      # --- Create GitHub release ---
-      - name: Create GitHub release
-        uses: softprops/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ steps.vars.outputs.gh_ref }}
-          name: ${{ steps.vars.outputs.gh_ref }}
-          body: ${{ steps.extract_release_notes.outputs.release_notes }}
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            ❌ 🚀 *[PENPOT] Error releasing penpot.*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra

.github/workflows/tests.yml

@@ -1,289 +0,0 @@
-name: "CI"
-
-defaults:
-  run:
-    shell: bash
-
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-  push:
-    branches:
-      - develop
-      - staging
-
-concurrency:
-  group: ${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    name: "Linter"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Check clojure code format
-        run: |
-          ./scripts/lint
-
-  test-common:
-    name: "Common Tests"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Run tests on JVM
-        working-directory: ./common
-        run: |
-          clojure -M:dev:test
-
-      - name: Run tests on NODE
-        working-directory: ./common
-        run: |
-          ./scripts/test
-
-  test-frontend:
-    name: "Frontend Tests"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Unit Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test
-
-      - name: Component Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test-components
-
-  test-render-wasm:
-    name: "Render WASM Tests"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Format
-        working-directory: ./render-wasm
-        run: |
-          cargo fmt --check
-
-      - name: Lint
-        working-directory: ./render-wasm
-        run: |
-          ./lint
-
-      - name: Test
-        working-directory: ./render-wasm
-        run: |
-          ./test
-
-  test-backend:
-    name: "Backend Tests"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    services:
-      postgres:
-        image: postgres:17
-        # Provide the password for postgres
-        env:
-          POSTGRES_USER: penpot_test
-          POSTGRES_PASSWORD: penpot_test
-          POSTGRES_DB: penpot_test
-
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-      redis:
-        image: valkey/valkey:9
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Run tests
-        working-directory: ./backend
-        env:
-          PENPOT_TEST_DATABASE_URI: "postgresql://postgres/penpot_test"
-          PENPOT_TEST_DATABASE_USERNAME: penpot_test
-          PENPOT_TEST_DATABASE_PASSWORD: penpot_test
-          PENPOT_TEST_REDIS_URI: "redis://redis/1"
-
-        run: |
-          clojure -M:dev:test --reporter kaocha.report/documentation
-
-  test-library:
-    name: "Library Tests"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Run tests
-        working-directory: ./library
-        run: |
-          ./scripts/test
-
-  build-integration:
-    name: "Build Integration Bundle"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Build Bundle
-        working-directory: ./frontend
-        run: |
-          ./scripts/build 0.0.0
-
-      - name: Store Bundle Cache
-        uses: actions/cache@v4
-        with:
-          key: "integration-bundle-${{ github.sha }}"
-          path: frontend/resources/public
-
-
-  test-integration-1:
-    name: "Integration Tests 1/4"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-    needs: build-integration
-
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Restore Cache
-        uses: actions/cache/restore@v4
-        with:
-          key: "integration-bundle-${{ github.sha }}"
-          path: frontend/resources/public
-
-      - name: Run Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test-e2e --shard="1/4";
-
-      - name: Upload test result
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: integration-tests-result-1
-          path: frontend/test-results/
-          overwrite: true
-          retention-days: 3
-
-  test-integration-2:
-    name: "Integration Tests 2/4"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-    needs: build-integration
-
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Restore Cache
-        uses: actions/cache/restore@v4
-        with:
-          key: "integration-bundle-${{ github.sha }}"
-          path: frontend/resources/public
-
-      - name: Run Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test-e2e --shard="2/4";
-
-      - name: Upload test result
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: integration-tests-result-2
-          path: frontend/test-results/
-          overwrite: true
-          retention-days: 3
-
-  test-integration-3:
-    name: "Integration Tests 3/4"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-    needs: build-integration
-
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Restore Cache
-        uses: actions/cache/restore@v4
-        with:
-          key: "integration-bundle-${{ github.sha }}"
-          path: frontend/resources/public
-
-      - name: Run Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test-e2e --shard="3/4";
-
-      - name: Upload test result
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: integration-tests-result-3
-          path: frontend/test-results/
-          overwrite: true
-          retention-days: 3
-
-  test-integration-4:
-    name: "Integration Tests 4/4"
-    runs-on: ubuntu-24.04
-    container: penpotapp/devenv:latest
-    needs: build-integration
-
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Restore Cache
-        uses: actions/cache/restore@v4
-        with:
-          key: "integration-bundle-${{ github.sha }}"
-          path: frontend/resources/public
-
-      - name: Run Tests
-        working-directory: ./frontend
-        run: |
-          ./scripts/test-e2e --shard="4/4";
-
-      - name: Upload test result
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: integration-tests-result-4
-          path: frontend/test-results/
-          overwrite: true
-          retention-days: 3

.gitignore

@@ -31,7 +31,6 @@
 /.clj-kondo/.cache
 /_dump
 /notes
-/playground/
 /backend/*.md
 /backend/*.sql
 /backend/*.txt
@@ -80,4 +79,3 @@ node_modules
 /playwright/.cache/
 /render-wasm/target/
 /**/.yarn/*
-/.pnpm-store

.nvmrc

@@ -1 +1 @@
-v22.19.0
+v22.13.1

CHANGES.md

@@ -1,236 +1,16 @@
 # CHANGELOG
 
-## 2.13.0 (Unreleased)
+## 2.10.0 (Unreleased)
-
-### :boom: Breaking changes & Deprecations
 
 ### :rocket: Epics and highlights
 
-### :heart: Community contributions (Thank you!)
-
-### :sparkles: New features & Enhancements
-
-### :bug: Bugs fixed
-
-- Fix problem when drag+duplicate a full grid [Taiga #12565](https://tree.taiga.io/project/penpot/issue/12565)
-- Fix problem when pasting elements in reverse flex layout [Taiga #12460](https://tree.taiga.io/project/penpot/issue/12460)
-
-
-## 2.12.0 (Unreleased)
-
-### :boom: Breaking changes & Deprecations
-
-#### Backend RPC API changes
-
-The backend RPC API URLS are changed from `/api/rpc/command/<name>` to
-`/api/main/methods/<name>`. The previous PATH is preserved for backward
-compatibility; however, if you are a user of this API, it is strongly
-recommended that you adapt your code to use the new PATH.
-
-
-#### Updated SSO Callback URL
-
-The OAuth / Single Sign-On (SSO) callback endpoint has changed to
-align with the new OpenID Connect (OIDC) implementation.
-
-Old callback URL:
-
-```
-https://<your_domain>/api/auth/oauth/<oauth_provider>/callback
-```
-
-New callback URL:
-
-```
-https://<your_domain>/api/auth/oidc/callback
-```
-
-**Action required:**
-
-If you have SSO/Social-Auth configured on your on-premise instance,
-the following actions are required before update:
-
-Update your OAuth or SSO provider configuration (e.g., Okta, Google,
-Azure AD, etc.) to use the new callback URL. Failure to update may
-result in authentication failures after upgrading.
-
-**Reason for change:**
-
-This update standardizes all authentication flows under the single URL
-and makis it more modular, enabling the ability to configure SSO auth
-provider dinamically.
-
-
-#### Changes on default docker compose
-
-We have updated the `docker/images/docker-compose.yaml` with a small
-change related to the `PENPOT_SECRET_KEY`. Since this version, this
-environment variable is also required on exporter. So if you are using
-penpot on-premise you will need to apply the same changes on your own
-`docker-compose.yaml` file.
-
-We have removed the Minio server from the `docker/images/docker-compose.yml`
-example. It's still usable as before, we just removed the example.
-
-### :rocket: Epics and highlights
-
-### :heart: Community contributions (Thank you!)
-
-- Ensure consistent snap behavior across all zoom levels [Github #7774](https://github.com/penpot/penpot/pull/7774) by [@Tokytome](https://github.com/Tokytome)
-
-### :sparkles: New features & Enhancements
-
-- Add the ability to select boards to export as PDF [Taiga #12320](https://tree.taiga.io/project/penpot/issue/12320)
-- Add toggle for switching boolean property values [Taiga #12341](https://tree.taiga.io/project/penpot/us/12341)
-- Make the file export process more reliable [Taiga #12555](https://tree.taiga.io/project/penpot/us/12555)
-- Add auth flow changes [Taiga #12333](https://tree.taiga.io/project/penpot/us/12333)
-- Add new shape validation mechanism for shapes [Github #7696](https://github.com/penpot/penpot/pull/7696)
-- Apply color tokens from sidebar [Taiga #11353](https://tree.taiga.io/project/penpot/us/11353)
-- Display tokens in the inspect tab [Taiga #9313](https://tree.taiga.io/project/penpot/us/9313)
-- Refactor clipboard behavior to assess some minor inconsistencies and make pasting binary data faster. [Taiga #12571](https://tree.taiga.io/project/penpot/task/12571)
-
-### :bug: Bugs fixed
-
-- Fix text line-height values are wrong [Taiga #12252](https://tree.taiga.io/project/penpot/issue/12252)
-- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
-- Fix pan cursor not disabling viewport guides [Github #6985](https://github.com/penpot/penpot/issues/6985)
-- Fix viewport resize on locked shapes [Taiga #11974](https://tree.taiga.io/project/penpot/issue/11974)
-- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
-- Fix on copy instance inside a components chain touched are missing [Taiga #12371](https://tree.taiga.io/project/penpot/issue/12371)
-- Fix problem with multiple selection and shadows [Github #7437](https://github.com/penpot/penpot/issues/7437)
-- Fix search shortcut [Taiga #10265](https://tree.taiga.io/project/penpot/issue/10265)
-- Fix shortcut conflict in text editor (increase/decrease font size vs word selection)
-- Fix problem with plugins generating code for pages different than current one [Taiga #12312](https://tree.taiga.io/project/penpot/issue/12312)
-- Fix input confirmation behavior is not uniform [Taiga #12294](https://tree.taiga.io/project/penpot/issue/12294)
-- Fix copy/pasting application/transit+json [Taiga #12721](https://tree.taiga.io/project/penpot/issue/12721)
-- Fix problem with plugins content attribute [Plugins #209](https://github.com/penpot/penpot-plugins/issues/209)
-
-## 2.11.1
-
-- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
-
-## 2.11.0
-
-### :boom: Breaking changes & Deprecations
-
-- Deprecated configuration variables with the prefix `PENPOT_ASSETS_*`, and will be
-  removed in future versions:
-
-  - The `PENPOT_ASSETS_STORAGE_BACKEND` becomes `PENPOT_OBJECTS_STORAGE_BACKEND` and its
-    values passes from (`assets-fs` or `assets-s3`) to (`fs` or `s3`)
-  - The `PENPOT_STORAGE_ASSETS_FS_DIRECTORY` becomes `PENPOT_OBJECTS_STORAGE_FS_DIRECTORY`
-  - The `PENPOT_STORAGE_ASSETS_S3_BUCKET` becomes `PENPOT_OBJECTS_STORAGE_S3_BUCKET`
-  - The `PENPOT_STORAGE_ASSETS_S3_REGION` becomes `PENPOT_OBJECTS_STORAGE_S3_REGION`
-  - The `PENPOT_STORAGE_ASSETS_S3_ENDPOINT` becomes `PENPOT_OBJECTS_STORAGE_S3_ENDPOINT`
-  - The `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` replaced (see below)
-
-- Add `PENPOT_NETTY_IO_THREADS` and `PENPOT_EXECUTOR_THREADS` variables to provide the
-  control over concurrency of the shared resources used by netty. Penpot uses the netty IO
-  threads for AWS S3 SDK and Redis/Valkey communication, and the EXEC threads to perform
-  out of HTTP serving threads tasks such that cache invalidation, S3 response completion,
-  configuration reloading and many other auxiliar tasks. By default they use a half number
-  if available cpus with a minumum of 2 for both executors. You should not touch that
-  variables unless you are know what you are doing.
-
-- Replace the `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` with a more general configuration
-  `PENPOT_NETTY_IO_THREADS` used to configure a shared netty resources across different
-  services which use netty internally (redis connection, S3 SDK client). This
-  configuration is not very commonly used so don't expected real impact on any user.
-
-### :sparkles: New features & Enhancements
-
-- New composite token: Typography [Taiga #10200](https://tree.taiga.io/project/penpot/us/10200)
-- Show current Penpot version [Taiga #11603](https://tree.taiga.io/project/penpot/us/11603)
-- Switch several variant copies at the same time [Taiga #11411](https://tree.taiga.io/project/penpot/us/11411)
-- Invitations management improvements [Taiga #3479](https://tree.taiga.io/project/penpot/us/3479)
-- Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
-- Reorder properties for a component [Taiga #10225](https://tree.taiga.io/project/penpot/us/10225)
-- File Data storage layout refactor [Github #7345](https://github.com/penpot/penpot/pull/7345)
-- Make several queries optimization on comment threads [Github #7506](https://github.com/penpot/penpot/pull/7506)
-
-### :bug: Bugs fixed
-
-- Fix selection problems when devtools open [Taiga #11950](https://tree.taiga.io/project/penpot/issue/11950)
-- Fix long font names overlap [Taiga #11844](https://tree.taiga.io/project/penpot/issue/11844)
-- Fix paste behavior according to the selected element [Taiga #11979](https://tree.taiga.io/project/penpot/issue/11979)
-- Fix problem with export size [Github #7160](https://github.com/penpot/penpot/issues/7160)
-- Fix multi level library dependencies [Taiga #12155](https://tree.taiga.io/project/penpot/issue/12155)
-- Fix component context menu options order in assets tab [Taiga #11941](https://tree.taiga.io/project/penpot/issue/11941)
-- Fix error updating library [Taiga #12218](https://tree.taiga.io/project/penpot/issue/12218)
-- Fix restoring a variant in another file makes it overlap the existing variant [Taiga #12049](https://tree.taiga.io/project/penpot/issue/12049)
-- Fix auto-width changes to fixed when switching variants [Taiga #12172](https://tree.taiga.io/project/penpot/issue/12172)
-- Fix component number has no singular translation string [Taiga #12106](https://tree.taiga.io/project/penpot/issue/12106)
-- Fix adding/removing identical text fills [Taiga #12287](https://tree.taiga.io/project/penpot/issue/12287)
-- Fix scroll on the inspect tab [Taiga #12293](https://tree.taiga.io/project/penpot/issue/12293)
-- Fix lock proportion tooltip [Taiga #12326](https://tree.taiga.io/project/penpot/issue/12326)
-- Fix internal Error when selecting a set by name in the token theme editor [Taiga #12310](https://tree.taiga.io/project/penpot/issue/12310)
-- Fix drag & drop functionality is swapping instead or reordering [Taiga #12254](https://tree.taiga.io/project/penpot/issue/12254)
-- Fix variants not syncronizing tokens on switch [Taiga #12290](https://tree.taiga.io/project/penpot/issue/12290)
-- Fix incorrect behavior of Alt + Drag for variants [Taiga #12309](https://tree.taiga.io/project/penpot/issue/12309)
-- Fix text override is lost after switch [Taiga #12269](https://tree.taiga.io/project/penpot/issue/12269)
-- Fix exporting a board crashing the app [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12384)
-- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
-- Fix selected colors not showing colors from children shapes in multiple selection [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12385)
-- Fix scrollbar issue in design tab [Taiga #12367](https://tree.taiga.io/project/penpot/issue/12367)
-- Fix library update notificacions showing when they should not [Taiga #12397](https://tree.taiga.io/project/penpot/issue/12397)
-- Fix remove flex button doesn’t work within variant [Taiga #12314](https://tree.taiga.io/project/penpot/issue/12314)
-- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
-- Fix problem with certain text input in some editable labels (pages, components, tokens...) being in conflict with the drag/drop functionality [Taiga #12316](https://tree.taiga.io/project/penpot/issue/12316)
-- Fix not controlled theme renaming [Taiga #12411](https://tree.taiga.io/project/penpot/issue/12411)
-- Fix paste without selection sends the new element in the back [Taiga #12382](https://tree.taiga.io/project/penpot/issue/12382)
-- Fix options button does not work for comments created in the lower part of the screen [Taiga #12422](https://tree.taiga.io/project/penpot/issue/12422)
-- Fix problem when checking usage with removed teams [Taiga #12442](https://tree.taiga.io/project/penpot/issue/12442)
-- Fix focus mode persisting across page/file navigation [Taiga #12469](https://tree.taiga.io/project/penpot/issue/12469)
-- Fix shadow color validation [Github #7705](https://github.com/penpot/penpot/pull/7705)
-- Fix exception on selection blend-mode using keyboard [Github #7710](https://github.com/penpot/penpot/pull/7710)
-- Fix crash when using decimal (floating-point) values for X/Y or width/height [Taiga #12543](https://tree.taiga.io/project/penpot/issue/12543)
-
-## 2.10.1
-
-### :sparkles: New features & Enhancements
-
-- Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
-
-### :bug: Bugs fixed
-
-- Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
-
-## 2.10.0
-
-### :rocket: Epics and highlights
-
-- Variants
-
 ### :boom: Breaking changes & Deprecations
 
 ### :heart: Community contributions (Thank you!)
 
 ### :sparkles: New features & Enhancements
-
-- Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
 - Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
 - Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
-- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
-- New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
-- New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
-- New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
-- New letter spacing token [Taiga #10940](https://tree.taiga.io/project/penpot/us/10940)
-- New font weight token [Taiga #10939](https://tree.taiga.io/project/penpot/us/10939)
-- Upgrade Node to v22.18.0 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Upgrade the base docker image for penpot frontend to v1.29.1 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Create variant from an existing component [Taiga #2088](https://tree.taiga.io/project/penpot/us/2088)
-- Create variant from an existing variant [Taiga #8282](https://tree.taiga.io/project/penpot/us/8282)
-- Actions over a component with variants [Taiga #10503](https://tree.taiga.io/project/penpot/us/10503)
-- Create a variant by dragging a component into a component with variants [Taiga #8134](https://tree.taiga.io/project/penpot/us/8134)
-- Transform a variant into an individual component [Taiga #8141](https://tree.taiga.io/project/penpot/us/8141)
-- Delete variant [Taiga #6890](https://tree.taiga.io/project/penpot/us/6890)
-- Restore an orphaned copy of a variant [Taiga #10446](https://tree.taiga.io/project/penpot/us/10446)
-- Add, Edit & Delete variant properties name and value [Taiga #6892](https://tree.taiga.io/project/penpot/us/6892)
-- Retrieve variants [Taiga #6888](https://tree.taiga.io/project/penpot/us/6888)
-- Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
-- Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
-- Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
-- Fix problem with component swapping panel [Taiga #12175](https://tree.taiga.io/project/penpot/issue/12175)
 
 ### :bug: Bugs fixed
 
@@ -243,25 +23,8 @@ example. It's still usable as before, we just removed the example.
 - Fix font size/variant not updated when editing a text [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11552)
 - Fix issue where Alt + arrow keys shortcut interferes with letter-spacing when moving text layers [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11771)
 - Fix consistency issues on how font variants are visualized [Taiga #11499](https://tree.taiga.io/project/penpot/us/11499)
-- Fix parsing rx and ry SVG values for rect radius [Taiga #11861](https://tree.taiga.io/project/penpot/issue/11861)
-- Fix misleading affordance in saved versions [Taiga #11887](https://tree.taiga.io/project/penpot/issue/11887)
-- Fix pasting RTF text crashes penpot [Taiga #11717](https://tree.taiga.io/project/penpot/issue/11717)
-- Fix navigation arrows in Libraries & Templates carousel [Taiga #10609](https://tree.taiga.io/project/penpot/issue/10609)
-- Fix applying tokens with zero value to size [Taiga #11618](https://tree.taiga.io/project/penpot/issue/11618)
-- Fix typo [Taiga #11969](https://tree.taiga.io/project/penpot/issue/11969)
-- Fix typo [Taiga #11970](https://tree.taiga.io/project/penpot/issue/11970)
-- Fix typos [Taiga #11971](https://tree.taiga.io/project/penpot/issue/11971)
-- Fix inconsistent naming for "Flatten" [Taiga #8371](https://tree.taiga.io/project/penpot/issue/8371)
-- Layout item tokens should be unapplied when moving out of a layout [Taiga #11012](https://tree.taiga.io/project/penpot/issue/11012)
-- Fix incorrect date displayed for support plan [Taiga #11986](https://tree.taiga.io/project/penpot/issue/11986)
-- Fix can't import 'borderWidth' type token [#132](https://github.com/tokens-studio/penpot/issues/132)
-- Fix moving elements up or down while pressing alt [Taiga Issue #11992](https://tree.taiga.io/project/penpot/issue/11992)
-- Fix conflicting shortcuts (remove dec/inc line height and letter spacing) [Taiga #12102](https://tree.taiga.io/project/penpot/issue/12102)
-- Fix conflicting shortcuts (remove text-align shortcuts) [Taiga #12047](https://tree.taiga.io/project/penpot/issue/12047)
-- Fix export file with empty tokens library [Taiga #12137](https://tree.taiga.io/project/penpot/issue/12137)
-- Fix context menu on spacing tokens [Taiga #12141](https://tree.taiga.io/project/penpot/issue/12141)
 
-## 2.9.0
+## 2.9.0 (Unreleased)
 
 ### :rocket: Epics and highlights
 
@@ -289,7 +52,6 @@ example. It's still usable as before, we just removed the example.
 - Add the ability to show login dialog on profile settings [Github #6871](https://github.com/penpot/penpot/pull/6871)
 - Improve the application of tokens with object specific tokens [Taiga #10209](https://tree.taiga.io/project/penpot/us/10209)
 - Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
-- Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
 
 ### :bug: Bugs fixed
 
@@ -320,7 +82,6 @@ example. It's still usable as before, we just removed the example.
 - Fix export button width on inspect tab [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
 - Fix stroke width token application [Taiga #11724](https://tree.taiga.io/project/penpot/issue/11724)
 - Fix number token application on shape [Taiga #11331](https://tree.taiga.io/project/penpot/task/11331)
-- Fix auto height is fixed in the HTML inspect tab for text elements [Taiga #11680](https://tree.taiga.io/project/penpot/task/11680)
 
 ## 2.8.1
 
@@ -338,7 +99,7 @@ example. It's still usable as before, we just removed the example.
 
 **Penpot Library**
 
-The initial prototype is completly reworked to provide a more consistent API
+The initial prototype is completly reworked for provide a more consistent API
 and to have proper validation and params decoding. All the details can be found
 on [its own changelog](library/CHANGES.md)
 

README.md

@@ -77,14 +77,17 @@ Provide your team or organization with a completely owned collaborative design t
 ### Integrations ###
 Penpot offers integration into the development toolchain, thanks to its support for webhooks and an API accessible through access tokens.
 
-###  Building Design Systems: design tokens, components and variants  ###
+### What’s great for design ###
-Penpot brings design systems to code-minded teams: a single source of truth with native Design Tokens, Components, and Variants for scalable, reusable, and consistent UI across projects and platforms.
+With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
+
+### Design Tokens ###
+With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
 
 
 <br />
 
 <p align="center">
- <img src="https://github.com/user-attachments/assets/cce75ad6-f783-473f-8803-da9eb8255fef">
+  <img src="https://img.plasmic.app/img-optimizer/v1/img?src=https%3A%2F%2Fimg.plasmic.app%2Fimg-optimizer%2Fv1%2Fimg%2F9dd677c36afb477e9666ccd1d3f009ad.png" alt="Open Source" style="width: 65%;">
 </p>
 
 <br />

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.2"}
+  org.clojure/clojure {:mvn/version "1.12.1"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.7-4"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,7 +17,7 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.8.1.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.7.0.RELEASE"}
   ;; Minimal dependencies required by lettuce, we need to include them
   ;; explicitly because clojure dependency management does not support
   ;; yet the BOM format.
@@ -28,30 +28,29 @@
   com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.8"
+  {:git/tag "v11.4"
-   :git/sha "1d1b33f"
+   :git/sha "ce50d42"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
   com.github.seancorfield/next.jdbc
-  {:mvn/version "1.3.1070"}
+  {:mvn/version "1.3.1002"}
-
   metosin/reitit-core {:mvn/version "0.9.1"}
-  nrepl/nrepl {:mvn/version "1.4.0"}
+  nrepl/nrepl {:mvn/version "1.3.1"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.7"}
+  org.postgresql/postgresql {:mvn/version "42.7.6"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "7.0.2"}
+  com.zaxxer/HikariCP {:mvn/version "6.3.0"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.2"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.21.2"}
+  org.jsoup/jsoup {:mvn/version "1.20.1"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -61,12 +60,12 @@
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
-  dawran6/emoji {:mvn/version "0.2.0"}
+  dawran6/emoji {:mvn/version "0.1.5"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.4"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.3"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.33.10"}}
+  software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -81,14 +80,12 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:mvn/version "0.10.10"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
    :ns-default build}
 
   :test
   {:main-opts ["-m" "kaocha.runner"]
-   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
+   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"]
-              "--sun-misc-unsafe-memory-access=allow"
-              "--enable-native-access=ALL-UNNAMED"]
    :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
 
   :outdated

backend/dev/user.clj

@@ -6,14 +6,12 @@
 
 (ns user
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.debug :as debug]
    [app.common.exceptions :as ex]
    [app.common.files.helpers :as cfh]
    [app.common.fressian :as fres]
    [app.common.geom.matrix :as gmt]
-   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.perf :as perf]
    [app.common.pprint :as pp]
@@ -21,22 +19,20 @@
    [app.common.schema.desc-js-like :as smdj]
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.generators :as sg]
-   [app.common.schema.openapi :as oapi]
    [app.common.spec :as us]
-   [app.common.time :as ct]
+   [app.common.json :as json]
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]
-   [app.srepl.helpers :as h]
+   [app.srepl.helpers :as srepl.helpers]
-   [app.srepl.main :refer :all]
+   [app.srepl.main :as srepl]
    [app.util.blob :as blob]
+   [app.common.time :as ct]
    [clj-async-profiler.core :as prof]
    [clojure.contrib.humanize :as hum]
-   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]

backend/resources/app/email/feedback/en.html

@@ -8,41 +8,38 @@
   <body>
     <p>
       <strong>Feedback from:</strong><br />
-      <span>
+      {% if profile %}
-        <span>Name: </span>
+        <span>
-        <span><code>{{profile.fullname|abbreviate:25}}</code></span>
+          <span>Name: </span>
-      </span>
+          <span><code>{{profile.fullname|abbreviate:25}}</code></span>
-      <br />
+        </span>
-      <span>
+        <br />
-        <span>Email: </span>
+
-        <span>{{profile.email}}</span>
+        <span>
-      </span>
+          <span>Email: </span>
-      <br />
+          <span>{{profile.email}}</span>
-      <span>
+        </span>
-        <span>ID: </span>
+        <br />
-        <span><code>{{profile.id}}</code></span>
+
-      </span>
+        <span>
+          <span>ID: </span>
+          <span><code>{{profile.id}}</code></span>
+        </span>
+      {% else %}
+        <span>
+          <span>Email: </span>
+          <span>{{profile.email}}</span>
+        </span>
+      {% endif %}
     </p>
     <p>
       <strong>Subject:</strong><br />
-      <span>{{feedback-subject|abbreviate:300}}</span>
+      <span>{{subject|abbreviate:300}}</span>
     </p>
 
-    <p>
-      <strong>Type:</strong><br />
-      <span>{{feedback-type|abbreviate:300}}</span>
-    </p>
-
-    {% if feedback-error-href %}
-    <p>
-      <strong>Error HREF:</strong><br />
-      <span>{{feedback-error-href|abbreviate:500}}</span>
-    </p>
-    {% endif %}
-
     <p>
       <strong>Message:</strong><br />
-      {{feedback-content|linebreaks-br}}
+      {{content|linebreaks-br|safe}}
     </p>
   </body>
 </html>

backend/resources/app/email/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{feedback-subject}}
+[PENPOT FEEDBACK]: {{subject}}

backend/resources/app/email/feedback/en.txt

@@ -1,11 +1,9 @@
-From: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+{% if profile %}
-Subject: {{feedback-subject}}
+Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
-Type: {{feedback-type}}
+{% else %}
+Feedback from: {{email}}
+{% endif %}
 
-{% if feedback-error-href %}
+Subject: {{subject}}
-HREF: {{feedback-error-href}}
-{% endif -%}
 
-Message:
+{{content}}
-
-{{feedback-content}}

backend/resources/app/email/invite-to-team/en.subj

@@ -1 +1 @@
-{{invited-by|abbreviate:25}} has invited you to join the team “{{ team|abbreviate:25 }}”
+Invitation to join {{team}}

backend/resources/app/onboarding.edn

@@ -1,9 +1,6 @@
 [{:id "tokens-starter-kit"
   :name "Design tokens starter kit"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"}
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"},
- {:id "penpot-design-system"
-  :name "Penpot Design System | Pencil"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/penpot-app.penpot"}
  {:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}
@@ -13,6 +10,9 @@
  {:id "plants-app"
   :name "UI mockup example"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/Plants-app.penpot"}
+ {:id "penpot-design-system"
+  :name "Design system example"
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Penpot%20-%20Design%20System%20v2.1.penpot"}
  {:id "tutorial-for-beginners"
   :name "Tutorial for beginners"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/tutorial-for-beginners.penpot"}

backend/resources/app/templates/api-doc.tmpl

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="robots" content="noindex,nofollow">
     <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>{{label|upper}} API Documentation</title>
+    <title>Builtin API Documentation - Penpot</title>
 
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
@@ -19,7 +19,7 @@
   <body>
     <main>
       <header>
-        <h1>{{label|upper}}: API Documentation (v{{version}})</h1>
+        <h1>Penpot API Documentation (v{{version}})</h1>
         <small class="menu">
           [
           <nav>
@@ -31,10 +31,9 @@
       </header>
       <section class="doc-content">
         <h2>INTRODUCTION</h2>
-        <p>This documentation is intended to be a general overview of
+        <p>This documentation is intended to be a general overview of the penpot RPC API.
-          the {{label}} API.  If you prefer, you can
+          If you prefer, you can use <a href="/api/openapi.json">OpenAPI</a>
-          use <a href="{{openapi}}">Swagger/OpenAPI</a> as
+          and/or <a href="/api/openapi">SwaggerUI</a> as alternative.</p>
-          alternative.</p>
 
         <h2>GENERAL NOTES</h2>
 
@@ -44,7 +43,7 @@
         that starts with <b>get-</b> in the name, can use GET HTTP
         method which in many cases benefits from the HTTP cache.</p>
 
-        {% block auth-section %}
+
         <h3>Authentication</h3>
         <p>The penpot backend right now offers two way for authenticate the request:
         <b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
@@ -57,10 +56,9 @@
         <p>The access token can be obtained on the appropriate section on profile settings
         and it should be provided using <b>`Authorization`</b> header with <b>`Token
         &lt;token-string&gt;`</b> value.</p>
-        {% endblock %}
 
         <h3>Content Negotiation</h3>
-        <p>This API operates indistinctly with: <b>`application/json`</b>
+        <p>The penpot API by default operates indistinctly with: <b>`application/json`</b>
         and <b>`application/transit+json`</b> content types. You should specify the
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
@@ -77,16 +75,13 @@
         standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
         API</a></p>
 
-        {% block limits-section %}
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share
         the same rate limit). For now the limits are not documented because we are
         studying and analyzing the data. As a general rule, it should not be abused, if an
         abusive use is detected, we will proceed to block the user's access to the
         API.</p>
-        {% endblock %}
 
-        {% block webhooks-section %}
         <h3>Webhooks</h3>
         <p>All methods that emit webhook events are marked with flag <b>WEBHOOK</b>, the
         data structure defined on each method represents the <i>payload</i> of the
@@ -102,11 +97,9 @@
   "profileId": "db601c95-045f-808b-8002-361312e63531"
 }
         </pre>
-        {% endblock %}
-
       </section>
       <section class="rpc-doc-content">
-        <h2>METHODS REFERENCE:</h2>
+        <h2>RPC METHODS REFERENCE:</h2>
         <ul class="rpc-items">
           {% for item in methods %}
             {% include "app/templates/api-doc-entry.tmpl" with item=item %}

backend/resources/app/templates/debug.tmpl

@@ -45,41 +45,7 @@ Debug Main Page
       </form>
     </fieldset>
 
-    <fieldset>
-      <legend>VIRTUAL CLOCK</legend>
 
-      <desc>
-        <p>
-          CURRENT CLOCK: <b>{{current-clock}}</b>
-          <br />
-          CURRENT OFFSET: <b>{{current-offset}}</b>
-          <br />
-          CURRENT TIME: <b>{{current-time}}</b>
-        </p>
-
-        <p>Examples: 3h, -7h, 24h (allowed suffixes: h, s)</p>
-      </desc>
-
-      <form method="post" action="/dbg/actions/set-virtual-clock">
-        <div class="row">
-          <input type="text" name="offset" placeholder="3h" value="" />
-        </div>
-
-        <div class="row">
-          <label for="force-verify">Are you sure?</label>
-          <input id="force-verify" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-        <div class="row">
-          <input type="submit" name="submit" value="Submit" />
-          <input type="submit" name="reset" value="Reset" />
-        </div>
-      </form>
-    </fieldset>
 
   </section>
 

backend/resources/app/templates/main-api-doc.tmpl

@@ -1 +0,0 @@
-{% extends "app/templates/api-doc.tmpl" %}

backend/resources/app/templates/management-api-doc.tmpl

@@ -1,10 +0,0 @@
-{% extends "app/templates/api-doc.tmpl" %}
-
-{% block auth-section %}
-{% endblock %}
-
-{% block limits-section %}
-{% endblock %}
-
-{% block webhooks-section %}
-{% endblock %}

backend/resources/app/templates/openapi.tmpl

@@ -7,7 +7,7 @@
     name="description"
     content="SwaggerUI"
   />
-  <title>{{label|upper}} API</title>
+  <title>PENPOT Swagger UI</title>
   <style>{{swagger-css|safe}}</style>
 </head>
 <body>
@@ -16,7 +16,7 @@
 <script>
   window.onload = () => {
     window.ui = SwaggerUIBundle({
-      url: '{{uri}}',
+      url: '{{public-uri}}/api/openapi.json',
       dom_id: '#swagger-ui',      
       presets: [
         SwaggerUIBundle.presets.apis,

backend/resources/log4j2-devenv-repl.xml

@@ -25,7 +25,8 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http" level="info" />
+    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/resources/log4j2-devenv.xml

@@ -25,7 +25,8 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http" level="info" />
+    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/scripts/_env

@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-
-export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key
-export PENPOT_SECRET_KEY=super-secret-devenv-key
-export PENPOT_HOST=devenv
-
-export PENPOT_FLAGS="\
-       $PENPOT_FLAGS \
-       enable-login-with-password
-       disable-login-with-ldap \
-       disable-login-with-oidc \
-       disable-login-with-google \
-       disable-login-with-github \
-       disable-login-with-gitlab \
-       enable-backend-worker \
-       enable-backend-asserts \
-       disable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
-       enable-audit-log \
-       enable-transit-readable-response \
-       enable-demo-users \
-       enable-user-feedback \
-       disable-secure-session-cookies \
-       enable-smtp \
-       enable-prepl-server \
-       enable-urepl-server \
-       enable-rpc-climit \
-       enable-rpc-rlimit \
-       enable-quotes \
-       enable-soft-rpc-rlimit \
-       enable-auto-file-snapshot \
-       enable-webhooks \
-       enable-access-tokens \
-       disable-tiered-file-data-storage \
-       enable-file-validation \
-       enable-file-schema-validation \
-       enable-redis-cache \
-       enable-subscriptions";
-
-# Default deletion delay for devenv
-export PENPOT_DELETION_DELAY="24h"
-
-# Setup default upload media file size to 100MiB
-export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
-
-# Setup default multipart upload size to 300MiB
-export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
-
-export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com"
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-export PENPOT_OBJECTS_STORAGE_BACKEND=s3
-export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
-export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
-
-export JAVA_OPTS="\
-       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -Djdk.attach.allowAttachSelf \
-       -Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -Djdk.tracePinnedThreads=full \
-       -Dim4java.useV7=true \
-       -XX:+UnlockExperimentalVMOptions \
-       -XX:+UseShenandoahGC \
-       -XX:+UseCompactObjectHeaders \
-       -XX:ShenandoahGCMode=generational \
-       -XX:-OmitStackTraceInFastThrow  \
-       --sun-misc-unsafe-memory-access=allow \
-       --enable-preview \
-       --enable-native-access=ALL-UNNAMED";
-
-function setup_minio() {
-    # Initialize MINIO config
-    mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
-    mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-    mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-    if [ "$?" = "1" ]; then
-        mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-    fi
-    mc mb penpot-s3/penpot -p -q
-}
-
-

backend/scripts/repl

@@ -1,13 +1,112 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
+export PENPOT_SECRET_KEY=super-secret-devenv-key
-source $SCRIPT_DIR/_env;
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-login-with-ldap \
+       enable-login-with-password
+       enable-login-with-oidc \
+       enable-login-with-google \
+       enable-login-with-github \
+       enable-login-with-gitlab \
+       enable-backend-worker \
+       enable-backend-asserts \
+       disable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-audit-log \
+       enable-transit-readable-response \
+       enable-demo-users \
+       disable-secure-session-cookies \
+       enable-smtp \
+       enable-prepl-server \
+       enable-urepl-server \
+       enable-rpc-climit \
+       enable-rpc-rlimit \
+       enable-quotes \
+       enable-soft-rpc-rlimit \
+       enable-auto-file-snapshot \
+       enable-webhooks \
+       enable-access-tokens \
+       disable-tiered-file-data-storage \
+       enable-file-validation \
+       enable-file-schema-validation \
+       enable-subscriptions \
+       disable-subscriptions-old";
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
+# export PENPOT_DATABASE_USERNAME="penpot"
+# export PENPOT_DATABASE_PASSWORD="penpot"
+# export PENPOT_DATABASE_READONLY=true
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
+# export PENPOT_DATABASE_USERNAME="penpot_pre"
+# export PENPOT_DATABASE_PASSWORD="penpot_pre"
+
+# export PENPOT_LOGGERS_LOKI_URI="http://172.17.0.1:3100/loki/api/v1/push"
+# export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
 
 # Initialize MINIO config
-setup_minio;
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
+
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow  \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
-export JAVA_OPTS="$JAVA_OPTS -Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
 export OPTIONS="-A:jmx-remote -A:dev"
+
+# Setup HEAP
+# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
+# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
+
+# Increase virtual thread pool size
+# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
+
+# Disable C2 Compiler
+# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
+
+# Disable all compilers
+# export OPTIONS="$OPTIONS -J-Xint"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
+
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 

backend/scripts/repl-test

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+source /home/penpot/environ
+export PENPOT_FLAGS="$PENPOT_FLAGS disable-backend-worker"
+
+export OPTIONS="
+       -A:jmx-remote -A:dev \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-Dlog4j2.configurationFile=log4j2-experiments.xml \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints \
+       -J-Djdk.tracePinnedThreads=full \
+       -J-XX:+UseTransparentHugePages \
+       -J-XX:ReservedCodeCacheSize=1g \
+       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
+       -J--enable-preview";
+
+# Setup HEAP
+export OPTIONS="$OPTIONS -J-Xms320g -J-Xmx320g -J-XX:+AlwaysPreTouch"
+
+export PENPOT_HTTP_SERVER_IO_THREADS=2
+export PENPOT_HTTP_SERVER_WORKER_THREADS=2
+
+# Increase virtual thread pool size
+# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
+
+# Disable C2 Compiler
+# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
+
+# Disable all compilers
+# export OPTIONS="$OPTIONS -J-Xint"
+
+# Setup GC
+export OPTIONS="$OPTIONS -J-XX:+UseG1GC -J-Xlog:gc:logs/gc.log"
+
+# Setup GC
+#export OPTIONS="$OPTIONS -J-XX:+UseZGC -J-XX:+ZGenerational -J-Xlog:gc:logs/gc.log"
+
+# Enable ImageMagick v7.x support
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
+
+export OPTIONS_EVAL="nil"
+# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
+
+set -ex
+exec clojure $OPTIONS -M -e "$OPTIONS_EVAL" -m rebel-readline.main

backend/scripts/run

@@ -1,13 +1,44 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-backend-asserts \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-file-snapshot \
+       enable-tiered-file-data-storage";
 
-source $SCRIPT_DIR/_env;
+export JAVA_OPTS="
-export OPTIONS="-A:dev"
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints";
+
+export CLOJURE_OPTIONS="-A:dev"
+
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 
 entrypoint=${1:-app.main};
-
 shift 1;
 set -ex
 
-exec clojure $OPTIONS -A:dev -M -m $entrypoint "$@";
+clojure $CLOJURE_OPTIONS -A:dev -M -m $entrypoint "$@";

backend/scripts/start-dev

@@ -1,11 +1,70 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
+export PENPOT_SECRET_KEY=super-secret-devenv-key
-source $SCRIPT_DIR/_env;
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-prepl-server \
+       enable-urepl-server \
+       enable-nrepl-server \
+       enable-webhooks \
+       enable-backend-asserts \
+       enable-audit-log \
+       enable-login-with-ldap \
+       enable-transit-readable-response \
+       enable-demo-users \
+       disable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       disable-secure-session-cookies \
+       enable-rpc-climit \
+       enable-smtp \
+       enable-quotes \
+       enable-file-snapshot \
+       enable-access-tokens \
+       disable-tiered-file-data-storage \
+       enable-file-validation \
+       enable-file-schema-validation \
+       enable-subscriptions \
+       disable-subscriptions-old";
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
 # Initialize MINIO config
-setup_minio;
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+
+entrypoint=${1:-app.main};
+
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:-OmitStackTraceInFastThrow  \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
+
+export OPTIONS="-A:jmx-remote -A:dev"
 
-shift 1;
 set -ex
-exec clojure -A:jmx-remote -A:dev -M -m app.main "$@";
+clojure $OPTIONS -M -m $entrypoint;

backend/src/app/binfile/common.clj

@@ -19,7 +19,6 @@
    [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.common.weak :as weak]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
@@ -34,7 +33,8 @@
    [clojure.set :as set]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
-   [datoteka.io :as io]))
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
 
 (set! *warn-on-reflection* true)
 
@@ -141,11 +141,13 @@
   ([index coll attr]
    (reduce #(index-object %1 %2 attr) index coll)))
 
-(defn- decode-row-features
+(defn decode-row
-  [{:keys [features] :as row}]
+  [{:keys [data changes features] :as row}]
   (when row
     (cond-> row
-      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
 
 (def sql:get-minimal-file
   "SELECT f.id,
@@ -159,158 +161,23 @@
   [cfg id & {:as opts}]
   (db/get-with-sql cfg [sql:get-minimal-file id] opts))
 
-(def sql:files-with-data
+(defn decode-file
-  "SELECT f.id,
+  "A general purpose file decoding function that resolves all external
-          f.project_id,
+  pointers, run migrations and return plain vanilla file map"
-          f.created_at,
+  [cfg {:keys [id] :as file} & {:keys [migrate?] :or {migrate? true}}]
-          f.modified_at,
+  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)]
-          f.deleted_at,
+    (let [file (->> file
-          f.name,
+                    (fmigr/resolve-applied-migrations cfg)
-          f.is_shared,
+                    (fdata/resolve-file-data cfg))
-          f.has_media_trimmed,
+          libs (delay (get-resolved-file-libraries cfg file))]
-          f.revn,
-          f.data AS legacy_data,
-          f.ignore_sync_until,
-          f.comment_thread_seqn,
-          f.features,
-          f.version,
-          f.vern,
-          p.team_id,
-          coalesce(fd.backend, 'legacy-db') AS backend,
-          fd.metadata AS metadata,
-          fd.data AS data
-     FROM file AS f
-     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
-    INNER JOIN project AS p ON (p.id = f.project_id)")
 
-(def sql:get-file
+      (-> file
-  (str sql:files-with-data " WHERE f.id = ?"))
+          (update :features db/decode-pgarray #{})
-
+          (update :data blob/decode)
-(def sql:get-file-without-data
+          (update :data fdata/process-pointers deref)
-  (str "WITH files AS (" sql:files-with-data ")"
+          (update :data fdata/process-objects (partial into {}))
-       "SELECT f.id,
+          (update :data assoc :id id)
-               f.project_id,
+          (cond-> migrate? (fmg/migrate-file libs))))))
-               f.created_at,
-               f.modified_at,
-               f.deleted_at,
-               f.name,
-               f.is_shared,
-               f.has_media_trimmed,
-               f.revn,
-               f.ignore_sync_until,
-               f.comment_thread_seqn,
-               f.features,
-               f.version,
-               f.vern,
-               f.team_id
-          FROM files AS f
-         WHERE f.id = ?"))
-
-(defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)
-            pmap/*tracked* (pmap/create-tracked)]
-    (let [libs (delay (get-resolved-file-libraries cfg file))
-          ;; For avoid unnecesary overhead of creating multiple
-          ;; pointers and handly internally with objects map in their
-          ;; worst case (when probably all shapes and all pointers
-          ;; will be readed in any case), we just realize/resolve them
-          ;; before applying the migration to the file.
-          file (-> (fdata/realize cfg file)
-                   (fmg/migrate-file libs))]
-
-      (if (or read-only? (db/read-only? conn))
-        file
-        (do ;; When file is migrated, we break the rule of no
-          ;; perform mutations on get operations and update the
-          ;; file with all migrations applied
-          (update-file! cfg file)
-          (fmigr/resolve-applied-migrations cfg file))))))
-
-(defn- get-file*
-  [{:keys [::db/conn] :as cfg} id
-   {:keys [migrate?
-           realize?
-           decode?
-           skip-locked?
-           include-deleted?
-           load-data?
-           throw-if-not-exists?
-           lock-for-update?
-           lock-for-share?]
-    :or {lock-for-update? false
-         lock-for-share? false
-         load-data? true
-         migrate? true
-         decode? true
-         include-deleted? false
-         throw-if-not-exists? true
-         realize? false}
-    :as options}]
-
-  (assert (db/connection? conn) "expected cfg with valid connection")
-  (when (and (not load-data?)
-             (or lock-for-share? lock-for-share? skip-locked?))
-    (throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false")))
-
-  (let [sql
-        (if load-data?
-          sql:get-file
-          sql:get-file-without-data)
-
-        sql
-        (cond
-          lock-for-update?
-          (str sql " FOR UPDATE of f")
-
-          lock-for-share?
-          (str sql " FOR SHARE of f")
-
-          :else
-          sql)
-
-        sql
-        (if skip-locked?
-          (str sql " SKIP LOCKED")
-          sql)
-
-        file
-        (db/get-with-sql conn [sql id]
-                         {::db/throw-if-not-exists false
-                          ::db/remove-deleted (not include-deleted?)})
-
-        file
-        (-> file
-            (d/update-when :features db/decode-pgarray #{})
-            (d/update-when :metadata fdata/decode-metadata))]
-
-    (if file
-      (if load-data?
-        (let [file
-              (->> file
-                   (fmigr/resolve-applied-migrations cfg)
-                   (fdata/resolve-file-data cfg))
-
-              will-migrate?
-              (and migrate? (fmg/need-migration? file))]
-
-          (if decode?
-            (cond->> (fdata/decode-file-data cfg file)
-              (and realize? (not will-migrate?))
-              (fdata/realize cfg)
-
-              will-migrate?
-              (migrate-file cfg options))
-
-            file))
-        file)
-
-      (when-not (or skip-locked? (not throw-if-not-exists?))
-        (ex/raise :type :not-found
-                  :code :object-not-found
-                  :hint "database object not found"
-                  :table :file
-                  :file-id id)))))
 
 (defn get-file
   "Get file, resolve all features and apply migrations.
@@ -319,7 +186,10 @@
   operations on file, because it removes the ovehead of lazy fetching
   and decoding."
   [cfg file-id & {:as opts}]
-  (db/run! cfg get-file* file-id opts))
+  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                 (some->> (db/get* conn :file {:id file-id}
+                                   (assoc opts ::db/remove-deleted false))
+                          (decode-file cfg)))))
 
 (defn clean-file-features
   [file]
@@ -343,12 +213,12 @@
   (let [conn (db/get-connection cfg)
         ids  (db/create-array conn "uuid" ids)]
     (->> (db/exec! conn [sql:get-teams ids])
-         (map decode-row-features))))
+         (map decode-row))))
 
 (defn get-team
   [cfg team-id]
   (-> (db/get cfg :team {:id team-id})
-      (decode-row-features)))
+      (decode-row)))
 
 (defn get-fonts
   [cfg team-id]
@@ -440,6 +310,7 @@
             (do
               (l/trc :hint "lookup index"
                      :file-id (str file-id)
+                     :snap-id (str (:snapshot-id file))
                      :id (str id)
                      :result (str (get mobj :id)))
               (get mobj :id))
@@ -456,6 +327,7 @@
     (doseq [[old-id item] missing-index]
       (l/dbg :hint "create missing references"
              :file-id (str file-id)
+             :snap-id (str (:snapshot-id file))
              :old-id (str old-id)
              :id (str (:id item)))
       (db/insert! conn :file-media-object item
@@ -466,16 +338,12 @@
 (def sql:get-file-media
   "SELECT * FROM file_media_object WHERE id = ANY(?)")
 
-(defn get-file-media*
-  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
-  (let [used (cfh/collect-used-media data)
-        used (db/create-array conn "uuid" used)]
-    (->> (db/exec! conn [sql:get-file-media used])
-         (mapv (fn [row] (assoc row :file-id id))))))
-
 (defn get-file-media
-  [cfg file]
+  [cfg {:keys [data] :as file}]
-  (db/run! cfg get-file-media* file))
+  (db/run! cfg (fn [{:keys [::db/conn]}]
+                 (let [used (cfh/collect-used-media data)
+                       used (db/create-array conn "uuid" used)]
+                   (db/exec! conn [sql:get-file-media used])))))
 
 (def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
@@ -550,7 +418,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id :include-deleted? true)]
+              (if-let [library (get-file cfg library-id)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data
@@ -606,8 +474,8 @@
         ;; all of them, not only the applied
         (vary-meta dissoc ::fmg/migrated))))
 
-(defn- encode-file
+(defn encode-file
-  [cfg {:keys [id features] :as file}]
+  [{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}]
   (let [file (if (and (contains? features "fdata/objects-map")
                       (:data file))
                (fdata/enable-objects-map file)
@@ -624,33 +492,18 @@
 
     (-> file
         (d/update-when :features into-array)
-        (d/update-when :data blob/encode))))
+        (d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data)))))))
 
 (defn- file->params
   [file]
   (-> (select-keys file file-attrs)
-      (assoc :data nil)
       (dissoc :team-id)
       (dissoc :migrations)))
 
-(defn- file->file-data-params
-  [{:keys [id] :as file} & {:as opts}]
-  (let [created-at  (or (:created-at file) (ct/now))
-        modified-at (or (:modified-at file) created-at)]
-    (d/without-nils
-     {:id id
-      :type "main"
-      :file-id id
-      :data (:data file)
-      :metadata (:metadata file)
-      :created-at created-at
-      :modified-at modified-at})))
-
 (defn insert-file!
   "Insert a new file into the database table. Expectes a not-encoded file.
   Returns nil."
   [{:keys [::db/conn] :as cfg} file & {:as opts}]
-  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
 
   (when (:migrations file)
     (fmigr/upsert-migrations! conn file))
@@ -658,43 +511,35 @@
   (let [file (encode-file cfg file)]
     (db/insert! conn :file
                 (file->params file)
-                (assoc opts ::db/return-keys false))
+                {::db/return-keys false})
-
-    (->> (file->file-data-params file)
-         (fdata/upsert! cfg))
-
     nil))
 
 (defn update-file!
   "Update an existing file on the database. Expects not encoded file."
   [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
 
-  (if (::reset-migrations? opts false)
+  (if (::reset-migrations opts false)
     (fmigr/reset-migrations! conn file)
     (fmigr/upsert-migrations! conn file))
 
   (let [file
         (encode-file cfg file)
 
-        file-params
+        params
-        (file->params (dissoc file :id))
+        (file->params (dissoc file :id))]
 
-        file-data-params
+    (db/update! conn :file params
-        (file->file-data-params file)]
-
-    (db/update! conn :file file-params
                 {:id id}
                 {::db/return-keys false})
 
-    (fdata/upsert! cfg file-data-params)
     nil))
 
 (defn save-file!
   "Applies all the final validations and perist the file, binfile
   specific, should not be used outside of binfile domain.
+
   Returns nil"
   [{:keys [::timestamp] :as cfg} file & {:as opts}]
-
   (assert (ct/inst? timestamp) "expected valid timestamp")
 
   (let [file (-> file
@@ -719,7 +564,7 @@
           (l/error :hint "file schema validation error" :cause result))))
 
     (if (::overwrite cfg)
-      (update-file! cfg file (assoc opts ::reset-migrations? true))
+      (update-file! cfg file (assoc opts ::reset-migrations true))
       (insert-file! cfg file opts))))
 
 (def ^:private sql:get-file-libraries
@@ -749,7 +594,7 @@
           l.version
      FROM libs AS l
     INNER JOIN project AS p ON (p.id = l.project_id)
-    WHERE l.deleted_at IS NULL;")
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
 
 (defn get-file-libraries
   [conn file-id]
@@ -758,22 +603,14 @@
          ;; FIXME: :is-indirect set to false to all rows looks
          ;; completly useless
          (map #(assoc % :is-indirect false))
-         (map decode-row-features))
+         (map decode-row))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
+;; FIXME: this will use a lot of memory if file uses too many big
+;; libraries, we should load required libraries on demand
 (defn get-resolved-file-libraries
-  "Get all file libraries including itself. Returns an instance of
+  "A helper for preload file libraries"
-  LoadableWeakValueMap that allows do not have strong references to
+  [{:keys [::db/conn] :as cfg} file]
-  the loaded libraries and reduce possible memory pressure on having
+  (->> (get-file-libraries conn (:id file))
-  all this libraries loaded at same time on processing file validation
+       (into [file] (map #(get-file cfg (:id %))))
-  or file migration.
+       (d/index-by :id)))
-
-  This still requires at least one library at time to be loaded while
-  access to it is performed, but it improves considerable not having
-  the need of loading all the libraries at the same time."
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [library-ids (->> (get-file-libraries conn (:id file))
-                         (map :id)
-                         (cons (:id file)))
-        load-fn     #(get-file cfg % :migrate? false)]
-    (weak/loadable-weak-value-map library-ids load-fn {id file})))

backend/src/app/binfile/migrations.clj

@@ -36,6 +36,11 @@
       "fdata/shape-data-type"
       nil
 
+      ;; There is no migration needed, but we don't want to allow
+      ;; copy paste nor import of variant files into no-variant teams
+      "variants/v1"
+      nil
+
       (ex/raise :type :internal
                 :code :no-migration-defined
                 :hint (str/ffmt "no migation for feature '%' on file importation" feature)

backend/src/app/binfile/v1.clj

@@ -346,7 +346,7 @@
           thumbnails (->> (bfc/get-file-object-thumbnails cfg file-id)
                           (mapv #(dissoc % :file-id)))
 
-          file       (cond-> (bfc/get-file cfg file-id :realize? true)
+          file       (cond-> (bfc/get-file cfg file-id)
                        detach?
                        (-> (ctf/detach-external-references file-id)
                            (dissoc :libraries))

backend/src/app/binfile/v2.clj

@@ -153,7 +153,7 @@
 
 (defn- write-file!
   [cfg file-id]
-  (let [file   (bfc/get-file cfg file-id :realize? true)
+  (let [file   (bfc/get-file cfg file-id)
         thumbs (bfc/get-file-object-thumbnails cfg file-id)
         media  (bfc/get-file-media cfg file)
         rels   (bfc/get-files-rels cfg #{file-id})]

backend/src/app/binfile/v3.clj

@@ -27,7 +27,7 @@
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
-   [app.common.types.tokens-lib :as ctob]
+   [app.common.types.tokens-lib :as cto]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -41,10 +41,8 @@
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
-   java.io.File
    java.io.InputStream
    java.io.OutputStreamWriter
-   java.lang.AutoCloseable
    java.util.zip.ZipEntry
    java.util.zip.ZipFile
    java.util.zip.ZipOutputStream))
@@ -105,25 +103,25 @@
   (sm/encoder ctp/schema:page sm/json-transformer))
 
 (def encode-shape
-  (sm/encoder cts/schema:shape sm/json-transformer))
+  (sm/encoder ::cts/shape sm/json-transformer))
 
 (def encode-media
-  (sm/encoder ctf/schema:media sm/json-transformer))
+  (sm/encoder ::ctf/media sm/json-transformer))
 
 (def encode-component
-  (sm/encoder ctc/schema:component sm/json-transformer))
+  (sm/encoder ::ctc/component sm/json-transformer))
 
 (def encode-color
   (sm/encoder ctcl/schema:library-color sm/json-transformer))
 
 (def encode-typography
-  (sm/encoder cty/schema:typography sm/json-transformer))
+  (sm/encoder ::cty/typography sm/json-transformer))
 
 (def encode-tokens-lib
-  (sm/encoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/encoder ::cto/tokens-lib sm/json-transformer))
 
 (def encode-plugin-data
-  (sm/encoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
 
 (def encode-storage-object
   (sm/encoder schema:storage-object sm/json-transformer))
@@ -140,7 +138,7 @@
   (sm/decoder ctf/schema:media sm/json-transformer))
 
 (def decode-component
-  (sm/decoder ctc/schema:component sm/json-transformer))
+  (sm/decoder ::ctc/component sm/json-transformer))
 
 (def decode-color
   (sm/decoder ctcl/schema:library-color sm/json-transformer))
@@ -149,19 +147,19 @@
   (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
-  (sm/decoder ctp/schema:page sm/json-transformer))
+  (sm/decoder ::ctp/page sm/json-transformer))
 
 (def decode-shape
-  (sm/decoder cts/schema:shape sm/json-transformer))
+  (sm/decoder ::cts/shape sm/json-transformer))
 
 (def decode-typography
-  (sm/decoder cty/schema:typography sm/json-transformer))
+  (sm/decoder ::cty/typography sm/json-transformer))
 
 (def decode-tokens-lib
-  (sm/decoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/decoder cto/schema:tokens-lib sm/json-transformer))
 
 (def decode-plugin-data
-  (sm/decoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/decoder ::ctpg/plugin-data sm/json-transformer))
 
 (def decode-storage-object
   (sm/decoder schema:storage-object sm/json-transformer))
@@ -175,31 +173,31 @@
   (sm/check-fn schema:manifest))
 
 (def validate-file
-  (sm/check-fn ctf/schema:file))
+  (sm/check-fn ::ctf/file))
 
 (def validate-page
-  (sm/check-fn ctp/schema:page))
+  (sm/check-fn ::ctp/page))
 
 (def validate-shape
-  (sm/check-fn cts/schema:shape))
+  (sm/check-fn ::cts/shape))
 
 (def validate-media
-  (sm/check-fn ctf/schema:media))
+  (sm/check-fn ::ctf/media))
 
 (def validate-color
   (sm/check-fn ctcl/schema:library-color))
 
 (def validate-component
-  (sm/check-fn ctc/schema:component))
+  (sm/check-fn ::ctc/component))
 
 (def validate-typography
-  (sm/check-fn cty/schema:typography))
+  (sm/check-fn ::cty/typography))
 
 (def validate-tokens-lib
-  (sm/check-fn ctob/schema:tokens-lib))
+  (sm/check-fn ::cto/tokens-lib))
 
 (def validate-plugin-data
-  (sm/check-fn ctpg/schema:plugin-data))
+  (sm/check-fn ::ctpg/plugin-data))
 
 (def validate-storage-object
   (sm/check-fn schema:storage-object))
@@ -224,12 +222,9 @@
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
-  (let [detach? (and (not embed-assets) (not include-libraries))]
+  (let [detach?  (and (not embed-assets) (not include-libraries))]
     (db/tx-run! cfg (fn [cfg]
-                      (cond-> (bfc/get-file cfg file-id
+                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
-                                            {:realize? true
-                                             :include-deleted? true
-                                             :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
                             (dissoc :libraries))
@@ -255,12 +250,10 @@
 
         (write-entry! output path params)
 
-        (events/tap :progress {:section :storage-object :id id})
-
         (with-open [input (sto/get-object-data storage sobject)]
-          (.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
+          (.putNextEntry output (ZipEntry. (str "objects/" id ext)))
           (io/copy input output :size (:size sobject))
-          (.closeEntry ^ZipOutputStream output))))))
+          (.closeEntry output))))))
 
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
@@ -281,8 +274,6 @@
 
         thumbnails   (bfc/get-file-object-thumbnails cfg file-id)]
 
-    (events/tap :progress {:section :file :id file-id})
-
     (vswap! bfc/*state* update :files assoc file-id
             {:id file-id
              :name (:name file)
@@ -290,12 +281,14 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data)))
+                 (assoc :options (:options data))
 
-          file (-> file
+                 :always
-                   (dissoc :data)
+                 (dissoc :data))
-                   (dissoc :deleted-at)
+
-                   (encode-file))
+          file (cond-> file
+                 :always
+                 (encode-file))
 
           path (str "files/" file-id ".json")]
       (write-entry! output path file))
@@ -354,8 +347,7 @@
             typography (encode-typography object)]
         (write-entry! output path typography)))
 
-    (when (and tokens-lib
+    (when tokens-lib
-               (not (ctob/empty-lib? tokens-lib)))
       (let [path           (str "files/" file-id "/tokens.json")
             encoded-tokens (encode-tokens-lib tokens-lib)]
         (write-entry! output path encoded-tokens)))))
@@ -455,7 +447,7 @@
 (defn- read-manifest
   [^ZipFile input]
   (let [entry (get-zip-entry input "manifest.json")]
-    (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+    (with-open [reader (zip-entry-reader input entry)]
       (let [manifest (json/read reader :key-fn json/read-kebab-key)]
         (decode-manifest manifest)))))
 
@@ -545,12 +537,12 @@
 
 (defn- read-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
 (defn- read-plain-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader)))
 
 (defn- read-file
@@ -718,7 +710,7 @@
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
+  [{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}]
   (let [file-id'   (bfc/lookup-index file-id)
         file       (read-file cfg file-id)
         media      (read-file-media cfg file-id)
@@ -731,48 +723,26 @@
            :version (:version file)
            ::l/sync? true)
 
-    (vswap! bfc/*state* update :index bfc/update-index media :id)
+    (events/tap :progress {:section :file :name file-name})
 
-    (events/tap :progress {:section :media :file-id file-id})
+    (when media
+      ;; Update index with media
+      (l/dbg :hint "update media index"
+             :file-id (str file-id')
+             :total (count media)
+             ::l/sync? true)
 
-    (doseq [item media]
+      (vswap! bfc/*state* update :index bfc/update-index (map :id media))
-      (let [params (-> item
+      (vswap! bfc/*state* update :media into media))
-                       (update :id bfc/lookup-index)
-                       (assoc :file-id file-id')
-                       (d/update-when :media-id bfc/lookup-index)
-                       (d/update-when :thumbnail-id bfc/lookup-index))]
 
-        (l/dbg :hint "inserting media object"
+    (when thumbnails
-               :file-id (str file-id')
+      (l/dbg :hint "update thumbnails index"
-               :id (str (:id params))
+             :file-id (str file-id')
-               :media-id (str (:media-id params))
+             :total (count thumbnails)
-               :thumbnail-id (str (:thumbnail-id params))
+             ::l/sync? true)
-               :old-id (str (:id item))
-               ::l/sync? true)
 
-        (db/insert! conn :file-media-object params
+      (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
-                    ::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
+      (vswap! bfc/*state* update :thumbnails into thumbnails))
-
-    (events/tap :progress {:section :thumbnails :file-id file-id})
-
-    (doseq [item thumbnails]
-      (let [media-id  (bfc/lookup-index (:media-id item))
-            object-id (-> (assoc item :file-id file-id')
-                          (cth/fmt-object-id))
-            params    {:file-id file-id'
-                       :object-id object-id
-                       :tag (:tag item)
-                       :media-id media-id}]
-
-        (l/dbg :hint "inserting object thumbnail"
-               :file-id (str file-id')
-               :media-id (str media-id)
-               ::l/sync? true)
-
-        (db/insert! conn :file-tagged-object-thumbnail params
-                    ::db/on-conflict-do-nothing? true)))
-
-    (events/tap :progress {:section :file :file-id file-id})
 
     (let [data (-> (read-file-data cfg file-id)
                    (d/without-nils)
@@ -821,47 +791,95 @@
         entries (keep (match-storage-entry-fn) entries)]
 
     (doseq [{:keys [id entry]} entries]
-      (let [object  (->> (read-entry input entry)
+      (let [object (->> (read-entry input entry)
-                         (decode-storage-object)
+                        (decode-storage-object)
-                         (validate-storage-object))
+                        (validate-storage-object))]
 
-            ext     (cmedia/mtype->extension (:content-type object))
+        (when (not= id (:id object))
-            path    (str "objects/" id ext)
-            content (->> path
-                         (get-zip-entry input)
-                         (zip-entry-storage-content input))]
-
-        (when (not= (:size object) (sto/get-size content))
           (ex/raise :type :validation
                     :code :inconsistent-penpot-file
-                    :hint "found corrupted storage object: size does not match"
+                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"
-                    :path path
+                    :expected-id (str id)
-                    :expected-size (:size object)
+                    :found-id (str (:id object))))
-                    :found-size (sto/get-size content)))
 
-        (when-let [hash (get object :hash)]
+        (let [ext     (cmedia/mtype->extension (:content-type object))
-          (when (not= hash (sto/get-hash content))
+              path    (str "objects/" id ext)
+              content (->> path
+                           (get-zip-entry input)
+                           (zip-entry-storage-content input))]
+
+          (when (not= (:size object) (sto/get-size content))
             (ex/raise :type :validation
                       :code :inconsistent-penpot-file
-                      :hint "found corrupted storage object: hash does not match"
+                      :hint "found corrupted storage object: size does not match"
                       :path path
-                      :expected-hash (:hash object)
+                      :expected-size (:size object)
-                      :found-hash (sto/get-hash content))))
+                      :found-size (sto/get-size content)))
 
-        (let [params  (-> object
+          (when-let [hash (get object :hash)]
-                          (dissoc :id :size)
+            (when (not= hash (sto/get-hash content))
-                          (assoc ::sto/content content)
+              (ex/raise :type :validation
-                          (assoc ::sto/deduplicate? true)
+                        :code :inconsistent-penpot-file
-                          (assoc ::sto/touched-at timestamp))
+                        :hint "found corrupted storage object: hash does not match"
-              sobject (sto/put-object! storage params)]
+                        :path path
+                        :expected-hash (:hash object)
+                        :found-hash (sto/get-hash content))))
 
-          (l/dbg :hint "persisted storage object"
+          (let [params  (-> object
-                 :id (str (:id sobject))
+                            (dissoc :id :size)
-                 :prev-id (str id)
+                            (assoc ::sto/content content)
-                 :bucket (:bucket params)
+                            (assoc ::sto/deduplicate? true)
-                 ::l/sync? true)
+                            (assoc ::sto/touched-at timestamp))
+                sobject (sto/put-object! storage params)]
 
-          (vswap! bfc/*state* update :index assoc id (:id sobject)))))))
+            (l/dbg :hint "persisted storage object"
+                   :id (str (:id sobject))
+                   :prev-id (str id)
+                   :bucket (:bucket params)
+                   ::l/sync? true)
+
+            (vswap! bfc/*state* update :index assoc id (:id sobject))))))))
+
+(defn- import-file-media
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:section :media})
+
+  (doseq [item (:media @bfc/*state*)]
+    (let [params (-> item
+                     (update :id bfc/lookup-index)
+                     (update :file-id bfc/lookup-index)
+                     (d/update-when :media-id bfc/lookup-index)
+                     (d/update-when :thumbnail-id bfc/lookup-index))]
+
+      (l/dbg :hint "inserting file media object"
+             :old-id (str (:id item))
+             :id (str (:id params))
+             :file-id (str (:file-id params))
+             ::l/sync? true)
+
+      (db/insert! conn :file-media-object params
+                  ::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))))
+
+(defn- import-file-thumbnails
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:section :thumbnails})
+  (doseq [item (:thumbnails @bfc/*state*)]
+    (let [file-id   (bfc/lookup-index (:file-id item))
+          media-id  (bfc/lookup-index (:media-id item))
+          object-id (-> (assoc item :file-id file-id)
+                        (cth/fmt-object-id))
+          params    {:file-id file-id
+                     :object-id object-id
+                     :tag (:tag item)
+                     :media-id media-id}]
+
+      (l/dbg :hint "inserting file object thumbnail"
+             :file-id (str file-id)
+             :media-id (str media-id)
+             ::l/sync? true)
+
+      (db/insert! conn :file-tagged-object-thumbnail params
+                  {::db/on-conflict-do-nothing? true}))))
 
 (defn- import-files*
   [{:keys [::manifest] :as cfg}]
@@ -869,8 +887,6 @@
 
   (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
 
-  (import-storage-objects cfg)
-
   (let [files  (get manifest :files)
         result (reduce (fn [result {:keys [id] :as file}]
                          (let [name' (get file :name)
@@ -883,6 +899,10 @@
                        files)]
 
     (import-file-relations cfg)
+    (import-storage-objects cfg)
+    (import-file-media cfg)
+    (import-file-thumbnails cfg)
+
     (bfm/apply-pending-migrations! cfg)
 
     result))
@@ -907,8 +927,9 @@
     (binding [bfc/*options* cfg
               bfc/*reference-file* ref-file]
 
-      (import-storage-objects cfg)
       (import-file cfg file)
+      (import-storage-objects cfg)
+      (import-file-media cfg)
 
       (bfc/invalidate-thumbnails cfg file-id)
       (bfm/apply-pending-migrations! cfg)
@@ -985,8 +1006,8 @@
     (try
       (l/info :hint "start exportation" :export-id (str id))
       (binding [bfc/*state* (volatile! (bfc/initial-state))]
-        (with-open [^AutoCloseable output (io/output-stream output)]
+        (with-open [output (io/output-stream output)]
-          (with-open [^AutoCloseable output (ZipOutputStream. output)]
+          (with-open [output (ZipOutputStream. output)]
             (let [cfg (assoc cfg ::output output)]
               (export-files cfg)
               (export-storage-objects cfg)))))
@@ -1030,7 +1051,7 @@
 
     (l/info :hint "import: started" :id (str id))
     (try
-      (with-open [input (ZipFile. ^File (fs/file input))]
+      (with-open [input (ZipFile. (fs/file input))]
         (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
@@ -1045,6 +1066,6 @@
 
 (defn get-manifest
   [path]
-  (with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
+  (with-open [input (ZipFile. (fs/file path))]
     (-> (read-manifest input)
         (validate-manifest))))

backend/src/app/config.clj

@@ -5,6 +5,7 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.config
+  "A configuration management."
   (:refer-clojure :exclude [get])
   (:require
    [app.common.data :as d]
@@ -46,19 +47,14 @@
    :auto-file-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
-
    :host "localhost"
    :tenant "default"
 
    :redis-uri "redis://redis/0"
 
-   :file-data-backend "legacy-db"
-
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
-   :auth-token-cookie-name "auth-token"
-
    :assets-path "/internal/assets/"
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
@@ -92,7 +88,7 @@
     [:secret-key {:optional true} :string]
 
     [:tenant {:optional false} :string]
-    [:public-uri {:optional false} ::sm/uri]
+    [:public-uri {:optional false} :string]
     [:host {:optional false} :string]
 
     [:http-server-port {:optional true} ::sm/int]
@@ -100,9 +96,7 @@
     [:http-server-max-body-size {:optional true} ::sm/int]
     [:http-server-max-multipart-body-size {:optional true} ::sm/int]
     [:http-server-io-threads {:optional true} ::sm/int]
-    [:http-server-max-worker-threads {:optional true} ::sm/int]
+    [:http-server-worker-threads {:optional true} ::sm/int]
-
-    [:management-api-key {:optional true} :string]
 
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
@@ -111,8 +105,7 @@
     [:auto-file-snapshot-timeout {:optional true} ::ct/duration]
 
     [:media-max-file-size {:optional true} ::sm/int]
-    [:deletion-delay {:optional true} ::ct/duration]
+    [:deletion-delay {:optional true} ::ct/duration] ;; REVIEW
-    [:file-clean-delay {:optional true} ::ct/duration]
     [:telemetry-enabled {:optional true} ::sm/boolean]
     [:default-blob-version {:optional true} ::sm/int]
     [:allow-demo-users {:optional true} ::sm/boolean]
@@ -153,6 +146,7 @@
     [:quotes-team-access-requests-per-team {:optional true} ::sm/int]
     [:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
 
+    [:auth-data-cookie-domain {:optional true} :string]
     [:auth-token-cookie-name {:optional true} :string]
     [:auth-token-cookie-max-age {:optional true} ::ct/duration]
 
@@ -167,7 +161,7 @@
     [:google-client-id {:optional true} :string]
     [:google-client-secret {:optional true} :string]
     [:oidc-client-id {:optional true} :string]
-    [:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
+    [:oidc-user-info-source {:optional true} :keyword]
     [:oidc-client-secret {:optional true} :string]
     [:oidc-base-uri {:optional true} :string]
     [:oidc-token-uri {:optional true} :string]
@@ -216,27 +210,24 @@
     [:prepl-host {:optional true} :string]
     [:prepl-port {:optional true} ::sm/int]
 
-    [:file-data-backend {:optional true} [:enum "db" "legacy-db" "storage"]]
-
     [:media-directory {:optional true} :string] ;; REVIEW
     [:media-uri {:optional true} :string]
     [:assets-path {:optional true} :string]
 
-    [:netty-io-threads {:optional true} ::sm/int]
+    ;; Legacy, will be removed in 2.5
-    [:executor-threads {:optional true} ::sm/int]
-
-    ;; DEPRECATED
     [:assets-storage-backend {:optional true} :keyword]
     [:storage-assets-fs-directory {:optional true} :string]
     [:storage-assets-s3-bucket {:optional true} :string]
     [:storage-assets-s3-region {:optional true} :keyword]
     [:storage-assets-s3-endpoint {:optional true} ::sm/uri]
+    [:storage-assets-s3-io-threads {:optional true} ::sm/int]
 
     [:objects-storage-backend {:optional true} :keyword]
     [:objects-storage-fs-directory {:optional true} :string]
     [:objects-storage-s3-bucket {:optional true} :string]
     [:objects-storage-s3-region {:optional true} :keyword]
-    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]]))
+    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
+    [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
 (defn- parse-flags
   [config]
@@ -309,11 +300,6 @@
   (or (c/get config :deletion-delay)
       (ct/duration {:days 7})))
 
-(defn get-file-clean-delay
-  []
-  (or (c/get config :file-clean-delay)
-      (ct/duration {:days 2})))
-
 (defn get
   "A configuration getter. Helps code be more testable."
   ([key]
@@ -321,9 +307,5 @@
   ([key default]
    (c/get config key default)))
 
-(defn logging-context
-  []
-  {:version/backend (:full version)})
-
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -298,7 +298,7 @@
 (defn insert!
   "A helper that builds an insert sql statement and executes it. By
   default returns the inserted row with all the field; you can delimit
-  the returned columns with the `::sql/columns` option."
+  the returned columns with the `::columns` option."
   [ds table params & {:as opts}]
   (let [conn (get-connectable ds)
         sql  (sql/insert table params opts)
@@ -379,7 +379,9 @@
 
 (defn is-row-deleted?
   [{:keys [deleted-at]}]
-  (some? deleted-at))
+  (and (ct/inst? deleted-at)
+       (< (inst-ms deleted-at)
+          (inst-ms (ct/now)))))
 
 (defn get*
   "Retrieve a single row from database that matches a simple filters. Do
@@ -404,15 +406,15 @@
                 :hint "database object not found"))
     row))
 
+
 (defn get-with-sql
   [ds sql & {:as opts}]
-  (let [rows
+  (let [rows (cond->> (exec! ds sql opts)
-        (cond->> (exec! ds sql opts)
+               (::remove-deleted opts true)
-          (::remove-deleted opts true)
+               (remove is-row-deleted?)
-          (remove is-row-deleted?)
 
-          :always
+               :always
-          (not-empty))]
+               (not-empty))]
 
     (when (and (not rows) (::throw-if-not-exists opts true))
       (ex/raise :type :not-found
@@ -421,6 +423,7 @@
 
     (first rows)))
 
+
 (def ^:private default-plan-opts
   (-> default-opts
       (assoc :fetch-size 1000)
@@ -575,10 +578,10 @@
   [system f & params]
   (cond
     (connection? system)
-    (apply run! {::conn system} f params)
+    (run! {::conn system} f)
 
     (pool? system)
-    (apply run! {::pool system} f params)
+    (run! {::pool system} f)
 
     (::conn system)
     (apply f system params)
@@ -704,12 +707,6 @@
   (and (sql-exception? cause)
        (= "40001" (.getSQLState ^java.sql.SQLException cause))))
 
-(defn duplicate-key-error?
-  [cause]
-  (and (sql-exception? cause)
-       (= "23505" (.getSQLState ^java.sql.SQLException cause))))
-
-
 (extend-protocol jdbc.prepare/SettableParameter
   clojure.lang.Keyword
   (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]

backend/src/app/db/sql.clj

@@ -53,15 +53,8 @@
          opts (cond-> opts
                 (::order-by opts)   (assoc :order-by (::order-by opts))
                 (::columns opts)    (assoc :columns (::columns opts))
-
+                (::for-update opts) (assoc :suffix "FOR UPDATE")
-                (or (::db/for-update opts)
+                (::for-share opts)  (assoc :suffix "FOR SHARE"))]
-                    (::for-update opts))
-                (assoc :suffix "FOR UPDATE")
-
-                (or (::db/for-share opts)
-                    (::for-share opts))
-                (assoc :suffix "FOR SHARE"))]
-
      (sql/for-query table where-params opts))))
 
 (defn update

backend/src/app/email.clj

@@ -7,7 +7,6 @@
 (ns app.email
   "Main api for send emails."
   (:require
-   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -94,44 +93,36 @@
                headers)))
 
 (defn- assign-body
-  [^MimeMessage mmsg {:keys [body charset attachments] :or {charset "utf-8"}}]
+  [^MimeMessage mmsg {:keys [body charset] :or {charset "utf-8"}}]
-  (let [mixed-mpart  (MimeMultipart. "mixed")]
+  (let [mpart (MimeMultipart. "mixed")]
     (cond
       (string? body)
-      (let [text-part (MimeBodyPart.)]
+      (let [bpart (MimeBodyPart.)]
-        (.setText text-part ^String body ^String charset)
+        (.setContent bpart ^String body (str "text/plain; charset=" charset))
-        (.addBodyPart mixed-mpart text-part))
+        (.addBodyPart mpart bpart))
+
+      (vector? body)
+      (let [mmp (MimeMultipart. "alternative")
+            mbp (MimeBodyPart.)]
+        (.addBodyPart mpart mbp)
+        (.setContent mbp mmp)
+        (doseq [item body]
+          (let [mbp (MimeBodyPart.)]
+            (.setContent mbp
+                         ^String (:content item)
+                         ^String (str (:type item "text/plain") "; charset=" charset))
+            (.addBodyPart mmp mbp))))
 
       (map? body)
-      (let [content-part      (MimeBodyPart.)
+      (let [bpart (MimeBodyPart.)]
-            alternative-mpart (MimeMultipart. "alternative")]
+        (.setContent bpart
-
+                     ^String (:content body)
-        (when-let [content (get body "text/html")]
+                     ^String (str (:type body "text/plain") "; charset=" charset))
-          (let [html-part (MimeBodyPart.)]
+        (.addBodyPart mpart bpart))
-            (.setContent html-part ^String content
-                         (str "text/html; charset=" charset))
-            (.addBodyPart alternative-mpart html-part)))
-
-        (when-let [content (get body "text/plain")]
-          (let [text-part (MimeBodyPart.)]
-            (.setText text-part ^String content ^String charset)
-            (.addBodyPart alternative-mpart text-part)))
-
-        (.setContent content-part alternative-mpart)
-        (.addBodyPart mixed-mpart content-part))
 
       :else
-      (throw (IllegalArgumentException. "invalid email body provided")))
+      (throw (ex-info "Unsupported type" {:body body})))
-
+    (.setContent mmsg mpart)
-    (doseq [[name content] attachments]
-
-      (prn "attachment" name)
-      (let [attachment-part (MimeBodyPart.)]
-        (.setFileName attachment-part ^String name)
-        (.setContent attachment-part ^String content (str "text/plain; charset=" charset))
-        (.addBodyPart mixed-mpart attachment-part)))
-
-    (.setContent mmsg mixed-mpart)
     mmsg))
 
 (defn- opts->props
@@ -219,26 +210,24 @@
       (ex/raise :type :internal
                 :code :missing-email-templates))
     {:subject subj
-     :body (d/without-nils
+     :body (into
-            {"text/plain" text
+            [{:type "text/plain"
-             "text/html" html})}))
+              :content text}]
+            (when html
+              [{:type "text/html"
+                :content html}]))}))
 
-(def ^:private schema:params
+(def ^:private schema:context
-  [:map {:title "Email Params"}
+  [:map
    [:to [:or ::sm/email [::sm/vec ::sm/email]]]
    [:reply-to {:optional true} ::sm/email]
    [:from {:optional true} ::sm/email]
    [:lang {:optional true} ::sm/text]
-   [:subject {:optional true} ::sm/text]
    [:priority {:optional true} [:enum :high :low]]
-   [:extra-data {:optional true} ::sm/text]
+   [:extra-data {:optional true} ::sm/text]])
-   [:body {:optional true}
-    [:or :string [:map-of :string :string]]]
-   [:attachments {:optional true}
-    [:map-of :string :string]]])
 
-(def ^:private check-params
+(def ^:private check-context
-  (sm/check-fn schema:params))
+  (sm/check-fn schema:context))
 
 (defn template-factory
   [& {:keys [id schema]}]
@@ -246,9 +235,9 @@
   (let [check-fn (if schema
                    (sm/check-fn schema)
                    (constantly nil))]
-    (fn [params]
+    (fn [context]
-      (let [params (-> params check-params check-fn)
+      (let [context (-> context check-context check-fn)
-            email  (build-email-template id params)]
+            email   (build-email-template id context)]
         (when-not email
           (ex/raise :type :internal
                     :code :email-template-does-not-exists
@@ -256,40 +245,35 @@
                     :template-id id))
 
         (cond-> (assoc email :id (name id))
-          (:extra-data params)
+          (:extra-data context)
-          (assoc :extra-data (:extra-data params))
+          (assoc :extra-data (:extra-data context))
 
-          (seq (:attachments params))
+          (:from context)
-          (assoc :attachments (:attachments params))
+          (assoc :from (:from context))
 
-          (:from params)
+          (:reply-to context)
-          (assoc :from (:from params))
+          (assoc :reply-to (:reply-to context))
 
-          (:reply-to params)
+          (:to context)
-          (assoc :reply-to (:reply-to params))
+          (assoc :to (:to context)))))))
-
-          (:to params)
-          (assoc :to (:to params)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PUBLIC HIGH-LEVEL API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn render
-  [email-factory params]
+  [email-factory context]
-  (email-factory params))
+  (email-factory context))
 
 (defn send!
   "Schedule an already defined email to be sent using asynchronously
   using worker task."
-  [{:keys [::conn ::factory] :as params}]
+  [{:keys [::conn ::factory] :as context}]
   (assert (db/connectable? conn) "expected a valid database connection or pool")
 
   (let [email (if factory
-                (factory params)
+                (factory context)
-                (-> params
+                (dissoc context ::conn))]
-                    (dissoc params)
-                    (check-params)))]
     (wrk/submit! {::wrk/task :sendmail
                   ::wrk/delay 0
                   ::wrk/max-retries 4
@@ -359,10 +343,8 @@
 
 (def ^:private schema:feedback
   [:map
-   [:feedback-subject ::sm/text]
+   [:subject ::sm/text]
-   [:feedback-type ::sm/text]
+   [:content ::sm/text]])
-   [:feedback-content ::sm/text]
-   [:profile :map]])
 
 (def user-feedback
   "A profile feedback email."

backend/src/app/features/fdata.clj

@@ -9,22 +9,48 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.files.helpers :as cfh]
+   [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
-   [app.common.schema :as sm]
+   [app.common.types.path :as path]
-   [app.common.time :as ct]
-   [app.common.types.objects-map :as omap]
-   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap.legacy]
+   [app.util.objects-map :as omap]
-   [app.util.pointer-map :as pmap]))
+   [app.util.pointer-map :as pmap]
+   [app.worker :as wrk]
+   [promesa.exec :as px]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OFFLOAD
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn offloaded?
+  [file]
+  (= "objects-storage" (:data-backend file)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OBJECTS-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn enable-objects-map
+  [file & _opts]
+  (let [update-page
+        (fn [page]
+          (if (and (pmap/pointer-map? page)
+                   (not (pmap/loaded? page)))
+            page
+            (update page :objects omap/wrap)))
+
+        update-data
+        (fn [fdata]
+          (update fdata :pages-index d/update-vals update-page))]
+
+    (-> file
+        (update :data update-data)
+        (update :features conj "fdata/objects-map"))))
+
 (defn process-objects
   "Apply a function to all objects-map on the file. Usualy used for convert
   the objects-map instances to plain maps"
@@ -34,237 +60,41 @@
             (fn [page]
               (update page :objects
                       (fn [objects]
-                        (if (or (omap/objects-map? objects)
+                        (if (omap/objects-map? objects)
-                                (omap.legacy/objects-map? objects))
                           (update-fn objects)
                           objects)))))
     fdata))
 
-
-(defn realize-objects
-  "Process a file and remove all instances of objects map realizing them
-  to a plain data. Used in operation where is more efficient have the
-  whole file loaded in memory or we going to persist it in an
-  alterantive storage."
-  [_cfg file]
-  (update file :data process-objects (partial into {})))
-
-(defn enable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (update page :objects omap/wrap))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features conj "fdata/objects-map"))))
-
-(defn disable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (update page :objects #(into {} %)))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features disj "fdata/objects-map"))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; STORAGE
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmulti resolve-file-data
-  (fn [_cfg file] (get file :backend "legacy-db")))
-
-(defmethod resolve-file-data "legacy-db"
-  [_cfg {:keys [legacy-data] :as file}]
-  (-> file
-      (assoc :data legacy-data)
-      (dissoc :legacy-data)))
-
-(defmethod resolve-file-data "db"
-  [_cfg file]
-  (dissoc file :legacy-data))
-
-(defmethod resolve-file-data "storage"
-  [cfg {:keys [metadata] :as file}]
-  (let [storage (sto/resolve cfg ::db/reuse-conn true)
-        ref-id  (:storage-ref-id metadata)
-        data    (->> (sto/get-object storage ref-id)
-                     (sto/get-object-bytes storage))]
-    (-> file
-        (assoc :data data)
-        (dissoc :legacy-data))))
-
-(defn decode-file-data
-  [_cfg {:keys [data] :as file}]
-  (cond-> file
-    (bytes? data)
-    (assoc :data (blob/decode data))))
-
-(def ^:private sql:insert-file-data
-  "INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at,
-                          type, backend, metadata, data)
-   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
-
-(def ^:private sql:upsert-file-data
-  (str sql:insert-file-data
-       " ON CONFLICT (file_id, id)
-         DO UPDATE SET modified_at=?,
-                       deleted_at=?,
-                       backend=?,
-                       metadata=?,
-                       data=?"))
-
-(defn- upsert-in-database
-  [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}]
-  (let [created-at  (or created-at (ct/now))
-        metadata    (some-> metadata db/json)
-        modified-at (or modified-at created-at)]
-
-    (db/exec-one! cfg [sql:upsert-file-data
-                       file-id id
-                       created-at
-                       modified-at
-                       deleted-at
-                       type
-                       backend
-                       metadata
-                       data
-                       modified-at
-                       deleted-at
-                       backend
-                       metadata
-                       data])))
-
-(defn- handle-persistence
-  [cfg {:keys [type backend id file-id data] :as params}]
-
-  (cond
-    (= backend "storage")
-    (let [storage  (sto/resolve cfg)
-          content  (sto/content data)
-          sobject  (sto/put-object! storage
-                                    {::sto/content content
-                                     ::sto/touch true
-                                     :bucket "file-data"
-                                     :content-type "application/octet-stream"
-                                     :file-id file-id
-                                     :id id})
-          metadata {:storage-ref-id (:id sobject)}
-          params   (-> params
-                       (assoc :metadata metadata)
-                       (assoc :data nil))]
-      (upsert-in-database cfg params))
-
-    (= backend "db")
-    (->> (dissoc params :metadata)
-         (upsert-in-database cfg))
-
-    (= backend "legacy-db")
-    (cond
-      (= type "main")
-      (do
-        (db/delete! cfg :file-data
-                    {:id id :file-id file-id :type "main"}
-                    {::db/return-keys false})
-        (db/update! cfg :file
-                    {:data data}
-                    {:id file-id}
-                    {::db/return-keys false}))
-
-      (= type "snapshot")
-      (do
-        (db/delete! cfg :file-data
-                    {:id id :file-id file-id :type "snapshot"}
-                    {::db/return-keys false})
-        (db/update! cfg :file-change
-                    {:data data}
-                    {:file-id file-id :id id}
-                    {::db/return-keys false}))
-
-      (= type "fragment")
-      (upsert-in-database cfg
-                          (-> (dissoc params :metadata)
-                              (assoc :backend "db")))
-
-      :else
-      (throw (RuntimeException. "not implemented")))
-
-    :else
-    (throw (IllegalArgumentException.
-            (str "backend '" backend "' not supported")))))
-
-(defn process-metadata
-  [cfg metadata]
-  (when-let [storage-id (:storage-ref-id metadata)]
-    (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-      (sto/touch-object! storage storage-id))))
-
-(defn- default-backend
-  [backend]
-  (or backend (cf/get :file-data-backend)))
-
-(def ^:private schema:metadata
-  [:map {:title "Metadata"}
-   [:storage-ref-id {:optional true} ::sm/uuid]])
-
-(def decode-metadata-with-schema
-  (sm/decoder schema:metadata sm/json-transformer))
-
-(defn decode-metadata
-  [metadata]
-  (some-> metadata
-          (db/decode-json-pgobject)
-          (decode-metadata-with-schema)))
-
-(def ^:private schema:update-params
-  [:map {:closed true}
-   [:id ::sm/uuid]
-   [:type [:enum "main" "snapshot" "fragment"]]
-   [:file-id ::sm/uuid]
-   [:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
-   [:metadata {:optional true} [:maybe schema:metadata]]
-   [:data {:optional true} bytes?]
-   [:created-at {:optional true} ::ct/inst]
-   [:modified-at {:optional true} [:maybe ::ct/inst]]
-   [:deleted-at {:optional true} [:maybe ::ct/inst]]])
-
-(def ^:private check-update-params
-  (sm/check-fn schema:update-params :hint "invalid params received for update"))
-
-(defn upsert!
-  "Create or update file data"
-  [cfg params & {:as opts}]
-  (let [params (-> (check-update-params params)
-                   (update :backend default-backend))]
-
-    (some->> (:metadata params)
-             (process-metadata cfg))
-
-    (-> (handle-persistence cfg params)
-        (db/get-update-count)
-        (pos?))))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; POINTER-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn get-file-data
+  "Get file data given a file instance."
+  [system file]
+  (if (offloaded? file)
+    (let [storage (sto/resolve system ::db/reuse-conn true)]
+      (->> (sto/get-object storage (:data-ref-id file))
+           (sto/get-object-bytes storage)))
+    (:data file)))
+
+(defn resolve-file-data
+  [system file]
+  (let [data (get-file-data system file)]
+    (assoc file :data data)))
+
+(defn decode-file-data
+  [{:keys [::wrk/executor]} {:keys [data] :as file}]
+  (cond-> file
+    (bytes? data)
+    (assoc :data (px/invoke! executor #(blob/decode data)))))
+
 (defn load-pointer
   "A database loader pointer helper"
-  [cfg file-id id]
+  [system file-id id]
-  (let [fragment (some-> (db/get* cfg :file-data
+  (let [fragment (db/get* system :file-data-fragment
-                                  {:id id :file-id file-id :type "fragment"}
+                          {:id id :file-id file-id}
-                                  {::sql/columns [:data :backend :id :metadata]})
+                          {::sql/columns [:data :data-backend :data-ref-id :id]})]
-                         (update :metadata decode-metadata))]
 
     (l/trc :hint "load pointer"
            :file-id (str file-id)
@@ -278,21 +108,22 @@
                 :file-id file-id
                 :fragment-id id))
 
-    (-> (resolve-file-data cfg fragment)
+    (let [data (get-file-data system fragment)]
-        (get :data)
+      ;; FIXME: conditional thread scheduling for decoding big objects
-        (blob/decode))))
+      (blob/decode data))))
 
 (defn persist-pointers!
   "Persist all currently tracked pointer objects"
-  [cfg file-id]
+  [system file-id]
-  (doseq [[id item] @pmap/*tracked*]
+  (let [conn (db/get-connection system)]
-    (when (pmap/modified? item)
+    (doseq [[id item] @pmap/*tracked*]
-      (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
+      (when (pmap/modified? item)
-      (let [content (-> item deref blob/encode)]
+        (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
-        (upsert! cfg {:id id
+        (let [content (-> item deref blob/encode)]
-                      :file-id file-id
+          (db/insert! conn :file-data-fragment
-                      :type "fragment"
+                      {:id id
-                      :data content})))))
+                       :file-id file-id
+                       :data content}))))))
 
 (defn process-pointers
   "Apply a function to all pointers on the file. Usuly used for
@@ -306,14 +137,6 @@
         (d/update-vals update-fn')
         (update :pages-index d/update-vals update-fn'))))
 
-(defn realize-pointers
-  "Process a file and remove all instances of pointers realizing them to
-  a plain data. Used in operation where is more efficient have the
-  whole file loaded in memory."
-  [cfg {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial load-pointer cfg id)]
-    (update file :data process-pointers deref)))
-
 (defn get-used-pointer-ids
   "Given a file, return all pointer ids used in the data."
   [fdata]
@@ -333,12 +156,47 @@
       (update :features conj "fdata/pointer-map")))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; GENERAL PURPOSE HELPERS
+;; PATH-DATA
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn realize
+(defn enable-path-data
-  "A helper that combines realize-pointers and realize-objects"
+  "Enable the fdata/path-data feature on the file."
-  [cfg file]
+  [file & _opts]
-  (->> file
+  (letfn [(update-object [object]
-       (realize-pointers cfg)
+            (if (or (cfh/path-shape? object)
-       (realize-objects cfg)))
+                    (cfh/bool-shape? object))
+              (update object :content path/content)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features conj "fdata/path-data"))))
+
+(defn disable-path-data
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content vec)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (when-let [conn db/*conn*]
+      (db/delete! conn :file-migration {:file-id (:id file)
+                                        :name "0003-convert-path-content"}))
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features disj "fdata/path-data")
+        (update :migrations disj "0003-convert-path-content")
+        (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))

backend/src/app/features/file_snapshots.clj

@@ -1,446 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.features.file-snapshots
-  (:require
-   [app.binfile.common :as bfc]
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.features :as-alias cfeat]
-   [app.common.files.migrations :as fmg]
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.uuid :as uuid]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.db.sql :as-alias sql]
-   [app.features.fdata :as fdata]
-   [app.storage :as sto]
-   [app.util.blob :as blob]
-   [app.worker :as wrk]
-   [cuerdas.core :as str]))
-
-(def sql:snapshots
-  "SELECT c.id,
-          c.label,
-          c.created_at,
-          c.updated_at AS modified_at,
-          c.deleted_at,
-          c.profile_id,
-          c.created_by,
-          c.locked_by,
-          c.revn,
-          c.features,
-          c.migrations,
-          c.version,
-          c.file_id,
-          c.data AS legacy_data,
-          fd.data AS data,
-          coalesce(fd.backend, 'legacy-db') AS backend,
-          fd.metadata AS metadata
-     FROM file_change AS c
-     LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id
-                                   AND fd.id = c.id
-                                   AND fd.type = 'snapshot')
-    WHERE c.label IS NOT NULL")
-
-(defn- decode-snapshot
-  [snapshot]
-  (some-> snapshot
-          (-> (d/update-when :metadata fdata/decode-metadata)
-              (d/update-when :migrations db/decode-pgarray [])
-              (d/update-when :features db/decode-pgarray #{}))))
-
-(def ^:private sql:get-minimal-file
-  "SELECT f.id,
-          f.revn,
-          f.modified_at,
-          f.deleted_at,
-          fd.backend AS backend,
-          fd.metadata AS metadata
-     FROM file AS f
-     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
-    WHERE f.id = ?")
-
-(defn- get-minimal-file
-  [cfg id & {:as opts}]
-  (-> (db/get-with-sql cfg [sql:get-minimal-file id] opts)
-      (d/update-when :metadata fdata/decode-metadata)))
-
-(def ^:private sql:get-snapshot-without-data
-  (str "WITH snapshots AS (" sql:snapshots ")"
-       "SELECT c.id,
-               c.label,
-               c.revn,
-               c.created_at,
-               c.modified_at,
-               c.deleted_at,
-               c.profile_id,
-               c.created_by,
-               c.locked_by,
-               c.features,
-               c.metadata,
-               c.migrations,
-               c.version,
-               c.file_id
-         FROM snapshots AS c
-        WHERE c.id = ?
-          AND CASE WHEN c.created_by = 'user'
-                   THEN c.deleted_at IS NULL
-                   WHEN c.created_by = 'system'
-                   THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz
-               END"))
-
-(defn get-minimal-snapshot
-  [cfg snapshot-id]
-  (let [now (ct/now)]
-    (-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id now]
-                         {::db/remove-deleted false})
-        (decode-snapshot))))
-
-(def ^:private sql:get-snapshot
-  (str sql:snapshots
-       " AND c.file_id = ?
-         AND c.id = ?
-         AND CASE WHEN c.created_by = 'user'
-                  THEN (c.deleted_at IS NULL)
-                  WHEN c.created_by = 'system'
-                  THEN (c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz)
-              END"))
-
-(defn- get-snapshot
-  "Get snapshot with decoded data"
-  [cfg file-id snapshot-id]
-  (let [now (ct/now)]
-    (->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now]
-                          {::db/remove-deleted false})
-         (decode-snapshot)
-         (fdata/resolve-file-data cfg)
-         (fdata/decode-file-data cfg))))
-
-(def ^:private sql:get-visible-snapshots
-  (str "WITH "
-       "snapshots1 AS ( " sql:snapshots "),"
-       "snapshots2 AS (
-          SELECT c.id,
-                 c.label,
-                 c.revn,
-                 c.version,
-                 c.created_at,
-                 c.modified_at,
-                 c.created_by,
-                 c.locked_by,
-                 c.profile_id,
-                 c.deleted_at
-            FROM snapshots1 AS c
-           WHERE c.file_id = ?
-       ), snapshots3 AS (
-          (SELECT * FROM snapshots2
-            WHERE created_by = 'system'
-              AND (deleted_at IS NULL OR
-                   deleted_at >= ?::timestamptz)
-            LIMIT 500)
-          UNION ALL
-          (SELECT * FROM snapshots2
-            WHERE created_by = 'user'
-              AND deleted_at IS NULL
-            LIMIT 500)
-       )
-       SELECT * FROM snapshots3
-        ORDER BY created_at DESC"))
-
-(defn get-visible-snapshots
-  "Return a list of snapshots fecheable from the API, it has a limited
-  set of fields and applies big but safe limits over all available
-  snapshots. It return a ordered vector by the snapshot date of
-  creation."
-  [cfg file-id]
-  (let [now (ct/now)]
-    (->> (db/exec! cfg [sql:get-visible-snapshots file-id now])
-         (mapv decode-snapshot))))
-
-(def ^:private schema:decoded-file
-  [:map {:title "DecodedFile"}
-   [:id ::sm/uuid]
-   [:revn :int]
-   [:vern :int]
-   [:data :map]
-   [:version :int]
-   [:features ::cfeat/features]
-   [:migrations [::sm/set :string]]])
-
-(def ^:private schema:snapshot
-  [:map {:title "Snapshot"}
-   [:id ::sm/uuid]
-   [:revn [::sm/int {:min 0}]]
-   [:version [::sm/int {:min 0}]]
-   [:features ::cfeat/features]
-   [:migrations [::sm/set ::sm/text]]
-   [:profile-id {:optional true} ::sm/uuid]
-   [:label ::sm/text]
-   [:file-id ::sm/uuid]
-   [:created-by [:enum "system" "user" "admin"]]
-   [:deleted-at {:optional true} ::ct/inst]
-   [:modified-at ::ct/inst]
-   [:created-at ::ct/inst]])
-
-(def ^:private check-snapshot
-  (sm/check-fn schema:snapshot))
-
-(def ^:private check-decoded-file
-  (sm/check-fn schema:decoded-file))
-
-(defn- generate-snapshot-label
-  []
-  (let [ts (-> (ct/now)
-               (ct/format-inst)
-               (str/replace #"[T:\.]" "-")
-               (str/rtrim "Z"))]
-    (str "snapshot-" ts)))
-
-(def ^:private schema:create-params
-  [:map {:title "SnapshotCreateParams"}
-   [:profile-id ::sm/uuid]
-   [:created-by {:optional true} [:enum "user" "system"]]
-   [:label {:optional true} ::sm/text]
-   [:session-id {:optional true} ::sm/uuid]
-   [:modified-at {:optional true} ::ct/inst]
-   [:deleted-at {:optional true} ::ct/inst]])
-
-(def ^:private check-create-params
-  (sm/check-fn schema:create-params))
-
-(defn create!
-  "Create a file snapshot; expects a non-encoded file"
-  [cfg file & {:as params}]
-  (let [{:keys [label created-by deleted-at profile-id session-id]}
-        (check-create-params params)
-
-        file
-        (check-decoded-file file)
-
-        created-by
-        (or created-by "system")
-
-        snapshot-id
-        (uuid/next)
-
-        created-at
-        (ct/now)
-
-        deleted-at
-        (or deleted-at
-            (if (= created-by "system")
-              (ct/in-future (cf/get-deletion-delay))
-              nil))
-
-        label
-        (or label (generate-snapshot-label))
-
-        snapshot
-        (cond-> {:id snapshot-id
-                 :revn (:revn file)
-                 :version (:version file)
-                 :file-id (:id file)
-                 :features (:features file)
-                 :migrations (:migrations file)
-                 :label label
-                 :created-at created-at
-                 :modified-at created-at
-                 :created-by created-by}
-
-          deleted-at
-          (assoc :deleted-at deleted-at)
-
-          :always
-          (check-snapshot))]
-
-    (db/insert! cfg :file-change
-                (-> snapshot
-                    (update :features into-array)
-                    (update :migrations into-array)
-                    (assoc :updated-at created-at)
-                    (assoc :profile-id profile-id)
-                    (assoc :session-id session-id)
-                    (dissoc :modified-at))
-                {::db/return-keys false})
-
-    (fdata/upsert! cfg
-                   {:id snapshot-id
-                    :file-id (:id file)
-                    :type "snapshot"
-                    :data (blob/encode (:data file))
-                    :created-at created-at
-                    :deleted-at deleted-at})
-
-    snapshot))
-
-(def ^:private schema:update-params
-  [:map {:title "SnapshotUpdateParams"}
-   [:id ::sm/uuid]
-   [:file-id ::sm/uuid]
-   [:label ::sm/text]
-   [:modified-at {:optional true} ::ct/inst]])
-
-(def ^:private check-update-params
-  (sm/check-fn schema:update-params))
-
-(defn update!
-  [cfg params]
-
-  (let [{:keys [id file-id label modified-at]}
-        (check-update-params params)
-
-        modified-at
-        (or modified-at (ct/now))]
-
-    (db/update! cfg :file-data
-                {:deleted-at nil
-                 :modified-at modified-at}
-                {:file-id file-id
-                 :id id
-                 :type "snapshot"}
-                {::db/return-keys false})
-
-    (-> (db/update! cfg :file-change
-                    {:label label
-                     :created-by "user"
-                     :updated-at modified-at
-                     :deleted-at nil}
-                    {:file-id file-id
-                     :id id}
-                    {::db/return-keys false})
-        (db/get-update-count)
-        (pos?))))
-
-(defn restore!
-  [{:keys [::db/conn] :as cfg} file-id snapshot-id]
-  (let [file (get-minimal-file conn file-id {::db/for-update true})
-        vern (rand-int Integer/MAX_VALUE)
-
-        storage
-        (sto/resolve cfg {::db/reuse-conn true})
-
-        snapshot
-        (get-snapshot cfg file-id snapshot-id)]
-
-    (when-not snapshot
-      (ex/raise :type :not-found
-                :code :snapshot-not-found
-                :hint "unable to find snapshot with the provided label"
-                :snapshot-id snapshot-id
-                :file-id file-id))
-
-    (when-not (:data snapshot)
-      (ex/raise :type :internal
-                :code :snapshot-without-data
-                :hint "snapshot has no data"
-                :label (:label snapshot)
-                :file-id file-id))
-
-    (let [;; If the snapshot has applied migrations stored, we reuse
-          ;; them, if not, we take a safest set of migrations as
-          ;; starting point. This is because, at the time of
-          ;; implementing snapshots, migrations were not taken into
-          ;; account so we need to make this backward compatible in
-          ;; some way.
-          migrations
-          (or (:migrations snapshot)
-              (fmg/generate-migrations-from-version 67))
-
-          file
-          (-> file
-              (update :revn inc)
-              (assoc :migrations migrations)
-              (assoc :data (:data snapshot))
-              (assoc :vern vern)
-              (assoc :version (:version snapshot))
-              (assoc :has-media-trimmed false)
-              (assoc :modified-at (:modified-at snapshot))
-              (assoc :features (:features snapshot)))]
-
-      (l/dbg :hint "restoring snapshot"
-             :file-id (str file-id)
-             :label (:label snapshot)
-             :snapshot-id (str (:id snapshot)))
-
-      ;; In the same way, on reseting the file data, we need to restore
-      ;; the applied migrations on the moment of taking the snapshot
-      (bfc/update-file! cfg file ::bfc/reset-migrations? true)
-
-      ;; FIXME: this should be separated functions, we should not have
-      ;; inline sql here.
-
-      ;; clean object thumbnails
-      (let [sql (str "update file_tagged_object_thumbnail "
-                     "   set deleted_at = now() "
-                     " where file_id=? returning media_id")
-            res (db/exec! conn [sql file-id])]
-        (doseq [media-id (into #{} (keep :media-id) res)]
-          (sto/touch-object! storage media-id)))
-
-      ;; clean file thumbnails
-      (let [sql (str "update file_thumbnail "
-                     "   set deleted_at = now() "
-                     " where file_id=? returning media_id")
-            res (db/exec! conn [sql file-id])]
-        (doseq [media-id (into #{} (keep :media-id) res)]
-          (sto/touch-object! storage media-id)))
-
-      vern)))
-
-(defn delete!
-  [cfg & {:keys [id file-id deleted-at]}]
-  (assert (uuid? id) "missing id")
-  (assert (uuid? file-id) "missing file-id")
-  (assert (ct/inst? deleted-at) "missing deleted-at")
-
-  (wrk/submit! {::db/conn (db/get-connection cfg)
-                ::wrk/task :delete-object
-                ::wrk/params {:object :snapshot
-                              :deleted-at deleted-at
-                              :file-id file-id
-                              :id id}})
-  (db/update! cfg :file-change
-              {:deleted-at deleted-at}
-              {:id id :file-id file-id}
-              {::db/return-keys false})
-  true)
-
-(def ^:private sql:get-snapshots
-  (str sql:snapshots " AND c.file_id = ?"))
-
-(defn lock-by!
-  [conn id profile-id]
-  (-> (db/update! conn :file-change
-                  {:locked-by profile-id}
-                  {:id id}
-                  {::db/return-keys false})
-      (db/get-update-count)
-      (pos?)))
-
-(defn unlock!
-  [conn id]
-  (-> (db/update! conn :file-change
-                  {:locked-by nil}
-                  {:id id}
-                  {::db/return-keys false})
-      (db/get-update-count)
-      (pos?)))
-
-(defn reduce-snapshots
-  "Process the file snapshots using efficient reduction; the file
-  reduction comes with all snapshots, including maked as deleted"
-  [cfg file-id xform f init]
-  (let [conn  (db/get-connection cfg)
-        xform (comp
-               (map (partial fdata/resolve-file-data cfg))
-               (map (partial fdata/decode-file-data cfg))
-               xform)]
-
-    (->> (db/plan conn [sql:get-snapshots file-id] {:fetch-size 1})
-         (transduce xform f init))))

backend/src/app/http.clj

@@ -17,16 +17,17 @@
    [app.http.awsns :as-alias awsns]
    [app.http.debug :as-alias debug]
    [app.http.errors :as errors]
-   [app.http.management :as mgmt]
    [app.http.middleware :as mw]
-   [app.http.security :as sec]
    [app.http.session :as session]
    [app.http.websocket :as-alias ws]
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
+   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
+   [app.worker :as wrk]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
    [yetti.adapter :as yt]
@@ -53,8 +54,6 @@
   [:map
    [::port ::sm/int]
    [::host ::sm/text]
-   [::io-threads {:optional true} ::sm/int]
-   [::max-worker-threads {:optional true} ::sm/int]
    [::max-body-size {:optional true} ::sm/int]
    [::max-multipart-body-size {:optional true} ::sm/int]
    [::router {:optional true} [:fn r/router?]]
@@ -65,41 +64,31 @@
   (assert (sm/check schema:server-params params)))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [::handler ::router ::host ::port ::mtx/metrics] :as cfg}]
+  [_ {:keys [::handler ::router ::host ::port ::wrk/executor] :as cfg}]
   (l/info :hint "starting http server" :port port :host host)
-  (let [on-dispatch
+  (let [options {:http/port port
-        (fn [_ start-at-ns]
+                 :http/host host
-          (let [timing (- (System/nanoTime) start-at-ns)
+                 :http/max-body-size (::max-body-size cfg)
-                timing (int (/ timing 1000000))]
+                 :http/max-multipart-body-size (::max-multipart-body-size cfg)
-            (mtx/run! metrics
+                 :xnio/direct-buffers false
-                      :id :http-server-dispatch-timing
+                 :xnio/io-threads (or (::io-threads cfg)
-                      :val timing)))
+                                      (max 3 (px/get-available-processors)))
+                 :xnio/dispatch executor
+                 :ring/compat :ring2
+                 :socket/backlog 4069}
 
-        options
+        handler (cond
-        {:http/port port
+                  (some? router)
-         :http/host host
+                  (router-handler router)
-         :http/max-body-size (::max-body-size cfg)
-         :http/max-multipart-body-size (::max-multipart-body-size cfg)
-         :xnio/direct-buffers false
-         :xnio/io-threads (::io-threads cfg)
-         :xnio/max-worker-threads (::max-worker-threads cfg)
-         :ring/compat :ring2
-         :events/on-dispatch on-dispatch
-         :socket/backlog 4069}
 
-        handler
+                  (some? handler)
-        (cond
+                  handler
-          (some? router)
-          (router-handler router)
 
-          (some? handler)
+                  :else
-          handler
+                  (throw (UnsupportedOperationException. "handler or router are required")))
 
-          :else
+        options (d/without-nils options)
-          (throw (UnsupportedOperationException. "handler or router are required")))
+        server  (yt/server handler options)]
-
-        server
-        (yt/server handler (d/without-nils options))]
 
     (assoc cfg ::server (yt/start! server))))
 
@@ -148,12 +137,12 @@
   [:map
    [::ws/routes schema:routes]
    [::rpc/routes schema:routes]
+   [::rpc.doc/routes schema:routes]
    [::oidc/routes schema:routes]
    [::assets/routes schema:routes]
    [::debug/routes schema:routes]
    [::mtx/routes schema:routes]
    [::awsns/routes schema:routes]
-   [::mgmt/routes schema:routes]
    ::session/manager
    ::setup/props
    ::db/pool])
@@ -166,12 +155,10 @@
   [_ cfg]
   (rr/router
    [["" {:middleware [[mw/server-timing]
-                      [sec/sec-fetch-metadata]
                       [mw/params]
                       [mw/format-response]
-                      [mw/auth {:bearer (partial session/decode-token cfg)
+                      [session/soft-auth cfg]
-                                :cookie (partial session/decode-token cfg)
+                      [actoken/soft-auth cfg]
-                                :token  (partial actoken/decode-token cfg)}]
                       [mw/parse-request]
                       [mw/errors errors/handle]
                       [mw/restrict-methods]]}
@@ -183,9 +170,9 @@
      ["/webhooks"
       (::awsns/routes cfg)]
 
-     ["/management"
-      (::mgmt/routes cfg)]
-
      (::ws/routes cfg)
-     (::oidc/routes cfg)
+
-     (::rpc/routes cfg)]]))
+     ["/api" {:middleware [[mw/cors]]}
+      (::oidc/routes cfg)
+      (::rpc.doc/routes cfg)
+      (::rpc/routes cfg)]]]))

backend/src/app/http/access_token.clj

@@ -9,19 +9,23 @@
    [app.common.logging :as l]
    [app.config :as cf]
    [app.db :as db]
-   [app.http :as-alias http]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
-   [app.tokens :as tokens]))
+   [app.tokens :as tokens]
+   [yetti.request :as yreq]))
 
-(defn decode-token
+(def header-re #"^Token\s+(.*)")
-  [cfg token]
+
-  (try
+(defn- get-token
-    (tokens/verify cfg {:token token :iss "access-token"})
+  [request]
-    (catch Throwable cause
+  (some->> (yreq/get-header request "authorization")
-      (l/trc :hint "exception on decoding token"
+           (re-matches header-re)
-             :token token
+           (second)))
-             :cause cause))))
+
+(defn- decode-token
+  [props token]
+  (when token
+    (tokens/verify props {:token token :iss "access-token"})))
 
 (def sql:get-token-data
   "SELECT perms, profile_id, expires_at
@@ -31,28 +35,47 @@
            OR (expires_at > now()));")
 
 (defn- get-token-data
-  [pool claims]
+  [pool token-id]
   (when-not (db/read-only? pool)
-    (when-let [token-id (get claims :tid)]
+    (some-> (db/exec-one! pool [sql:get-token-data token-id])
-      (some-> (db/exec-one! pool [sql:get-token-data token-id])
+            (update :perms db/decode-pgarray #{}))))
-              (update :perms db/decode-pgarray #{})))))
+
+(defn- wrap-soft-auth
+  "Soft Authentication, will be executed synchronously on the undertow
+  worker thread."
+  [handler {:keys [::setup/props]}]
+  (letfn [(handle-request [request]
+            (try
+              (let [token  (get-token request)
+                    claims (decode-token props token)]
+                (cond-> request
+                  (map? claims)
+                  (assoc ::id (:tid claims))))
+              (catch Throwable cause
+                (l/trace :hint "exception on decoding malformed token" :cause cause)
+                request)))]
+
+    (fn [request]
+      (handler (handle-request request)))))
 
 (defn- wrap-authz
+  "Authorization middleware, will be executed synchronously on vthread."
   [handler {:keys [::db/pool]}]
   (fn [request]
-    (let [{:keys [type claims]} (get request ::http/auth-data)]
+    (let [{:keys [perms profile-id expires-at]} (some->> (::id request) (get-token-data pool))]
-      (if (= :token type)
+      (handler (cond-> request
-        (let [{:keys [perms profile-id expires-at]} (some->> claims (get-token-data pool))]
+                 (some? perms)
-          ;; FIXME: revisit this, this data looks unused
+                 (assoc ::perms perms)
-          (handler (cond-> request
+                 (some? profile-id)
-                     (some? perms)
+                 (assoc ::profile-id profile-id)
-                     (assoc ::perms perms)
+                 (some? expires-at)
-                     (some? profile-id)
+                 (assoc ::expires-at expires-at))))))
-                     (assoc ::profile-id profile-id)
-                     (some? expires-at)
-                     (assoc ::expires-at expires-at))))
 
-        (handler request)))))
+(def soft-auth
+  {:name ::soft-auth
+   :compile (fn [& _]
+              (when (contains? cf/flags :access-tokens)
+                wrap-soft-auth))})
 
 (def authz
   {:name ::authz

backend/src/app/http/awsns.clj

@@ -17,9 +17,11 @@
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.worker :as-alias wrk]
    [clojure.data.json :as j]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [yetti.request :as yreq]
    [yetti.response :as-alias yres]))
 
@@ -38,8 +40,8 @@
   [_ cfg]
   (letfn [(handler [request]
             (let [data (-> request yreq/body slurp)]
-              (handle-request cfg data)
+              (px/run! :vthread (partial handle-request cfg data)))
-              {::yres/status 200}))]
+            {::yres/status 200})]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 
@@ -107,7 +109,7 @@
   [cfg headers]
   (let [tdata (get headers "x-penpot-data")]
     (when-not (str/empty? tdata)
-      (let [result (tokens/verify cfg {:token tdata :iss :profile-identity})]
+      (let [result (tokens/verify (::setup/props cfg) {:token tdata :iss :profile-identity})]
         (:profile-id result)))))
 
 (defn- parse-notification

backend/src/app/http/client.clj

@@ -9,7 +9,8 @@
   (:require
    [app.common.schema :as sm]
    [integrant.core :as ig]
-   [java-http-clj.core :as http])
+   [java-http-clj.core :as http]
+   [promesa.core :as p])
   (:import
    java.net.http.HttpClient))
 
@@ -28,9 +29,14 @@
 
 (defn send!
   ([client req] (send! client req {}))
-  ([client req {:keys [response-type] :or {response-type :string}}]
+  ([client req {:keys [response-type sync?] :or {response-type :string sync? false}}]
    (assert (client? client) "expected valid http client")
-   (http/send req {:client client :as response-type})))
+   (if sync?
+     (http/send req {:client client :as response-type})
+     (try
+       (http/send-async req {:client client :as response-type})
+       (catch Throwable cause
+         (p/rejected cause))))))
 
 (defn- resolve-client
   [params]
@@ -50,8 +56,8 @@
   ([cfg-or-client request]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request {})))
+     (send! client request {:sync? true})))
   ([cfg-or-client request options]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request options))))
+     (send! client request (merge {:sync? true} options)))))

backend/src/app/http/debug.clj

@@ -27,7 +27,6 @@
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
-   [app.setup.clock :as clock]
    [app.srepl.main :as srepl]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
@@ -50,17 +49,11 @@
 
 (defn index-handler
   [_cfg _request]
-  (let [{:keys [clock offset]} @clock/current]
+  {::yres/status  200
-    {::yres/status  200
+   ::yres/headers {"content-type" "text/html"}
-     ::yres/headers {"content-type" "text/html"}
+   ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
-     ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
+                      (tmpl/render {:version (:full cf/version)
-                        (tmpl/render {:version (:full cf/version)
+                                    :supported-features cfeat/supported-features}))})
-                                      :current-clock  (str clock)
-                                      :current-offset (if offset
-                                                        (ct/format-duration offset)
-                                                        "NO OFFSET")
-                                      :current-time  (ct/format-inst (ct/now) :http)
-                                      :supported-features cfeat/supported-features}))}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE CHANGES
@@ -397,6 +390,34 @@
                            ::yres/headers {"content-type" "text/plain"}
                            ::yres/body    (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))))
 
+
+(defn- reset-file-version
+  [cfg {:keys [params] :as request}]
+  (let [file-id (some-> params :file-id d/parse-uuid)
+        version (some-> params :version d/parse-integer)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? file-id)
+      (ex/raise :type :validation
+                :code :invalid-file-id
+                :hint "provided invalid file id"))
+
+    (when (nil? version)
+      (ex/raise :type :validation
+                :code :invalid-version
+                :hint "provided invalid version"))
+
+    (db/tx-run! cfg srepl/process-file! file-id #(assoc % :version version))
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
+
 (defn- handle-team-features
   [cfg {:keys [params] :as request}]
   (let [team-id    (some-> params :team-id d/parse-uuid)
@@ -441,24 +462,6 @@
          ::yres/headers {"content-type" "text/plain"}
          ::yres/body    "OK"}))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; VIRTUAL CLOCK
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- set-virtual-clock
-  [_ {:keys [params] :as request}]
-  (let [offset (some-> params :offset str/trim not-empty ct/duration)
-        reset? (contains? params :reset)]
-    (if (= "production" (cf/get :tenant))
-      {::yres/status 501
-       ::yres/body "OPERATION NOT ALLOWED"}
-      (do
-        (if (or reset? (zero? (inst-ms offset)))
-          (clock/set-offset! nil)
-          (clock/set-offset! offset))
-        {::yres/status 302
-         ::yres/headers {"location" "/dbg"}}))))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -545,10 +548,10 @@
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
     ["/actions" {:middleware [[errors]]}
-     ["/set-virtual-clock"
-      {:handler (partial set-virtual-clock cfg)}]
      ["/resend-email-verification"
       {:handler (partial resend-email-notification cfg)}]
+     ["/reset-file-version"
+      {:handler (partial reset-file-version cfg)}]
      ["/handle-team-features"
       {:handler (partial handle-team-features cfg)}]
      ["/file-export" {:handler (partial export-handler cfg)}]

backend/src/app/http/errors.clj

@@ -13,7 +13,6 @@
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
-   [app.http.auth :as-alias auth]
    [app.http.session :as-alias session]
    [app.util.inet :as inet]
    [clojure.spec.alpha :as s]
@@ -23,16 +22,18 @@
 (defn request->context
   "Extracts error report relevant context data from request."
   [request]
-  (let [{:keys [claims] :as auth} (get request ::http/auth-data)]
+  (let [claims (-> {}
-    (-> (cf/logging-context)
+                   (into (::session/token-claims request))
-        (assoc :request/path (:path request))
+                   (into (::actoken/token-claims request)))]
-        (assoc :request/method (:method request))
+    {:request/path       (:path request)
-        (assoc :request/params (:params request))
+     :request/method     (:method request)
-        (assoc :request/user-agent (yreq/get-header request "user-agent"))
+     :request/params     (:params request)
-        (assoc :request/ip-addr (inet/parse-request request))
+     :request/user-agent (yreq/get-header request "user-agent")
-        (assoc :request/profile-id (get claims :uid))
+     :request/ip-addr    (inet/parse-request request)
-        (assoc :request/auth-data auth)
+     :request/profile-id (:uid claims)
-        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
+     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
+     :version/backend    (:full cf/version)}))
+
 
 (defmulti handle-error
   (fn [cause _ _]
@@ -60,6 +61,8 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
+        (l/err :hint "restriction error"
+               :cause err)
         {::yres/status 400
          ::yres/body data}))))
 

backend/src/app/http/management.clj

@@ -1,240 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.management
-  "Internal mangement HTTP API"
-  (:require
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.schema.generators :as sg]
-   [app.common.time :as ct]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.http.middleware :as mw]
-   [app.main :as-alias main]
-   [app.rpc.commands.profile :as cmd.profile]
-   [app.setup :as-alias setup]
-   [app.tokens :as tokens]
-   [app.worker :as-alias wrk]
-   [integrant.core :as ig]
-   [yetti.response :as-alias yres]))
-
-;; ---- ROUTES
-
-(declare ^:private authenticate)
-(declare ^:private get-customer)
-(declare ^:private update-customer)
-
-(defmethod ig/assert-key ::routes
-  [_ params]
-  (assert (db/pool? (::db/pool params)) "expect valid database pool"))
-
-(def ^:private default-system
-  {:name ::default-system
-   :compile
-   (fn [_ _]
-     (fn [handler cfg]
-       (fn [request]
-         (handler cfg request))))})
-
-(def ^:private transaction
-  {:name ::transaction
-   :compile
-   (fn [data _]
-     (when (:transaction data)
-       (fn [handler]
-         (fn [cfg request]
-           (db/tx-run! cfg handler request)))))})
-
-(defmethod ig/init-key ::routes
-  [_ {:keys [::setup/props] :as cfg}]
-
-  (let [management-key (or (cf/get :management-api-key)
-                           (get props :management-key))]
-
-    ["" {:middleware [[mw/shared-key-auth management-key]
-                      [default-system cfg]
-                      [transaction]]}
-     ["/authenticate"
-      {:handler authenticate
-       :allowed-methods #{:post}}]
-
-     ["/get-customer"
-      {:handler get-customer
-       :transaction true
-       :allowed-methods #{:post}}]
-
-     ["/update-customer"
-      {:handler update-customer
-       :allowed-methods #{:post}
-       :transaction true}]]))
-
-;; ---- HELPERS
-
-(defn- coercer
-  [schema & {:as opts}]
-  (let [decode-fn (sm/decoder schema sm/json-transformer)
-        check-fn  (sm/check-fn schema opts)]
-    (fn [data]
-      (-> data decode-fn check-fn))))
-
-;; ---- API: AUTHENTICATE
-
-(defn- authenticate
-  [cfg request]
-  (let [token  (-> request :params :token)
-        result (tokens/verify cfg {:token token :iss "authentication"})]
-    {::yres/status 200
-     ::yres/body result}))
-
-;; ---- API: GET-CUSTOMER
-
-(def ^:private schema:get-customer
-  [:map [:id ::sm/uuid]])
-
-(def ^:private coerce-get-customer-params
-  (coercer schema:get-customer
-           :type :validation
-           :hint "invalid data provided for `get-customer` rpc call"))
-
-(def ^:private sql:get-customer-slots
-  "WITH teams AS (
-    SELECT tpr.team_id AS id,
-           tpr.profile_id AS profile_id
-      FROM team_profile_rel AS tpr
-     WHERE tpr.is_owner IS true
-       AND tpr.profile_id = ?
-  ), teams_with_slots AS (
-    SELECT tpr.team_id AS id,
-           count(*) AS total
-      FROM team_profile_rel AS tpr
-     WHERE tpr.team_id IN (SELECT id FROM teams)
-       AND tpr.can_edit IS true
-     GROUP BY 1
-     ORDER BY 2
-  )
-  SELECT max(total) AS total FROM teams_with_slots;")
-
-(defn- get-customer-slots
-  [cfg profile-id]
-  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
-    (:total result)))
-
-(defn- get-customer
-  [cfg request]
-  (let [profile-id (-> request :params coerce-get-customer-params :id)
-        profile    (cmd.profile/get-profile cfg profile-id)
-        result     {:id (get profile :id)
-                    :name (get profile :fullname)
-                    :email (get profile :email)
-                    :num-editors (get-customer-slots cfg profile-id)
-                    :subscription (-> profile :props :subscription)}]
-    {::yres/status 200
-     ::yres/body result}))
-
-
-;; ---- API: UPDATE-CUSTOMER
-
-(def ^:private schema:timestamp
-  (sm/type-schema
-   {:type ::timestamp
-    :pred ct/inst?
-    :type-properties
-    {:title "inst"
-     :description "The same as :app.common.time/inst but encodes to epoch"
-     :error/message "should be an instant"
-     :gen/gen (->> (sg/small-int)
-                   (sg/fmap (fn [v] (ct/inst v))))
-     :decode/string ct/inst
-     :encode/string inst-ms
-     :decode/json ct/inst
-     :encode/json inst-ms}}))
-
-(def ^:private schema:subscription
-  [:map {:title "Subscription"}
-   [:id ::sm/text]
-   [:customer-id ::sm/text]
-   [:type [:enum
-           "unlimited"
-           "professional"
-           "enterprise"]]
-   [:status [:enum
-             "active"
-             "canceled"
-             "incomplete"
-             "incomplete_expired"
-             "past_due"
-             "paused"
-             "trialing"
-             "unpaid"]]
-
-   [:billing-period [:enum
-                     "month"
-                     "day"
-                     "week"
-                     "year"]]
-   [:quantity :int]
-   [:description [:maybe ::sm/text]]
-   [:created-at schema:timestamp]
-   [:start-date [:maybe schema:timestamp]]
-   [:ended-at [:maybe schema:timestamp]]
-   [:trial-end [:maybe schema:timestamp]]
-   [:trial-start [:maybe schema:timestamp]]
-   [:cancel-at [:maybe schema:timestamp]]
-   [:canceled-at [:maybe schema:timestamp]]
-   [:current-period-end [:maybe schema:timestamp]]
-   [:current-period-start [:maybe schema:timestamp]]
-   [:cancel-at-period-end :boolean]
-
-   [:cancellation-details
-    [:map {:title "CancellationDetails"}
-     [:comment [:maybe ::sm/text]]
-     [:reason [:maybe ::sm/text]]
-     [:feedback [:maybe
-                 [:enum
-                  "customer_service"
-                  "low_quality"
-                  "missing_feature"
-                  "other"
-                  "switched_service"
-                  "too_complex"
-                  "too_expensive"
-                  "unused"]]]]]])
-
-(def ^:private schema:update-customer
-  [:map
-   [:id ::sm/uuid]
-   [:subscription [:maybe schema:subscription]]])
-
-(def ^:private coerce-update-customer-params
-  (coercer schema:update-customer
-           :type :validation
-           :hint "invalid data provided for `update-customer` rpc call"))
-
-(defn- update-customer
-  [cfg request]
-  (let [{:keys [id subscription]}
-        (-> request :params coerce-update-customer-params)
-
-        {:keys [props] :as profile}
-        (cmd.profile/get-profile cfg id ::db/for-update true)
-
-        props
-        (assoc props :subscription subscription)]
-
-    (l/dbg :hint "update customer"
-           :profile-id (str id)
-           :subscription-type (get subscription :type)
-           :subscription-status (get subscription :status)
-           :subscription-quantity (get subscription :quantity))
-
-    (db/update! cfg :profile
-                {:props (db/tjson props)}
-                {:id id}
-                {::db/return-keys false})
-
-    {::yres/status 201
-     ::yres/body nil}))

backend/src/app/http/middleware.clj

@@ -12,11 +12,8 @@
    [app.common.schema :as-alias sm]
    [app.common.transit :as t]
    [app.config :as cf]
-   [app.http :as-alias http]
    [app.http.errors :as errors]
-   [app.tokens :as tokens]
    [app.util.pointer-map :as pmap]
-   [buddy.core.codecs :as bc]
    [cuerdas.core :as str]
    [yetti.adapter :as yt]
    [yetti.middleware :as ymw]
@@ -243,77 +240,3 @@
              (if (contains? allowed method)
                (handler request)
                {::yres/status 405}))))))})
-
-(defn- wrap-auth
-  [handler decoders]
-  (let [token-re
-        #"(?i)^(Token|Bearer)\s+(.*)"
-
-        get-token-from-authorization
-        (fn [request]
-          (when-let [[_ token-type token] (some->> (yreq/get-header request "authorization")
-                                                   (re-matches token-re))]
-            (if (= "token" (str/lower token-type))
-              {:type :token
-               :token token}
-              {:type :bearer
-               :token token})))
-
-        get-token-from-cookie
-        (fn [request]
-          (let [cname (cf/get :auth-token-cookie-name)
-                token (some-> (yreq/get-cookie request cname) :value)]
-            (when-not (str/empty? token)
-              {:type :cookie
-               :token token})))
-
-        get-token
-        (some-fn get-token-from-cookie get-token-from-authorization)
-
-        process-request
-        (fn [request]
-          (if-let [{:keys [type token] :as auth} (get-token request)]
-            (let [decode-fn (get decoders type)]
-              (if (or (= type :cookie) (= type :bearer))
-                (let [metadata (tokens/decode-header token)]
-                  ;; NOTE: we only proceed to decode claims on new
-                  ;; cookie tokens. The old cookies dont need to be
-                  ;; decoded because they use the token string as ID
-                  (if (and (= (:kid metadata) 1)
-                           (= (:ver metadata) 1)
-                           (some? decode-fn))
-                    (assoc request ::http/auth-data (assoc auth
-                                                           :claims (decode-fn token)
-                                                           :metadata metadata))
-                    (assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
-
-                (if decode-fn
-                  (assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
-                  (assoc request ::http/auth-data auth))))
-
-            request))]
-
-    (fn [request]
-      (-> request process-request handler))))
-
-(def auth
-  {:name ::auth
-   :compile (constantly wrap-auth)})
-
-(defn- wrap-shared-key-auth
-  [handler shared-key]
-  (if shared-key
-    (let [shared-key (if (string? shared-key)
-                       shared-key
-                       (bc/bytes->b64-str shared-key true))]
-      (fn [request]
-        (let [key (yreq/get-header request "x-shared-key")]
-          (if (= key shared-key)
-            (handler request)
-            {::yres/status 403}))))
-    (fn [_ _]
-      {::yres/status 403})))
-
-(def shared-key-auth
-  {:name ::shared-key-auth
-   :compile (constantly wrap-shared-key-auth)})

backend/src/app/http/security.clj

@@ -1,55 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.security
-  "Additional security layer middlewares"
-  (:require
-   [app.config :as cf]
-   [yetti.request :as yreq]
-   [yetti.response :as yres]))
-
-(def ^:private safe-methods
-  #{:get :head :options})
-
-(defn- wrap-sec-fetch-metadata
-  "Sec-Fetch metadata security layer middleware"
-  [handler]
-  (fn [request]
-    (let [site (yreq/get-header request "sec-fetch-site")]
-      (cond
-        (= site "same-origin")
-        (handler request)
-
-        (or (= site "same-site")
-            (= site "cross-site"))
-        (if (contains? safe-methods (yreq/method request))
-          (handler request)
-          {::yres/status 403})
-
-        :else
-        (handler request)))))
-
-(def sec-fetch-metadata
-  {:name ::sec-fetch-metadata
-   :compile (fn [_ _]
-              (when (contains? cf/flags :sec-fetch-metadata-middleware)
-                wrap-sec-fetch-metadata))})
-
-(defn- wrap-client-header-check
-  "Check for a penpot custom header to be present as additional CSRF
-  protection"
-  [handler]
-  (fn [request]
-    (let [client (yreq/get-header request "x-client")]
-      (if (some? client)
-        (handler request)
-        {::yres/status 403}))))
-
-(def client-header-check
-  {:name ::client-header-check
-   :compile (fn [_ _]
-              (when (contains? cf/flags :client-header-check-middleware)
-                wrap-client-header-check))})

backend/src/app/http/session.clj

@@ -11,24 +11,29 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
-   [app.common.uuid :as uuid]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.http :as-alias http]
-   [app.http.auth :as-alias http.auth]
    [app.http.session.tasks :as-alias tasks]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [cuerdas.core :as str]
    [integrant.core :as ig]
-   [yetti.request :as yreq]
+   [yetti.request :as yreq]))
-   [yetti.response :as yres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; A default cookie name for storing the session.
+(def default-auth-token-cookie-name "auth-token")
+
+;; A cookie that we can use to check from other sites of the same
+;; domain if a user is authenticated.
+(def default-auth-data-cookie-name "auth-data")
+
 ;; Default value for cookie max-age
 (def default-cookie-max-age (ct/duration {:days 7}))
 
@@ -40,10 +45,10 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol ISessionManager
-  (read-session [_ id])
+  (read [_ key])
-  (create-session [_ params])
+  (write! [_ key data])
-  (update-session [_ session])
+  (update! [_ data])
-  (delete-session [_ id]))
+  (delete! [_ key]))
 
 (defn manager?
   [o]
@@ -58,82 +63,67 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private schema:params
-  [:map {:title "SessionParams" :closed true}
+  [:map {:title "session-params"}
+   [:user-agent ::sm/text]
    [:profile-id ::sm/uuid]
-   [:user-agent {:optional true} ::sm/text]
+   [:created-at ::ct/inst]])
-   [:sso-provider-id {:optional true} ::sm/uuid]
-   [:sso-session-id {:optional true} :string]])
 
 (def ^:private valid-params?
   (sm/validator schema:params))
 
+(defn- prepare-session-params
+  [key params]
+  (assert (string? key) "expected key to be a string")
+  (assert (not (str/blank? key)) "expected key to be not empty")
+  (assert (valid-params? params) "expected valid params")
+
+  {:user-agent (:user-agent params)
+   :profile-id (:profile-id params)
+   :created-at (:created-at params)
+   :updated-at (:created-at params)
+   :id key})
+
 (defn- database-manager
   [pool]
   (reify ISessionManager
-    (read-session [_ id]
+    (read [_ token]
-      (if (string? id)
+      (db/exec-one! pool (sql/select :http-session {:id token})))
-        ;; Backward compatibility
-        (let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
-          (-> session
-              (assoc :modified-at (:updated-at session))
-              (dissoc :updated-at)))
-        (db/exec-one! pool (sql/select :http-session-v2 {:id id}))))
 
-    (create-session [_ params]
+    (write! [_ key params]
-      (assert (valid-params? params) "expect valid session params")
+      (let [params (prepare-session-params key params)]
+        (db/insert! pool :http-session params)
+        params))
 
-      (let [now    (ct/now)
+    (update! [_ params]
-            params (-> params
+      (let [updated-at (ct/now)]
-                       (assoc :id (uuid/next))
+        (db/update! pool :http-session
-                       (assoc :created-at now)
+                    {:updated-at updated-at}
-                       (assoc :modified-at now))]
+                    {:id (:id params)})
-        (db/insert! pool :http-session-v2 params
+        (assoc params :updated-at updated-at)))
-                    {::db/return-keys true})))
 
-    (update-session [_ session]
+    (delete! [_ token]
-      (let [modified-at (ct/now)]
+      (db/delete! pool :http-session {:id token})
-        (if (string? (:id session))
-          (db/insert! pool :http-session-v2
-                      (-> session
-                          (assoc :id (uuid/next))
-                          (assoc :created-at modified-at)
-                          (assoc :modified-at modified-at)))
-          (db/update! pool :http-session-v2
-                      {:modified-at modified-at}
-                      {:id (:id session)}
-                      {::db/return-keys true}))))
-
-    (delete-session [_ id]
-      (if (string? id)
-        (db/delete! pool :http-session {:id id} {::db/return-keys false})
-        (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
       nil)))
 
 (defn inmemory-manager
   []
   (let [cache (atom {})]
     (reify ISessionManager
-      (read-session [_ id]
+      (read [_ token]
-        (get @cache id))
+        (get @cache token))
 
-      (create-session [_ params]
+      (write! [_ key params]
-        (assert (valid-params? params) "expect valid session params")
+        (let [params (prepare-session-params key params)]
+          (swap! cache assoc key params)
+          params))
 
-        (let [now     (ct/now)
+      (update! [_ params]
-              session (-> params
+        (let [updated-at (ct/now)]
-                          (assoc :id (uuid/next))
+          (swap! cache update (:id params) assoc :updated-at updated-at)
-                          (assoc :created-at now)
+          (assoc params :updated-at updated-at)))
-                          (assoc :modified-at now))]
-          (swap! cache assoc (:id session) session)
-          session))
 
-      (update-session [_ session]
+      (delete! [_ token]
-        (let [modified-at (ct/now)]
+        (swap! cache dissoc token)
-          (swap! cache update (:id session) assoc :modified-at modified-at)
-          (assoc session :modified-at modified-at)))
-
-      (delete-session [_ id]
-        (swap! cache dissoc id)
         nil))))
 
 (defmethod ig/assert-key ::manager
@@ -153,116 +143,109 @@
 ;; MANAGER IMPL
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare ^:private assign-session-cookie)
+(declare ^:private assign-auth-token-cookie)
-(declare ^:private clear-session-cookie)
+(declare ^:private assign-auth-data-cookie)
-
+(declare ^:private clear-auth-token-cookie)
-(defn- assign-token
+(declare ^:private clear-auth-data-cookie)
-  [cfg session]
+(declare ^:private gen-token)
-  (let [claims {:iss "authentication"
-                :aud "penpot"
-                :sid (:id session)
-                :iat (:modified-at session)
-                :uid (:profile-id session)
-                :sso-provider-id (:sso-provider-id session)
-                :sso-session-id (:sso-session-id session)}
-        header {:kid 1 :ver 1}
-        token  (tokens/generate cfg claims header)]
-    (assoc session :token token)))
 
 (defn create-fn
-  [{:keys [::manager] :as cfg} {profile-id :id :as profile}
+  [{:keys [::manager ::setup/props]} profile-id]
-   & {:keys [sso-provider-id sso-session-id]}]
-
   (assert (manager? manager) "expected valid session manager")
   (assert (uuid? profile-id) "expected valid uuid for profile-id")
 
   (fn [request response]
     (let [uagent  (yreq/get-header request "user-agent")
-          session (->> {:user-agent uagent
+          params  {:profile-id profile-id
-                        :profile-id profile-id
+                   :user-agent uagent
-                        :sso-provider-id sso-provider-id
+                   :created-at (ct/now)}
-                        :sso-session-id sso-session-id}
+          token   (gen-token props params)
-                       (d/without-nils)
+          session (write! manager token params)]
-                       (create-session manager)
+      (l/trace :hint "create" :profile-id (str profile-id))
-                       (assign-token cfg))]
+      (-> response
-
+          (assign-auth-token-cookie session)
-      (l/trc :hint "create" :id (str (:id session)) :profile-id (str profile-id))
+          (assign-auth-data-cookie session)))))
-      (assign-session-cookie response session))))
 
 (defn delete-fn
   [{:keys [::manager]}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request response]
-    (some->> (get request ::id) (delete-session manager))
+    (let [cname   (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
-    (clear-session-cookie response)))
+          cookie  (yreq/get-cookie request cname)]
+      (l/trace :hint "delete" :profile-id (:profile-id request))
+      (some->> (:value cookie) (delete! manager))
+      (-> response
+          (assoc :status 204)
+          (assoc :body nil)
+          (clear-auth-token-cookie)
+          (clear-auth-data-cookie)))))
 
-(defn decode-token
+(defn- gen-token
-  [cfg token]
+  [props {:keys [profile-id created-at]}]
-  (try
+  (tokens/generate props {:iss "authentication"
-    (tokens/verify cfg {:token token :iss "authentication"})
+                          :iat created-at
-    (catch Throwable cause
+                          :uid profile-id}))
-      (l/trc :hint "exception on decoding token"
+(defn- decode-token
-             :token token
+  [props token]
-             :cause cause))))
+  (when token
+    (tokens/verify props {:token token :iss "authentication"})))
 
-(defn get-session
+(defn- get-token
   [request]
-  (get request ::session))
+  (let [cname  (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+        cookie (some-> (yreq/get-cookie request cname) :value)]
+    (when-not (str/empty? cookie)
+      cookie)))
 
-(defn invalidate-others
+(defn- get-session
-  [cfg session]
+  [manager token]
-  (let [sql "delete from http_session_v2 where profile_id = ? and id != ?"]
+  (some->> token (read manager)))
-    (-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
-        (db/get-update-count))))
 
 (defn- renew-session?
-  [{:keys [id modified-at] :as session}]
+  [{:keys [updated-at] :as session}]
-  (or (string? id)
+  (and (ct/inst? updated-at)
-      (and (ct/inst? modified-at)
+       (let [elapsed (ct/diff updated-at (ct/now))]
-           (let [elapsed (ct/diff modified-at (ct/now))]
+         (neg? (compare default-renewal-max-age elapsed)))))
-             (neg? (compare default-renewal-max-age elapsed))))))
+
+(defn- wrap-soft-auth
+  [handler {:keys [::manager ::setup/props]}]
+  (assert (manager? manager) "expected valid session manager")
+  (letfn [(handle-request [request]
+            (try
+              (let [token  (get-token request)
+                    claims (decode-token props token)]
+                (cond-> request
+                  (map? claims)
+                  (-> (assoc ::token-claims claims)
+                      (assoc ::token token))))
+              (catch Throwable cause
+                (l/trace :hint "exception on decoding malformed token" :cause cause)
+                request)))]
+
+    (fn [request]
+      (handler (handle-request request)))))
 
 (defn- wrap-authz
-  [handler {:keys [::manager] :as cfg}]
+  [handler {:keys [::manager]}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request]
-    (let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
+    (let [session  (get-session manager (::token request))
-      (cond
+          request  (cond-> request
-        (= type :cookie)
+                     (some? session)
-        (let [session (case (:ver metadata)
+                     (assoc ::profile-id (:profile-id session)
-                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
+                            ::id (:id session)))
-                        0 (read-session manager token)
+          response (handler request)]
-                        1 (some->> (:sid claims) (read-session manager))
-                        nil)
 
-              request (cond-> request
+      (if (renew-session? session)
-                        (some? session)
+        (let [session (update! manager session)]
-                        (-> (assoc ::profile-id (:profile-id session))
+          (-> response
-                            (assoc ::session session)))
+              (assign-auth-token-cookie session)
+              (assign-auth-data-cookie session)))
+        response))))
 
-              response (handler request)]
+(def soft-auth
-
+  {:name ::soft-auth
-          (if (and session (renew-session? session))
+   :compile (constantly wrap-soft-auth)})
-            (let [session (->> session
-                               (update-session manager)
-                               (assign-token cfg))]
-              (assign-session-cookie response session))
-            response))
-
-        (= type :bearer)
-        (let [session (case (:ver metadata)
-                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
-                        0 (read-session manager token)
-                        1 (some->> (:sid claims) (read-session manager))
-                        nil)
-              request (cond-> request
-                        (some? session)
-                        (-> (assoc ::profile-id (:profile-id session))
-                            (assoc ::session session)))]
-          (handler request))
-
-        :else
-        (handler request)))))
 
 (def authz
   {:name ::authz
@@ -270,16 +253,16 @@
 
 ;; --- IMPL
 
-(defn- assign-session-cookie
+(defn- assign-auth-token-cookie
-  [response {token :token modified-at :modified-at}]
+  [response {token :id updated-at :updated-at}]
   (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        created-at modified-at
+        created-at (or updated-at (ct/now))
         renewal    (ct/plus created-at default-renewal-max-age)
         expires    (ct/plus created-at max-age)
         secure?    (contains? cf/flags :secure-session-cookies)
         strict?    (contains? cf/flags :strict-session-cookies)
         cors?      (contains? cf/flags :cors)
-        name       (cf/get :auth-token-cookie-name)
+        name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
         comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
         cookie     {:path "/"
                     :http-only true
@@ -288,17 +271,55 @@
                     :comment comment
                     :same-site (if cors? :none (if strict? :strict :lax))
                     :secure secure?}]
-    (update response ::yres/cookies assoc name cookie)))
+    (update response :cookies assoc name cookie)))
 
-(defn- clear-session-cookie
+(defn- assign-auth-data-cookie
+  [response {profile-id :profile-id updated-at :updated-at}]
+  (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
+        domain     (cf/get :auth-data-cookie-domain)
+        cname      default-auth-data-cookie-name
+
+        created-at (or updated-at (ct/now))
+        renewal    (ct/plus created-at default-renewal-max-age)
+        expires    (ct/plus created-at max-age)
+
+        comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
+        secure?    (contains? cf/flags :secure-session-cookies)
+        strict?    (contains? cf/flags :strict-session-cookies)
+        cors?      (contains? cf/flags :cors)
+
+        cookie     {:domain domain
+                    :expires expires
+                    :path "/"
+                    :comment comment
+                    :value (u/map->query-string {:profile-id profile-id})
+                    :same-site (if cors? :none (if strict? :strict :lax))
+                    :secure secure?}]
+
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc cname cookie))))
+
+(defn- clear-auth-token-cookie
   [response]
-  (let [cname (cf/get :auth-token-cookie-name)]
+  (let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
-    (update response ::yres/cookies assoc cname {:path "/" :value "" :max-age 0})))
+    (update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
+
+(defn- clear-auth-data-cookie
+  [response]
+  (let [cname  default-auth-data-cookie-name
+        domain (cf/get :auth-data-cookie-domain)]
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc cname {:domain domain :path "/" :value "" :max-age 0}))))
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASK: SESSION GC
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; FIXME: MOVE
+
 (defmethod ig/assert-key ::tasks/gc
   [_ params]
   (assert (db/pool? (::db/pool params)) "expected valid database pool")
@@ -311,23 +332,22 @@
 
 (def ^:private
   sql:delete-expired
-  "DELETE FROM http_session
+  "delete from http_session
-    WHERE updated_at < ?::timestamptz
+    where updated_at < now() - ?::interval
        or (updated_at is null and
-           created_at < ?::timestamptz)")
+           created_at < now() - ?::interval)")
 
 (defn- collect-expired-tasks
   [{:keys [::db/conn ::tasks/max-age]}]
-  (let [threshold (ct/minus (ct/now) max-age)
+  (let [interval (db/interval max-age)
-        result    (-> (db/exec-one! conn [sql:delete-expired threshold threshold])
+        result   (db/exec-one! conn [sql:delete-expired interval interval])
-                      (db/get-update-count))]
+        result   (:next.jdbc/update-count result)]
-    (l/dbg :task "gc"
+    (l/debug :task "gc"
-           :hint "clean http sessions"
+             :hint "clean http sessions"
-           :deleted result)
+             :deleted result)
     result))
 
 (defmethod ig/init-key ::tasks/gc
   [_ {:keys [::tasks/max-age] :as cfg}]
-  (l/dbg :hint "initializing session gc task" :max-age max-age)
+  (l/debug :hint "initializing session gc task" :max-age max-age)
-  (fn [_]
+  (fn [_] (db/tx-run! cfg collect-expired-tasks)))
-    (db/tx-run! cfg collect-expired-tasks)))

backend/src/app/http/sse.clj

@@ -33,7 +33,7 @@
                  (println "event:" (d/name name))
                  (println "data:" (t/encode-str data {:type :json-verbose}))
                  (println))]
-      (.getBytes ^String data "UTF-8"))
+      (.getBytes data "UTF-8"))
     (catch Throwable cause
       (l/err :hint "unexpected error on encoding value on sse stream"
              :cause cause)
@@ -44,8 +44,7 @@
 (def default-headers
   {"Content-Type" "text/event-stream;charset=UTF-8"
    "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
-   "Pragma" "no-cache"
+   "Pragma" "no-cache"})
-   "X-Accel-Buffering" "no"})
 
 (defn response
   [handler & {:keys [buf] :or {buf 32} :as opts}]
@@ -55,7 +54,7 @@
      ::yres/body (yres/stream-body
                   (fn [_ output]
                     (let [channel  (sp/chan :buf buf :xf (keep encode))
-                          listener (events/spawn-listener
+                          listener (events/start-listener
                                     channel
                                     (partial write! output)
                                     (partial pu/close! output))]

backend/src/app/loggers/audit.clj

@@ -25,8 +25,7 @@
    [app.util.inet :as inet]
    [app.util.services :as-alias sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]
+   [cuerdas.core :as str]))
-   [yetti.request :as yreq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -91,22 +90,6 @@
      ::ip-addr (::rpc/ip-addr params)
      ::context (d/without-nils context)}))
 
-(defn get-external-session-id
-  [request]
-  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-    (when-not (or (> (count session-id) 256)
-                  (= session-id "null")
-                  (str/blank? session-id))
-      session-id)))
-
-(defn- get-external-event-origin
-  [request]
-  (when-let [origin (yreq/get-header request "x-event-origin")]
-    (when-not (or (> (count origin) 256)
-                  (= origin "null")
-                  (str/blank? origin))
-      origin)))
-
 ;; --- SPECS
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -143,6 +126,8 @@
                          (::rpc/profile-id params)
                          uuid/zero)
 
+        session-id   (get params ::rpc/external-session-id)
+        event-origin (get params ::rpc/external-event-origin)
         props        (-> (or (::replace-props resultm)
                              (-> params
                                  (merge (::props resultm))
@@ -153,10 +138,8 @@
 
         token-id     (::actoken/id request)
         context      (-> (::context resultm)
-                         (assoc :external-session-id
+                         (assoc :external-session-id session-id)
-                                (get-external-session-id request))
+                         (assoc :external-event-origin event-origin)
-                         (assoc :external-event-origin
-                                (get-external-event-origin request))
                          (assoc :access-token-id (some-> token-id str))
                          (d/without-nils))
 

backend/src/app/loggers/audit/archive_task.clj

@@ -9,6 +9,7 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -52,12 +53,13 @@
 
 (defn- send!
   [{:keys [::uri] :as cfg} events]
-  (let [token   (tokens/generate cfg
+  (let [token   (tokens/generate (::setup/props cfg)
                                  {:iss "authentication"
+                                  :iat (ct/now)
                                   :uid uuid/zero})
         body    (t/encode {:events events})
         headers {"content-type" "application/transit+json"
-                 "origin" (str (cf/get :public-uri))
+                 "origin" (cf/get :public-uri)
                  "cookie" (u/map->query-string {:auth-token token})}
         params  {:uri uri
                  :timeout 12000

backend/src/app/loggers/database.clj

@@ -49,7 +49,7 @@
           ctx  (-> context
                    (assoc :tenant (cf/get :tenant))
                    (assoc :host (cf/get :host))
-                   (assoc :public-uri (str (cf/get :public-uri)))
+                   (assoc :public-uri (cf/get :public-uri))
                    (assoc :logger/name logger)
                    (assoc :logger/level level)
                    (dissoc :request/params :value :params :data))]

backend/src/app/main.clj

@@ -20,8 +20,7 @@
    [app.http.awsns :as http.awsns]
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
-   [app.http.management :as mgmt]
+   [app.http.session :as-alias session]
-   [app.http.session :as session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
    [app.loggers.webhooks :as-alias webhooks]
@@ -31,6 +30,7 @@
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
+   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [app.srepl :as-alias srepl]
    [app.storage :as-alias sto]
@@ -41,7 +41,6 @@
    [app.svgo :as-alias svgo]
    [app.util.cron]
    [app.worker :as-alias wrk]
-   [app.worker.executor]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -148,11 +147,23 @@
     ::mdef/labels []
     ::mdef/type :histogram}
 
-   :http-server-dispatch-timing
+   :executors-active-threads
-   {::mdef/name "penpot_http_server_dispatch_timing"
+   {::mdef/name "penpot_executors_active_threads"
-    ::mdef/help "Histogram of dispatch handler"
+    ::mdef/help "Current number of threads available in the executor service."
-    ::mdef/labels []
+    ::mdef/labels ["name"]
-    ::mdef/type :histogram}})
+    ::mdef/type :gauge}
+
+   :executors-completed-tasks
+   {::mdef/name "penpot_executors_completed_tasks_total"
+    ::mdef/help "Approximate number of completed tasks by the executor."
+    ::mdef/labels ["name"]
+    ::mdef/type :counter}
+
+   :executors-running-threads
+   {::mdef/name "penpot_executors_running_threads"
+    ::mdef/help "Current number of threads with state RUNNING."
+    ::mdef/labels ["name"]
+    ::mdef/type :gauge}})
 
 (def system-config
   {::db/pool
@@ -164,12 +175,14 @@
     ::db/max-size   (cf/get :database-max-pool-size 60)
     ::mtx/metrics   (ig/ref ::mtx/metrics)}
 
-   ;; Default netty IO pool (shared between several services)
+   ;; Default thread pool for IO operations
-   ::wrk/netty-io-executor
+   ::wrk/executor
-   {:threads (cf/get :netty-io-threads)}
+   {}
 
-   ::wrk/netty-executor
+   ::wrk/monitor
-   {:threads (cf/get :executor-threads)}
+   {::mtx/metrics  (ig/ref ::mtx/metrics)
+    ::wrk/executor (ig/ref ::wrk/executor)
+    ::wrk/name     "default"}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -180,27 +193,17 @@
    ::mtx/routes
    {::mtx/metrics (ig/ref ::mtx/metrics)}
 
-   ::rds/client
+   ::rds/redis
-   {::rds/uri
+   {::rds/uri      (cf/get :redis-uri)
-    (cf/get :redis-uri)
+    ::mtx/metrics  (ig/ref ::mtx/metrics)
-
+    ::wrk/executor (ig/ref ::wrk/executor)}
-    ::wrk/netty-executor
-    (ig/ref ::wrk/netty-executor)
-
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
-
-   ::rds/pool
-   {::rds/client  (ig/ref ::rds/client)
-    ::mtx/metrics (ig/ref ::mtx/metrics)}
 
    ::mbus/msgbus
-   {::wrk/executor (ig/ref ::wrk/netty-executor)
+   {::wrk/executor  (ig/ref ::wrk/executor)
-    ::rds/client   (ig/ref ::rds/client)
+    ::rds/redis     (ig/ref ::rds/redis)}
-    ::mtx/metrics  (ig/ref ::mtx/metrics)}
 
    :app.storage.tmp/cleaner
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    ::sto.gc-deleted/handler
    {::db/pool      (ig/ref ::db/pool)
@@ -228,10 +231,9 @@
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
     ::http/io-threads              (cf/get :http-server-io-threads)
-    ::http/max-worker-threads      (cf/get :http-server-max-worker-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
     ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)
-    ::mtx/metrics                  (ig/ref ::mtx/metrics)}
+    ::wrk/executor                 (ig/ref ::wrk/executor)}
 
    ::ldap/provider
    {:host           (cf/get :ldap-host)
@@ -259,33 +261,26 @@
    ::oidc.providers/generic
    {::http.client/client (ig/ref ::http.client/client)}
 
-   ::oidc/providers
-   [(ig/ref ::oidc.providers/google)
-    (ig/ref ::oidc.providers/github)
-    (ig/ref ::oidc.providers/gitlab)
-    (ig/ref ::oidc.providers/generic)]
-
    ::oidc/routes
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
     ::setup/props        (ig/ref ::setup/props)
-    ::oidc/providers     (ig/ref ::oidc/providers)
+    ::oidc/providers     {:google (ig/ref ::oidc.providers/google)
+                          :github (ig/ref ::oidc.providers/github)
+                          :gitlab (ig/ref ::oidc.providers/gitlab)
+                          :oidc   (ig/ref ::oidc.providers/generic)}
     ::session/manager    (ig/ref ::session/manager)
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   ::mgmt/routes
-   {::db/pool            (ig/ref ::db/pool)
-    ::setup/props        (ig/ref ::setup/props)}
-
    :app.http/router
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
     ::rpc/routes         (ig/ref ::rpc/routes)
+    ::rpc.doc/routes     (ig/ref ::rpc.doc/routes)
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
-    ::mgmt/routes        (ig/ref ::mgmt/routes)
     ::http.debug/routes  (ig/ref ::http.debug/routes)
     ::http.assets/routes (ig/ref ::http.assets/routes)
     ::http.ws/routes     (ig/ref ::http.ws/routes)
@@ -301,7 +296,6 @@
    {::db/pool         (ig/ref ::db/pool)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::mbus/msgbus     (ig/ref ::mbus/msgbus)
-    ::setup/props     (ig/ref ::setup/props)
     ::session/manager (ig/ref ::session/manager)}
 
    :app.http.assets/routes
@@ -312,24 +306,23 @@
 
    ::rpc/climit
    {::mtx/metrics        (ig/ref ::mtx/metrics)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::wrk/executor       (ig/ref ::wrk/executor)
     ::climit/config      (cf/get :rpc-climit-config)
     ::climit/enabled     (contains? cf/flags :rpc-climit)}
 
    :app.rpc/rlimit
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.rpc/methods
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
-    ::rds/pool           (ig/ref ::rds/pool)
+    ::wrk/executor       (ig/ref ::wrk/executor)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
     ::session/manager    (ig/ref ::session/manager)
     ::ldap/provider      (ig/ref ::ldap/provider)
     ::sto/storage        (ig/ref ::sto/storage)
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
-    ::rds/client         (ig/ref ::rds/client)
+    ::rds/redis          (ig/ref ::rds/redis)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
@@ -339,26 +332,14 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   :app.rpc/management-methods
+   :app.rpc.doc/routes
-   {::http.client/client (ig/ref ::http.client/client)
+   {:app.rpc/methods (ig/ref :app.rpc/methods)}
-    ::db/pool            (ig/ref ::db/pool)
-    ::rds/pool           (ig/ref ::rds/pool)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
-    ::session/manager    (ig/ref ::session/manager)
-    ::sto/storage        (ig/ref ::sto/storage)
-    ::mtx/metrics        (ig/ref ::mtx/metrics)
-    ::mbus/msgbus        (ig/ref ::mbus/msgbus)
-    ::rds/client         (ig/ref ::rds/client)
-    ::setup/props        (ig/ref ::setup/props)}
 
-   ::rpc/routes
+   :app.rpc/routes
-   {::rpc/methods        (ig/ref :app.rpc/methods)
+   {::rpc/methods     (ig/ref :app.rpc/methods)
-    ::rpc/management-methods (ig/ref :app.rpc/management-methods)
+    ::db/pool         (ig/ref ::db/pool)
-
+    ::session/manager (ig/ref ::session/manager)
-    ;; FIXME: revisit if db/pool is necessary here
+    ::setup/props     (ig/ref ::setup/props)}
-    ::db/pool                (ig/ref ::db/pool)
-    ::session/manager        (ig/ref ::session/manager)
-    ::setup/props            (ig/ref ::setup/props)}
 
    ::wrk/registry
    {::mtx/metrics (ig/ref ::mtx/metrics)
@@ -446,9 +427,6 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
-   ::setup/clock
-   {}
-
    :app.loggers.audit.archive-task/handler
    {::setup/props        (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
@@ -492,14 +470,13 @@
                             (cf/get :objects-storage-s3-bucket))
     ::sto.s3/io-threads (or (cf/get :storage-assets-s3-io-threads)
                             (cf/get :objects-storage-s3-io-threads))
-
+    ::wrk/executor      (ig/ref ::wrk/executor)}
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
 
    :app.storage.fs/backend
    {::sto.fs/directory (or (cf/get :storage-assets-fs-directory)
                            (cf/get :objects-storage-fs-directory))}})
 
+
 (def worker-config
   {::wrk/cron
    {::wrk/registry            (ig/ref ::wrk/registry)
@@ -535,7 +512,7 @@
         :task :audit-log-gc})]}
 
    ::wrk/dispatcher
-   {::rds/client  (ig/ref ::rds/client)
+   {::rds/redis   (ig/ref ::rds/redis)
     ::mtx/metrics (ig/ref ::mtx/metrics)
     ::db/pool     (ig/ref ::db/pool)
     ::wrk/tenant  (cf/get :tenant)}
@@ -544,7 +521,7 @@
    {::wrk/parallelism (cf/get ::worker-default-parallelism 1)
     ::wrk/queue       :default
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/client      (ig/ref ::rds/client)
+    ::rds/redis       (ig/ref ::rds/redis)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}
@@ -553,7 +530,7 @@
    {::wrk/parallelism (cf/get ::worker-webhook-parallelism 1)
     ::wrk/queue       :webhooks
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/client      (ig/ref ::rds/client)
+    ::rds/redis       (ig/ref ::rds/redis)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}})

backend/src/app/media.clj

@@ -17,7 +17,6 @@
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
-   [app.http.client :as http]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [buddy.core.bytes :as bb]
@@ -38,17 +37,16 @@
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(def default-max-file-size
-  (* 1024 1024 10)) ; 10 MiB
-
 (def schema:upload
-  [:map {:title "Upload"}
+  (sm/register!
-   [:filename :string]
+   ^{::sm/type ::upload}
-   [:size ::sm/int]
+   [:map {:title "Upload"}
-   [:path ::fs/path]
+    [:filename :string]
-   [:mtype {:optional true} :string]
+    [:size ::sm/int]
-   [:headers {:optional true}
+    [:path ::fs/path]
-    [:map-of :string :string]]])
+    [:mtype {:optional true} :string]
+    [:headers {:optional true}
+     [:map-of :string :string]]]))
 
 (def ^:private schema:input
   [:map {:title "Input"}
@@ -120,7 +118,7 @@
 (defn- parse-svg
   [text]
   (let [text (strip-doctype text)]
-    (dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
+    (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
       (xml/parse istream secure-parser-factory))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -245,7 +243,7 @@
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (merge input info {:ts (ct/now) :size (fs/size path)}))
+        (merge input info {:ts (ct/now)}))
 
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
@@ -265,7 +263,6 @@
           (assoc input
                  :width  width
                  :height height
-                 :size (fs/size path)
                  :ts (ct/now)))))))
 
 (defmethod process-error org.im4java.core.InfoException
@@ -275,54 +272,6 @@
             :hint "invalid image"
             :cause error))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; IMAGE HELPERS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn download-image
-  "Download an image from the provided URI and return the media input object"
-  [{:keys [::http/client]} uri]
-  (letfn [(parse-and-validate [{:keys [headers] :as response}]
-            (let [size     (some-> (get headers "content-length") d/parse-integer)
-                  mtype    (get headers "content-type")
-                  format   (cm/mtype->format mtype)
-                  max-size (cf/get :media-max-file-size default-max-file-size)]
-
-              (when-not size
-                (ex/raise :type :validation
-                          :code :unknown-size
-                          :hint "seems like the url points to resource with unknown size"))
-
-              (when (> size max-size)
-                (ex/raise :type :validation
-                          :code :file-too-large
-                          :hint (str/ffmt "the file size % is greater than the maximum %"
-                                          size
-                                          default-max-file-size)))
-
-              (when (nil? format)
-                (ex/raise :type :validation
-                          :code :media-type-not-allowed
-                          :hint "seems like the url points to an invalid media object"))
-
-              {:size size :mtype mtype :format format}))]
-
-    (let [{:keys [body] :as response} (http/req! client
-                                                 {:method :get :uri uri}
-                                                 {:response-type :input-stream})
-          {:keys [size mtype]} (parse-and-validate response)
-          path    (tmp/tempfile :prefix "penpot.media.download.")
-          written (io/write* path body :size size)]
-
-      (when (not= written size)
-        (ex/raise :type :internal
-                  :code :mismatch-write-size
-                  :hint "unexpected state: unable to write to file"))
-
-      {;; :size size
-       :path path
-       :mtype mtype})))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FONTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/migrations.clj

@@ -444,19 +444,7 @@
     :fn (mg/resource "app/migrations/sql/0140-mod-file-change-table.sql")}
 
    {:name "0140-add-locked-by-column-to-file-change-table"
-    :fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}
+    :fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}])
-
-   {:name "0141-add-idx-to-file-library-rel"
-    :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
-
-   {:name "0141-add-file-data-table.sql"
-    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}
-
-   {:name "0142-add-sso-provider-table"
-    :fn (mg/resource "app/migrations/sql/0142-add-sso-provider-table.sql")}
-
-   {:name "0143-http-session-v2-table"
-    :fn (mg/resource "app/migrations/sql/0143-add-http-session-v2-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/media_refs.clj

@@ -10,8 +10,8 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint]
+   [app.srepl.fixes.media-refs :refer [process-file]]
    [app.srepl.main :as srepl]
-   [app.srepl.procs.media-refs]
    [clojure.edn :as edn]))
 
 (def ^:private required-services
@@ -20,10 +20,7 @@
    :app.storage/storage
    :app.metrics/metrics
    :app.db/pool
-   :app.worker/netty-io-executor])
+   :app.worker/executor])
-
-(def default-options
-  {:rollback? false})
 
 (defn -main
   [& [options]]
@@ -31,20 +28,22 @@
     (let [config-var (requiring-resolve 'app.main/system-config)
           start-var  (requiring-resolve 'app.main/start-custom)
           stop-var   (requiring-resolve 'app.main/stop)
-
+          config     (select-keys @config-var required-services)]
-          config     (select-keys @config-var required-services)
-          options    (if (string? options)
-                       (ex/ignoring (edn/read-string options))
-                       {})
-          options    (-> (merge default-options options)
-                         (assoc :proc-fn #'app.srepl.procs.media-refs/fix-media-refs))]
 
       (start-var config)
-      (l/inf :hint "executing media-refs migration" :options options)
+
-      (srepl/process! options)
+      (let [options (if (string? options)
+                      (ex/ignoring (edn/read-string options))
+                      {})]
+
+        (l/inf :hint "executing media-refs migration" :options options)
+        (srepl/process-files! process-file options))
+
       (stop-var)
       (System/exit 0))
     (catch Throwable cause
       (ex/print-throwable cause)
       (flush)
       (System/exit -1))))
+
+

backend/src/app/migrations/sql/0141-add-file-data-table.sql

@@ -1,38 +0,0 @@
-CREATE TABLE file_data (
-  file_id uuid NOT NULL REFERENCES file(id) DEFERRABLE,
-  id uuid NOT NULL,
-
-  created_at timestamptz NOT NULL DEFAULT now(),
-  modified_at timestamptz NOT NULL DEFAULT now(),
-  deleted_at timestamptz NULL,
-
-  type text NOT NULL,
-  backend text NULL,
-
-  metadata jsonb NULL,
-  data bytea NULL,
-
-  PRIMARY KEY (file_id, id)
-
-) PARTITION BY HASH (file_id);
-
-CREATE TABLE file_data_00 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 0);
-CREATE TABLE file_data_01 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 1);
-CREATE TABLE file_data_02 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 2);
-CREATE TABLE file_data_03 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 3);
-CREATE TABLE file_data_04 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 4);
-CREATE TABLE file_data_05 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 5);
-CREATE TABLE file_data_06 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 6);
-CREATE TABLE file_data_07 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 7);
-CREATE TABLE file_data_08 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 8);
-CREATE TABLE file_data_09 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 9);
-CREATE TABLE file_data_10 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 10);
-CREATE TABLE file_data_11 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 11);
-CREATE TABLE file_data_12 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 12);
-CREATE TABLE file_data_13 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 13);
-CREATE TABLE file_data_14 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 14);
-CREATE TABLE file_data_15 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 15);
-
-CREATE INDEX file_data__deleted_at__idx
-    ON file_data (deleted_at, file_id, id)
- WHERE deleted_at IS NOT NULL;

backend/src/app/migrations/sql/0141-add-idx-to-file-library-rel.sql

@@ -1,2 +0,0 @@
-CREATE INDEX IF NOT EXISTS file_library_rel__library_file_id__idx
-    ON file_library_rel (library_file_id);

backend/src/app/migrations/sql/0142-add-sso-provider-table.sql

@@ -1,33 +0,0 @@
-CREATE TABLE sso_provider (
-  id uuid PRIMARY KEY,
-
-  created_at timestamptz NOT NULL DEFAULT now(),
-  modified_at timestamptz NOT NULL DEFAULT now(),
-
-  is_enabled boolean NOT NULL DEFAULT true,
-
-  type text NOT NULL CHECK (type IN ('oidc')),
-  domain text NOT NULL,
-
-  client_id text NOT NULL,
-  client_secret text NOT NULL,
-
-  base_uri text NOT NULL,
-  token_uri text NULL,
-  auth_uri text NULL,
-  user_uri text NULL,
-  jwks_uri text NULL,
-  logout_uri text NULL,
-
-  roles_attr text NULL,
-  email_attr text NULL,
-  name_attr text NULL,
-  user_info_source text NOT NULL DEFAULT 'token'
-     CHECK (user_info_source IN ('token', 'userinfo', 'auto')),
-
-  scopes text[] NULL,
-  roles text[] NULL
-);
-
-CREATE UNIQUE INDEX sso_provider__domain__idx
-    ON sso_provider(domain);

backend/src/app/migrations/sql/0143-add-http-session-v2-table.sql

@@ -1,23 +0,0 @@
-CREATE TABLE http_session_v2 (
-  id uuid PRIMARY KEY,
-
-  created_at timestamptz NOT NULL DEFAULT now(),
-  modified_at timestamptz NOT NULL DEFAULT now(),
-
-  profile_id uuid REFERENCES profile(id) ON DELETE CASCADE,
-  user_agent text NULL,
-
-  sso_provider_id uuid NULL REFERENCES sso_provider(id) ON DELETE CASCADE,
-  sso_session_id text NULL
-);
-
-CREATE INDEX http_session_v2__profile_id__idx
-    ON http_session_v2(profile_id);
-
-CREATE INDEX http_session_v2__sso_provider_id__idx
-    ON http_session_v2(sso_provider_id)
- WHERE sso_provider_id IS NOT NULL;
-
-CREATE INDEX http_session_v2__sso_session_id__idx
-    ON http_session_v2(sso_session_id)
- WHERE sso_session_id IS NOT NULL;

backend/src/app/msgbus.clj

@@ -16,6 +16,7 @@
    [app.redis :as rds]
    [app.worker :as wrk]
    [integrant.core :as ig]
+   [promesa.core :as p]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]))
 
@@ -58,16 +59,14 @@
          (assoc ::timeout (ct/duration {:seconds 30})))})
 
 (def ^:private schema:params
-  [:map
+  [:map ::rds/redis ::wrk/executor])
-   ::rds/client
-   ::wrk/executor])
 
 (defmethod ig/assert-key ::msgbus
   [_ params]
   (assert (sm/check schema:params params)))
 
 (defmethod ig/init-key ::msgbus
-  [_ {:keys [::buffer-size ::wrk/executor ::timeout] :as cfg}]
+  [_ {:keys [::buffer-size ::wrk/executor ::timeout ::rds/redis] :as cfg}]
   (l/info :hint "initialize msgbus" :buffer-size buffer-size)
   (let [cmd-ch (sp/chan :buf buffer-size)
         rcv-ch (sp/chan :buf (sp/dropping-buffer buffer-size))
@@ -75,9 +74,8 @@
                         :xf  xform-prefix-topic)
         state  (agent {})
 
-        ;; Open persistent connections to redis
+        pconn  (rds/connect redis :type :default :timeout timeout)
-        pconn  (rds/connect cfg :timeout timeout)
+        sconn  (rds/connect redis :type :pubsub :timeout timeout)
-        sconn  (rds/connect-pubsub cfg :timeout timeout)
 
         _      (set-error-handler! state #(l/error :cause % :hint "unexpected error on agent" ::l/sync? true))
         _      (set-error-mode! state :continue)
@@ -191,13 +189,14 @@
 
 (defn- create-listener
   [rcv-ch]
-  {:on-message (fn [_ topic message]
+  (rds/pubsub-listener
+   :on-message (fn [_ topic message]
                  ;; There are no back pressure, so we use a slidding
                  ;; buffer for cases when the pubsub broker sends
                  ;; more messages that we can process.
-                 (let [val {:topic topic :message (t/decode-str message)}]
+                 (let [val {:topic topic :message (t/decode message)}]
                    (when-not (sp/offer! rcv-ch val)
-                     (l/warn :msg "dropping message on subscription loop"))))})
+                     (l/warn :msg "dropping message on subscription loop"))))))
 
 (defn- process-input
   [{:keys [::state ::wrk/executor] :as cfg} topic message]
@@ -217,7 +216,8 @@
   (rds/add-listener sconn (create-listener rcv-ch))
 
   (px/thread
-    {:name "penpot/msgbus"}
+    {:name "penpot/msgbus/io-loop"
+     :virtual true}
     (try
       (loop []
         (let [timeout-ch (sp/timeout-chan 1000)
@@ -263,7 +263,7 @@
   intended to be used in core.async go blocks."
   [{:keys [::pconn] :as cfg} {:keys [topic message]}]
   (try
-    (rds/publish pconn topic (t/encode-str message))
+    (p/await! (rds/publish pconn topic (t/encode message)))
     (catch InterruptedException cause
       (throw cause))
     (catch Throwable cause

backend/src/app/redis.clj

@@ -6,22 +6,23 @@
 
 (ns app.redis
   "The msgbus abstraction implemented using redis as underlying backend."
-  (:refer-clojure :exclude [eval get set run!])
+  (:refer-clojure :exclude [eval])
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.generic-pool :as gpool]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.metrics :as mtx]
    [app.redis.script :as-alias rscript]
-   [app.worker :as wrk]
+   [app.util.cache :as cache]
-   [app.worker.executor]
+   [app.worker :as-alias wrk]
    [clojure.core :as c]
    [clojure.java.io :as io]
    [cuerdas.core :as str]
-   [integrant.core :as ig])
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px])
   (:import
    clojure.lang.MapEntry
    io.lettuce.core.KeyValue
@@ -31,10 +32,12 @@
    io.lettuce.core.RedisException
    io.lettuce.core.RedisURI
    io.lettuce.core.ScriptOutputType
-   io.lettuce.core.SetArgs
+   io.lettuce.core.api.StatefulConnection
    io.lettuce.core.api.StatefulRedisConnection
+   io.lettuce.core.api.async.RedisAsyncCommands
+   io.lettuce.core.api.async.RedisScriptingAsyncCommands
    io.lettuce.core.api.sync.RedisCommands
-   io.lettuce.core.api.sync.RedisScriptingCommands
+   io.lettuce.core.codec.ByteArrayCodec
    io.lettuce.core.codec.RedisCodec
    io.lettuce.core.codec.StringCodec
    io.lettuce.core.pubsub.RedisPubSubListener
@@ -42,238 +45,244 @@
    io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
    io.lettuce.core.resource.ClientResources
    io.lettuce.core.resource.DefaultClientResources
-   io.netty.channel.nio.NioEventLoopGroup
    io.netty.util.HashedWheelTimer
    io.netty.util.Timer
-   io.netty.util.concurrent.EventExecutorGroup
    java.lang.AutoCloseable
    java.time.Duration))
 
 (set! *warn-on-reflection* true)
 
-(def ^:const MAX-EVAL-RETRIES 18)
+(declare ^:private initialize-resources)
+(declare ^:private shutdown-resources)
+(declare ^:private impl-eval)
 
-(def default-timeout
+(defprotocol IRedis
-  (ct/duration "10s"))
+  (-connect [_ options])
-
+  (-get-or-connect [_ key options]))
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; IMPL & PRIVATE API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol IConnection
-  (-set-timeout [_ timeout] "set connection timeout")
+  (publish [_ topic message])
-  (-get-timeout [_] "get current timeout")
+  (rpush [_ key payload])
-  (-reset-timeout [_] "reset to default timeout"))
+  (blpop [_ timeout keys])
-
+  (eval [_ script]))
-(defprotocol IDefaultConnection
-  "Public API of default redis connection"
-  (-publish [_ topic message])
-  (-rpush [_ key payload])
-  (-blpop [_ timeout keys])
-  (-eval [_ script])
-  (-get [_ key])
-  (-set [_ key val args])
-  (-del [_ key-or-keys])
-  (-ping [_]))
 
 (defprotocol IPubSubConnection
-  (-add-listener [_ listener])
+  (add-listener [_ listener])
-  (-subscribe [_ topics])
+  (subscribe [_ topics])
-  (-unsubscribe [_ topics]))
+  (unsubscribe [_ topics]))
 
-(def ^:private default-codec
+(def default-codec
+  (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE))
+
+(def string-codec
   (RedisCodec/of StringCodec/UTF8 StringCodec/UTF8))
 
-(defn- impl-eval
+(sm/register!
-  [cmd cache metrics script]
+ {:type ::connection
-  (let [keys    (into-array String (map str (::rscript/keys script)))
+  :pred #(satisfies? IConnection %)
-        vals    (into-array String (map str (::rscript/vals script)))
+  :type-properties
-        sname   (::rscript/name script)
+  {:title "connection"
+   :description "redis connection instance"}})
 
-        read-script
+(sm/register!
-        (fn []
+ {:type ::pubsub-connection
-          (-> script ::rscript/path io/resource slurp))
+  :pred #(satisfies? IPubSubConnection %)
+  :type-properties
+  {:title "connection"
+   :description "redis connection instance"}})
 
-        load-script
+(defn redis?
-        (fn []
+  [o]
-          (let [id (.scriptLoad ^RedisScriptingCommands cmd
+  (satisfies? IRedis o))
-                                ^String (read-script))]
-            (swap! cache assoc sname id)
-            (l/trc :hint "load script" :name sname :id id)
 
-            id))
+(sm/register!
+ {:type ::redis
+  :pred redis?})
 
-        eval-script
+(def ^:private schema:script
-        (fn [id]
+  [:map {:title "script"}
-          (try
+   [::rscript/name qualified-keyword?]
-            (let [tpoint  (ct/tpoint)
+   [::rscript/path ::sm/text]
-                  result  (.evalsha ^RedisScriptingCommands cmd
+   [::rscript/keys {:optional true} [:vector :any]]
-                                    ^String id
+   [::rscript/vals {:optional true} [:vector :any]]])
-                                    ^ScriptOutputType ScriptOutputType/MULTI
-                                    ^"[Ljava.lang.String;" keys
-                                    ^"[Ljava.lang.String;" vals)
-                  elapsed (tpoint)]
 
-              (mtx/run! metrics {:id :redis-eval-timing
+(def valid-script?
-                                 :labels [(name sname)]
+  (sm/lazy-validator schema:script))
-                                 :val (inst-ms elapsed)})
 
-              (l/trc :hint "eval script"
+(defmethod ig/expand-key ::redis
-                     :name (name sname)
+  [k v]
-                     :id id
+  (let [cpus    (px/get-available-processors)
-                     :params (str/join "," (::rscript/vals script))
+        threads (max 1 (int (* cpus 0.2)))]
-                     :elapsed (ct/format-duration elapsed))
+    {k (-> (d/without-nils v)
+           (assoc ::timeout (ct/duration "10s"))
+           (assoc ::io-threads (max 3 threads))
+           (assoc ::worker-threads (max 3 threads)))}))
 
-              result)
+(def ^:private schema:redis-params
+  [:map {:title "redis-params"}
+   ::wrk/executor
+   ::mtx/metrics
+   [::uri ::sm/uri]
+   [::worker-threads ::sm/int]
+   [::io-threads ::sm/int]
+   [::timeout ::ct/duration]])
 
-            (catch io.lettuce.core.RedisNoScriptException _cause
+(defmethod ig/assert-key ::redis
-              ::load)
+  [_ params]
+  (assert (sm/check schema:redis-params params)))
 
-            (catch Throwable cause
+(defmethod ig/init-key ::redis
-              (when-let [on-error (::rscript/on-error script)]
+  [_ params]
-                (on-error cause))
+  (initialize-resources params))
-              (throw cause))))
 
-        eval-script'
+(defmethod ig/halt-key! ::redis
-        (fn [id]
+  [_ instance]
-          (loop [id      id
+  (d/close! instance))
-                 retries 0]
-            (if (> retries MAX-EVAL-RETRIES)
-              (ex/raise :type :internal
-                        :code ::max-eval-retries-reached
-                        :hint (str "unable to eval redis script " sname))
-              (let [result (eval-script id)]
-                (if (= result ::load)
-                  (recur (load-script)
-                         (inc retries))
-                  result)))))]
 
-    (if-let [id (c/get @cache sname)]
+(defn- initialize-resources
-      (eval-script' id)
+  "Initialize redis connection resources"
-      (-> (load-script)
+  [{:keys [::uri ::io-threads ::worker-threads ::wrk/executor ::mtx/metrics] :as params}]
-          (eval-script')))))
 
-(deftype Connection [^StatefulRedisConnection conn
+  (l/inf :hint "initialize redis resources"
-                     ^RedisCommands cmd
+         :uri (str uri)
-                     ^Duration timeout
+         :io-threads io-threads
-                     cache metrics]
+         :worker-threads worker-threads)
-  AutoCloseable
-  (close [_]
-    (ex/ignoring (.close conn)))
 
-  IConnection
+  (let [timer     (HashedWheelTimer.)
-  (-set-timeout [_ timeout]
+        resources (.. (DefaultClientResources/builder)
-    (.setTimeout conn ^Duration timeout))
+                      (ioThreadPoolSize ^long io-threads)
+                      (computationThreadPoolSize ^long worker-threads)
+                      (timer ^Timer timer)
+                      (build))
 
-  (-reset-timeout [_]
+        redis-uri (RedisURI/create ^String (str uri))
-    (.setTimeout conn timeout))
 
-  (-get-timeout [_]
+        shutdown  (fn [client conn]
-    (.getTimeout conn))
+                    (ex/ignoring (.close ^StatefulConnection conn))
+                    (ex/ignoring (.close ^RedisClient client))
+                    (l/trc :hint "disconnect" :hid (hash client)))
 
-  IDefaultConnection
+        on-remove (fn [key val cause]
-  (-publish [_ topic message]
+                    (l/trace :hint "evict connection (cache)" :key key :reason cause)
-    (.publish cmd ^String topic ^String message))
+                    (some-> val d/close!))
 
-  (-rpush [_ key elements]
+        cache     (cache/create :executor executor
-    (try
+                                :on-remove on-remove
-      (let [vals (make-array String (count elements))]
+                                :keepalive "5m")]
-        (loop [i 0 xs (seq elements)]
+    (reify
-          (when xs
+      java.lang.AutoCloseable
-            (aset ^"[[Ljava.lang.String;" vals i ^String (first xs))
+      (close [_]
-            (recur (inc i) (next xs))))
+        (ex/ignoring (cache/invalidate! cache))
+        (ex/ignoring (.shutdown ^ClientResources resources))
+        (ex/ignoring (.stop ^Timer timer)))
 
-        (.rpush cmd
+      IRedis
-                ^String key
+      (-get-or-connect [this key options]
-                ^"[[Ljava.lang.String;" vals))
+        (let [create (fn [_] (-connect this options))]
+          (cache/get cache key create)))
 
-      (catch RedisCommandInterruptedException cause
+      (-connect [_ options]
-        (throw (InterruptedException. (ex-message cause))))))
+        (let [timeout (or (:timeout options) (::timeout params))
+              codec   (get options :codec default-codec)
+              type    (get options :type :default)
+              client  (RedisClient/create ^ClientResources resources
+                                          ^RedisURI redis-uri)]
 
-  (-blpop [_ keys timeout]
+          (l/trc :hint "connect" :hid (hash client))
-    (try
+          (if (= type :pubsub)
-      (let [keys (into-array String keys)]
+            (let [conn (.connectPubSub ^RedisClient client
-        (when-let [res (.blpop cmd
+                                       ^RedisCodec codec)]
-                               ^double timeout
+              (.setTimeout ^StatefulConnection conn
-                               ^"[Ljava.lang.String;" keys)]
+                           ^Duration timeout)
-          (MapEntry/create
+              (reify
-           (.getKey ^KeyValue res)
+                IPubSubConnection
-           (.getValue ^KeyValue res))))
+                (add-listener [_ listener]
-      (catch RedisCommandInterruptedException cause
+                  (assert (instance? RedisPubSubListener listener) "expected listener instance")
-        (throw (InterruptedException. (ex-message cause))))))
+                  (.addListener ^StatefulRedisPubSubConnection conn
+                                ^RedisPubSubListener listener))
 
-  (-get [_ key]
+                (subscribe [_ topics]
-    (assert (string? key) "key expected to be string")
+                  (try
-    (.get cmd ^String key))
+                    (let [topics (into-array String (map str topics))
+                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
+                      (.subscribe ^RedisPubSubCommands cmd topics))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
-  (-set [_ key val args]
+                (unsubscribe [_ topics]
-    (.set cmd
+                  (try
-          ^String key
+                    (let [topics (into-array String (map str topics))
-          ^bytes val
+                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
-          ^SetArgs args))
+                      (.unsubscribe ^RedisPubSubCommands cmd topics))
-
+                    (catch RedisCommandInterruptedException cause
-  (-del [_ keys]
+                      (throw (InterruptedException. (ex-message cause))))))
-    (let [keys (into-array String keys)]
-      (.del cmd ^String/1 keys)))
-
-  (-ping [_]
-    (.ping cmd))
-
-  (-eval [_ script]
-    (impl-eval cmd cache metrics script)))
 
 
-(deftype SubscriptionConnection [^StatefulRedisPubSubConnection conn
+                AutoCloseable
-                                 ^RedisPubSubCommands cmd
+                (close [_] (shutdown client conn))))
-                                 ^Duration timeout]
-  AutoCloseable
-  (close [_]
-    (ex/ignoring (.close conn)))
 
-  IConnection
+            (let [conn (.connect ^RedisClient client ^RedisCodec codec)]
-  (-set-timeout [_ timeout]
+              (.setTimeout ^StatefulConnection conn ^Duration timeout)
-    (.setTimeout conn ^Duration timeout))
+              (reify
+                IConnection
+                (publish [_ topic message]
+                  (assert (string? topic) "expected topic to be string")
+                  (assert (bytes? message) "expected message to be a byte array")
 
-  (-reset-timeout [_]
+                  (let [pcomm (.async ^StatefulRedisConnection conn)]
-    (.setTimeout conn timeout))
+                    (.publish ^RedisAsyncCommands pcomm ^String topic ^bytes message)))
 
-  (-get-timeout [_]
+                (rpush [_ key payload]
-    (.getTimeout conn))
+                  (assert (or (and (vector? payload)
+                                   (every? bytes? payload))
+                              (bytes? payload)))
+                  (try
+                    (let [cmd  (.sync ^StatefulRedisConnection conn)
+                          data (if (vector? payload) payload [payload])
+                          vals (make-array (. Class (forName "[B")) (count data))]
 
-  IPubSubConnection
+                      (loop [i 0 xs (seq data)]
-  (-add-listener [_ listener]
+                        (when xs
-    (.addListener conn ^RedisPubSubListener listener))
+                          (aset ^"[[B" vals i ^bytes (first xs))
+                          (recur (inc i) (next xs))))
 
-  (-subscribe [_ topics]
+                      (.rpush ^RedisCommands cmd
-    (try
+                              ^String key
-      (let [topics (into-array String topics)]
+                              ^"[[B" vals))
-        (.subscribe cmd topics))
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause))))))
 
-  (-unsubscribe [_ topics]
+                    (catch RedisCommandInterruptedException cause
-    (try
+                      (throw (InterruptedException. (ex-message cause))))))
-      (let [topics (into-array String topics)]
-        (.unsubscribe cmd topics))
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause)))))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+                (blpop [_ timeout keys]
-;; PUBLIC API
+                  (try
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+                    (let [keys    (into-array Object (map str keys))
+                          cmd     (.sync ^StatefulRedisConnection conn)
+                          timeout (/ (double (inst-ms timeout)) 1000.0)]
+                      (when-let [res (.blpop ^RedisCommands cmd
+                                             ^double timeout
+                                             ^"[Ljava.lang.String;" keys)]
+                        (MapEntry/create
+                         (.getKey ^KeyValue res)
+                         (.getValue ^KeyValue res))))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
-(defn build-set-args
+                (eval [_ script]
-  [options]
+                  (assert (valid-script? script) "expected valid script")
-  (reduce-kv (fn [^SetArgs args k v]
+                  (impl-eval conn metrics script))
-               (case k
+
-                 :ex (if (instance? Duration v)
+                AutoCloseable
-                       (.ex args ^Duration v)
+                (close [_] (shutdown client conn))))))))))
-                       (.ex args (long v)))
+
-                 :px (.px args (long v))
+(defn connect
-                 :nx (if v (.nx args) args)
+  [instance & {:as opts}]
-                 :keep-ttl (if v (.keepttl args) args)))
+  (assert (satisfies? IRedis instance) "expected valid redis instance")
-             (SetArgs.)
+  (-connect instance opts))
-             options))
+
+(defn get-or-connect
+  [instance key & {:as opts}]
+  (assert (satisfies? IRedis instance) "expected valid redis instance")
+  (-get-or-connect instance key opts))
 
 (defn pubsub-listener
   [& {:keys [on-message on-subscribe on-unsubscribe]}]
@@ -302,172 +311,61 @@
       (when on-unsubscribe
         (on-unsubscribe nil topic count)))))
 
-(defn connect
+(def ^:private scripts-cache (atom {}))
-  [cfg & {:as options}]
-  (assert (contains? cfg ::mtx/metrics) "missing ::mtx/metrics on provided system")
-  (assert (contains? cfg ::client) "missing ::rds/client on provided system")
 
-  (let [state   (::client cfg)
+(defn- impl-eval
+  [^StatefulRedisConnection connection metrics script]
+  (let [cmd   (.async ^StatefulRedisConnection connection)
+        keys  (into-array String (map str (::rscript/keys script)))
+        vals  (into-array String (map str (::rscript/vals script)))
+        sname (::rscript/name script)]
 
-        cache   (::cache state)
+    (letfn [(on-error [cause]
-        client  (::client state)
+              (if (instance? io.lettuce.core.RedisNoScriptException cause)
-        timeout (or (some-> (:timeout options) ct/duration)
+                (do
-                    (::timeout state))
+                  (l/error :hint "no script found" :name sname :cause cause)
+                  (->> (load-script)
+                       (p/mcat eval-script)))
+                (if-let [on-error (::rscript/on-error script)]
+                  (on-error cause)
+                  (p/rejected cause))))
 
-        conn    (.connect ^RedisClient client
+            (eval-script [sha]
-                          ^RedisCodec default-codec)
+              (let [tpoint (ct/tpoint)]
-        cmd     (.sync ^StatefulRedisConnection conn)]
+                (->> (.evalsha ^RedisScriptingAsyncCommands cmd
+                               ^String sha
+                               ^ScriptOutputType ScriptOutputType/MULTI
+                               ^"[Ljava.lang.String;" keys
+                               ^"[Ljava.lang.String;" vals)
+                     (p/fmap (fn [result]
+                               (let [elapsed (tpoint)]
+                                 (mtx/run! metrics {:id :redis-eval-timing
+                                                    :labels [(name sname)]
+                                                    :val (inst-ms elapsed)})
+                                 (l/trace :hint "eval script"
+                                          :name (name sname)
+                                          :sha sha
+                                          :params (str/join "," (::rscript/vals script))
+                                          :elapsed (ct/format-duration elapsed))
+                                 result)))
+                     (p/merr on-error))))
 
-    (.setTimeout ^StatefulRedisConnection conn ^Duration timeout)
+            (read-script []
-    (->Connection conn cmd timeout cache (::mtx/metrics cfg))))
+              (-> script ::rscript/path io/resource slurp))
 
-(defn connect-pubsub
+            (load-script []
-  [cfg & {:as options}]
+              (l/trace :hint "load script" :name sname)
-  (let [state   (::client cfg)
+              (->> (.scriptLoad ^RedisScriptingAsyncCommands cmd
-        client  (::client state)
+                                ^String (read-script))
+                   (p/fmap (fn [sha]
+                             (swap! scripts-cache assoc sname sha)
+                             sha))))]
 
-        timeout (or (some-> (:timeout options) ct/duration)
+      (p/await!
-                    (::timeout state))
+       (if-let [sha (get @scripts-cache sname)]
-        conn    (.connectPubSub ^RedisClient client
+         (eval-script sha)
-                                ^RedisCodec default-codec)
+         (->> (load-script)
-        cmd     (.sync ^StatefulRedisPubSubConnection conn)]
+              (p/mapcat eval-script)))))))
-
-
-    (.setTimeout ^StatefulRedisPubSubConnection conn
-                 ^Duration timeout)
-    (->SubscriptionConnection conn cmd timeout)))
-
-(defn get
-  [conn key]
-  (assert (string? key) "key must be string instance")
-  (try
-    (-get conn key)
-    (catch RedisCommandTimeoutException cause
-      (l/err :hint "timeout on get redis key" :key key :cause cause)
-      nil)))
-
-(defn set
-  ([conn key val]
-   (set conn key val nil))
-  ([conn key val args]
-   (assert (string? key) "key must be string instance")
-   (assert (string? val) "val must be string instance")
-   (let [args (cond
-                (or (instance? SetArgs args)
-                    (nil? args))
-                args
-
-                (map? args)
-                (build-set-args args)
-
-                :else
-                (throw (IllegalArgumentException. "invalid args")))]
-
-     (try
-       (-set conn key val args)
-       (catch RedisCommandTimeoutException cause
-         (l/err :hint "timeout on set redis key" :key key :cause cause)
-         nil)))))
-
-(defn del
-  [conn key-or-keys]
-  (let [keys (if (vector? key-or-keys) key-or-keys [key-or-keys])]
-    (assert (every? string? keys) "only string keys allowed")
-    (try
-      (-del conn keys)
-      (catch RedisCommandTimeoutException cause
-        (l/err :hint "timeout on del redis key" :key key :cause cause)
-        nil))))
-
-(defn ping
-  [conn]
-  (-ping conn))
-
-(defn blpop
-  [conn key-or-keys timeout]
-  (let [keys    (if (vector? key-or-keys) key-or-keys [key-or-keys])
-        timeout (cond
-                  (ct/duration? timeout)
-                  (/ (double (inst-ms timeout)) 1000.0)
-
-                  (double? timeout)
-                  timeout
-
-                  (int? timeout)
-                  (/ (double timeout) 1000.0)
-
-                  :else
-                  0)]
-
-    (assert (every? string? keys) "only string keys allowed")
-    (-blpop conn keys timeout)))
-
-(defn rpush
-  [conn key elements]
-  (assert (string? key) "key must be string instance")
-  (assert (every? string? elements) "elements should be all strings")
-  (let [elements (vec elements)]
-    (-rpush conn key elements)))
-
-(defn publish
-  [conn topic payload]
-  (assert (string? topic) "expected topic to be string")
-  (assert (string? payload) "expected message to be a byte array")
-  (-publish conn topic payload))
-
-(def ^:private schema:script
-  [:map {:title "script"}
-   [::rscript/name qualified-keyword?]
-   [::rscript/path ::sm/text]
-   [::rscript/keys {:optional true} [:vector :any]]
-   [::rscript/vals {:optional true} [:vector :any]]])
-
-(def ^:private valid-script?
-  (sm/lazy-validator schema:script))
-
-(defn eval
-  [conn script]
-  (assert (valid-script? script) "expected valid script")
-  (-eval conn script))
-
-(defn add-listener
-  [conn listener]
-  (let [listener (cond
-                   (map? listener)
-                   (pubsub-listener listener)
-
-                   (instance? RedisPubSubListener listener)
-                   listener
-
-                   :else
-                   (throw (IllegalArgumentException. "invalid listener provided")))]
-
-    (-add-listener conn listener)))
-
-(defn subscribe
-  [conn topic-or-topics]
-  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
-    (assert (every? string? topics))
-    (-subscribe conn topics)))
-
-(defn unsubscribe
-  [conn topic-or-topics]
-  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
-    (assert (every? string? topics))
-    (-unsubscribe conn topics)))
-
-(defn set-timeout
-  [conn timeout]
-  (let [timeout (ct/duration timeout)]
-    (-set-timeout conn timeout)))
-
-(defn get-timeout
-  [conn]
-  (-get-timeout conn))
-
-(defn reset-timeout
-  [conn]
-  (-reset-timeout conn))
 
 (defn timeout-exception?
   [cause]
@@ -476,121 +374,3 @@
 (defn exception?
   [cause]
   (instance? RedisException cause))
-
-(defn get-pooled
-  [cfg]
-  (let [pool (::pool cfg)]
-    (gpool/get pool)))
-
-(defn close
-  [o]
-  (.close ^AutoCloseable o))
-
-(defn pool
-  [cfg & {:as options}]
-  (gpool/create :create-fn (partial connect cfg options)
-                :destroy-fn close
-                :dispose-fn -reset-timeout))
-
-(defn run!
-  [cfg f & args]
-  (if (gpool/pool? cfg)
-    (apply f {::pool cfg} f args)
-    (let [pool (::pool cfg)]
-      (with-open [^AutoCloseable conn (gpool/get pool)]
-        (apply f (assoc cfg ::conn @conn) args)))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; INITIALIZATION
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmethod ig/expand-key ::client
-  [k v]
-  {k (-> (d/without-nils v)
-         (assoc ::timeout (ct/duration "10s")))})
-
-(def ^:private schema:client
-  [:map {:title "RedisClient"}
-   [::timer [:fn #(instance? HashedWheelTimer %)]]
-   [::cache ::sm/atom]
-   [::timeout ::ct/duration]
-   [::resources [:fn #(instance? DefaultClientResources %)]]])
-
-(def check-client
-  (sm/check-fn schema:client))
-
-(sm/register! ::client schema:client)
-(sm/register!
- {:type ::pool
-  :pred gpool/pool?})
-
-(def ^:private schema:client-params
-  [:map {:title "redis-params"}
-   ::wrk/netty-io-executor
-   ::wrk/netty-executor
-   [::uri ::sm/uri]
-   [::timeout ::ct/duration]])
-
-(def ^:private check-client-params
-  (sm/check-fn schema:client-params))
-
-(defmethod ig/assert-key ::client
-  [_ params]
-  (check-client-params params))
-
-(defmethod ig/init-key ::client
-  [_ {:keys [::uri ::wrk/netty-io-executor ::wrk/netty-executor] :as params}]
-
-  (l/inf :hint "initialize redis client" :uri (str uri))
-
-  (let [timer     (HashedWheelTimer.)
-        cache     (atom {})
-
-        resources (.. (DefaultClientResources/builder)
-                      (eventExecutorGroup ^EventExecutorGroup netty-executor)
-
-                      ;; We provide lettuce with a shared event loop
-                      ;; group instance instead of letting lettuce to
-                      ;; create its own
-                      (eventLoopGroupProvider
-                       (reify io.lettuce.core.resource.EventLoopGroupProvider
-                         (allocate [_ _] netty-io-executor)
-                         (threadPoolSize [_]
-                           (.executorCount ^NioEventLoopGroup netty-io-executor))
-                         (release [_ _ _ _ _]
-                           ;; Do nothing
-                           )
-                         (shutdown [_ _ _ _]
-                           ;; Do nothing
-                           )))
-
-                      (timer ^Timer timer)
-                      (build))
-
-        redis-uri (RedisURI/create ^String (str uri))
-        client    (RedisClient/create ^ClientResources resources
-                                      ^RedisURI redis-uri)]
-
-    {::client client
-     ::cache cache
-     ::timer timer
-     ::timeout default-timeout
-     ::resources resources}))
-
-(defmethod ig/halt-key! ::client
-  [_ {:keys [::client ::timer ::resources]}]
-  (ex/ignoring (.shutdown ^RedisClient client))
-  (ex/ignoring (.shutdown ^ClientResources resources))
-  (ex/ignoring (.stop ^Timer timer)))
-
-(defmethod ig/assert-key ::pool
-  [_ {:keys [::client]}]
-  (check-client client))
-
-(defmethod ig/init-key ::pool
-  [_ cfg]
-  (pool cfg {:timeout (ct/duration 2000)}))
-
-(defmethod ig/halt-key! ::pool
-  [_ instance]
-  (.close ^java.lang.AutoCloseable instance))

backend/src/app/rpc.clj

@@ -13,23 +13,18 @@
    [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.time :as ct]
-   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
    [app.http.access-token :as actoken]
    [app.http.client :as-alias http.client]
-   [app.http.middleware :as mw]
-   [app.http.security :as sec]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.msgbus :as-alias mbus]
-   [app.redis :as rds]
    [app.rpc.climit :as climit]
    [app.rpc.cond :as cond]
-   [app.rpc.doc :as doc]
    [app.rpc.helpers :as rph]
    [app.rpc.retry :as retry]
    [app.rpc.rlimit :as rlimit]
@@ -40,6 +35,7 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [promesa.core :as p]
    [yetti.request :as yreq]
    [yetti.response :as yres]))
 
@@ -47,7 +43,7 @@
 
 (defn- default-handler
   [_]
-  (ex/raise :type :not-found))
+  (p/rejected (ex/error :type :not-found)))
 
 (defn- handle-response-transformation
   [response request mdata]
@@ -67,58 +63,67 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result  (rph/unwrap result)
+                   (let [result (rph/unwrap result)]
-                         status  (or (::http/status mdata)
+                     {::yres/status  (::http/status mdata 200)
-                                     (if (nil? result)
+                      ::yres/headers (::http/headers mdata {})
-                                       204
-                                       200))
-                         headers (cond-> (::http/headers mdata {})
-                                   (yres/stream-body? result)
-                                   (assoc "content-type" "application/octet-stream"))]
-                     {::yres/status  status
-                      ::yres/headers headers
                       ::yres/body    result}))]
-
     (-> response
         (handle-response-transformation request mdata)
         (handle-before-comple-hook mdata))))
 
-(defn- make-rpc-handler
+(defn get-external-session-id
+  [request]
+  (when-let [session-id (yreq/get-header request "x-external-session-id")]
+    (when-not (or (> (count session-id) 256)
+                  (= session-id "null")
+                  (str/blank? session-id))
+      session-id)))
+
+(defn- get-external-event-origin
+  [request]
+  (when-let [origin (yreq/get-header request "x-event-origin")]
+    (when-not (or (> (count origin) 256)
+                  (= origin "null")
+                  (str/blank? origin))
+      origin)))
+
+(defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
-  [methods]
+  [methods {:keys [params path-params method] :as request}]
-  (let [methods (update-vals methods peek)]
+  (let [handler-name (:type path-params)
-    (fn [{:keys [params path-params method] :as request}]
+        etag         (yreq/get-header request "if-none-match")
-      (let [handler-name (:type path-params)
+        profile-id   (or (::session/profile-id request)
-            etag         (yreq/get-header request "if-none-match")
+                         (::actoken/profile-id request))
-            profile-id   (or (::session/profile-id request)
-                             (::actoken/profile-id request))
-            ip-addr      (inet/parse-request request)
 
-            data         (-> params
+        ip-addr      (inet/parse-request request)
-                             (assoc ::handler-name handler-name)
+        session-id   (get-external-session-id request)
-                             (assoc ::ip-addr ip-addr)
+        event-origin (get-external-event-origin request)
-                             (assoc ::request-at (ct/now))
-                             (assoc ::cond/key etag)
-                             (cond-> (uuid? profile-id)
-                               (assoc ::profile-id profile-id)))
 
-            data         (with-meta data
+        data         (-> params
-                           {::http/request request})
+                         (assoc ::handler-name handler-name)
+                         (assoc ::ip-addr ip-addr)
+                         (assoc ::request-at (ct/now))
+                         (assoc ::external-session-id session-id)
+                         (assoc ::external-event-origin event-origin)
+                         (assoc ::session/id (::session/id request))
+                         (assoc ::cond/key etag)
+                         (cond-> (uuid? profile-id)
+                           (assoc ::profile-id profile-id)))
 
-            handler-fn   (get methods (keyword handler-name) default-handler)]
+        data         (vary-meta data assoc ::http/request request)
+        handler-fn   (get methods (keyword handler-name) default-handler)]
 
-        (when (and (or (= method :get)
+    (when (and (or (= method :get)
-                       (= method :head))
+                   (= method :head))
-                   (not (str/starts-with? handler-name "get-")))
+               (not (str/starts-with? handler-name "get-")))
-          (ex/raise :type :restriction
+      (ex/raise :type :restriction
-                    :code :method-not-allowed
+                :code :method-not-allowed
-                    :hint "method not allowed for this request"))
+                :hint "method not allowed for this request"))
 
-        ;; FIXME: why we have this cond enabled here, we need to move it outside this handler
+    (binding [cond/*enabled* true]
-        (binding [cond/*enabled* true]
+      (let [response (handler-fn data)]
-          (let [response (handler-fn data)]
+        (handle-response request response)))))
-            (handle-response request response)))))))
 
 (defn- wrap-metrics
   "Wrap service method with metrics measurement."
@@ -195,7 +200,7 @@
                         ::sm/explain (explain params)))))))
     f))
 
-(defn- wrap
+(defn- wrap-all
   [cfg f mdata]
   (as-> f $
     (wrap-db-transaction cfg $ mdata)
@@ -209,30 +214,17 @@
     (wrap-params-validation cfg $ mdata)
     (wrap-authentication cfg $ mdata)))
 
-(defn- wrap-management
+(defn- wrap
   [cfg f mdata]
-  (as-> f $
+  (l/trc :hint "register method" :name (::sv/name mdata))
-    (wrap-db-transaction cfg $ mdata)
+  (let [f (wrap-all cfg f mdata)]
-    (retry/wrap-retry cfg $ mdata)
+    (partial f cfg)))
-    (climit/wrap cfg $ mdata)
-    (wrap-metrics cfg $ mdata)
-    (wrap-audit cfg $ mdata)
-    (wrap-spec-conform cfg $ mdata)
-    (wrap-params-validation cfg $ mdata)
-    (wrap-authentication cfg $ mdata)))
 
 (defn- process-method
-  [cfg module wrap-fn [f mdata]]
+  [cfg [vfn mdata]]
-  (l/trc :hint "add method" :module module :name (::sv/name mdata))
+  [(keyword (::sv/name mdata)) [mdata (wrap cfg vfn mdata)]])
-  (let [f (wrap-fn cfg f mdata)
-        k (keyword (::sv/name mdata))]
-    [k [mdata (partial f cfg)]]))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defn- resolve-command-methods
-;; API METHODS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- resolve-methods
   [cfg]
   (let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
     (->> (sv/scan-ns
@@ -247,6 +239,7 @@
           'app.rpc.commands.files
           'app.rpc.commands.files-create
           'app.rpc.commands.files-share
+          'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
           'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails
@@ -261,7 +254,7 @@
           'app.rpc.commands.verify-token
           'app.rpc.commands.viewer
           'app.rpc.commands.webhooks)
-         (map (partial process-method cfg "rpc" wrap))
+         (map (partial process-method cfg))
          (into {}))))
 
 (def ^:private schema:methods-params
@@ -269,7 +262,6 @@
    ::session/manager
    ::http.client/client
    ::db/pool
-   ::rds/pool
    ::mbus/msgbus
    ::sto/storage
    ::mtx/metrics
@@ -285,50 +277,7 @@
 (defmethod ig/init-key ::methods
   [_ cfg]
   (let [cfg (d/without-nils cfg)]
-    (resolve-methods cfg)))
+    (resolve-command-methods cfg)))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; MANAGEMENT METHODS
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- resolve-management-methods
-  [cfg]
-  (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
-    (->> (sv/scan-ns
-          'app.rpc.management.subscription
-          'app.rpc.management.exporter)
-         (map (partial process-method cfg "management" wrap-management))
-         (into {}))))
-
-(def ^:private schema:management-methods-params
-  [:map {:title "management-methods-params"}
-   ::session/manager
-   ::http.client/client
-   ::db/pool
-   ::rds/pool
-   ::mbus/msgbus
-   ::sto/storage
-   ::mtx/metrics
-   ::setup/props])
-
-(defmethod ig/assert-key ::management-methods
-  [_ params]
-  (assert (sm/check schema:management-methods-params params)))
-
-(defmethod ig/init-key ::management-methods
-  [_ cfg]
-  (let [cfg (d/without-nils cfg)]
-    (resolve-management-methods cfg)))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ROUTES
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- redirect
-  [href]
-  (fn [_]
-    {::yres/status 308
-     ::yres/headers {"location" (str href)}}))
 
 (def ^:private schema:methods
   [:map-of :keyword [:tuple :map ::sm/fn]])
@@ -343,50 +292,11 @@
   (assert (db/pool? (::db/pool params)) "expect valid database pool")
   (assert (some? (::setup/props params)))
   (assert (session/manager? (::session/manager params)) "expect valid session manager")
-  (assert (valid-methods? (::methods params)) "expect valid methods map")
+  (assert (valid-methods? (::methods params)) "expect valid methods map"))
-  (assert (valid-methods? (::management-methods params)) "expect valid methods map"))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [::methods ::management-methods ::setup/props] :as cfg}]
+  [_ {:keys [::methods] :as cfg}]
-
+  (let [methods (update-vals methods peek)]
-  (let [public-uri     (cf/get :public-uri)
+    [["/rpc" {:middleware [[session/authz cfg]
-        management-key (or (cf/get :management-api-key)
+                           [actoken/authz cfg]]}
-                           (get props :management-key))]
+      ["/command/:type" {:handler (partial rpc-handler methods)}]]]))
-
-    ["/api"
-     ["/management"
-      ["/methods/:type"
-       {:middleware [[mw/shared-key-auth management-key]
-                     [session/authz cfg]]
-        :handler (make-rpc-handler management-methods)}]
-
-      (doc/routes :methods management-methods
-                  :label "management"
-                  :base-uri (u/join public-uri "/api/management")
-                  :description "MANAGEMENT API")]
-
-     ["/main"
-      ["/methods/:type"
-       {:middleware [[mw/cors]
-                     [sec/client-header-check]
-                     [session/authz cfg]
-                     [actoken/authz cfg]]
-        :handler (make-rpc-handler methods)}]
-
-      (doc/routes :methods methods
-                  :label "main"
-                  :base-uri (u/join public-uri "/api/main")
-                  :description "MAIN API")]
-
-     ;; BACKWARD COMPATIBILITY
-     ["/_doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
-     ["/doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
-     ["/openapi" {:handler (redirect (u/join public-uri "/api/main/doc/openapi"))}]
-     ["/openapi.join" {:handler (redirect (u/join public-uri "/api/main/doc/openapi.json"))}]
-
-     ["/rpc/command/:type"
-      {:middleware [[mw/cors]
-                    [sec/client-header-check]
-                    [session/authz cfg]
-                    [actoken/authz cfg]]
-       :handler (make-rpc-handler methods)}]]))

backend/src/app/rpc/climit.clj

@@ -21,6 +21,7 @@
    [clojure.set :as set]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [promesa.exec.bulkhead :as pbh])
   (:import
    clojure.lang.ExceptionInfo
@@ -288,9 +289,13 @@
           (get-limits cfg)))
 
 (defn invoke!
-  "Run a function in context of climit."
+  "Run a function in context of climit.
-  [{:keys [::rpc/climit] :as cfg} f params]
+  Intended to be used in virtual threads."
+  [{:keys [::executor ::rpc/climit] :as cfg} f params]
   (let [f (if climit
-            (build-exec-chain cfg f)
+            (let [f (if (some? executor)
+                      (fn [cfg params] (px/await! (px/submit! executor (fn [] (f cfg params)))))
+                      f)]
+              (build-exec-chain cfg f))
             f)]
     (f cfg params)))

backend/src/app/rpc/commands/access_token.clj

@@ -23,15 +23,14 @@
   (dissoc row :perms))
 
 (defn create-access-token
-  [{:keys [::db/conn] :as cfg} profile-id name expiration]
+  [{:keys [::db/conn ::setup/props]} profile-id name expiration]
-  (let [token-id   (uuid/next)
+  (let [created-at (ct/now)
-        expires-at (some-> expiration (ct/in-future))
+        token-id   (uuid/next)
-        created-at (ct/now)
+        token      (tokens/generate props {:iss "access-token"
-        token      (tokens/generate cfg {:iss "access-token"
+                                           :tid token-id
-                                         :uid profile-id
+                                           :iat created-at})
-                                         :iat created-at
-                                         :tid token-id})
 
+        expires-at (some-> expiration ct/in-future)
         token      (db/insert! conn :access-token
                                {:id token-id
                                 :name name

backend/src/app/rpc/commands/auth.clj

@@ -6,25 +6,21 @@
 
 (ns app.rpc.commands.auth
   (:require
-   [app.auth :as auth]
-   [app.auth.oidc :as oidc]
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
-   [app.common.uri :as u]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.email :as eml]
    [app.email.blacklist :as email.blacklist]
    [app.email.whitelist :as email.whitelist]
-   [app.http :as-alias http]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
-   [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
    [app.rpc.commands.profile :as profile]
@@ -33,7 +29,6 @@
    [app.rpc.helpers :as rph]
    [app.setup :as-alias setup]
    [app.setup.welcome-file :refer [create-welcome-file]]
-   [app.storage :as sto]
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.worker :as wrk]
@@ -67,7 +62,7 @@
               (ex/raise :type :validation
                         :code :account-without-password
                         :hint "the current account does not have password")
-              (let [result (auth/verify-password password (:password profile))]
+              (let [result (profile/verify-password cfg password (:password profile))]
                 (when (:update result)
                   (l/trc :hint "updating profile password"
                          :id (str (:id profile))
@@ -103,7 +98,7 @@
                                   (profile/strip-private-attrs))
 
                   invitation (when-let [token (:invitation-token params)]
-                               (tokens/verify cfg {:token token :iss :team-invitation}))
+                               (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
 
                   ;; If invitation member-id does not matches the profile-id, we just proceed to ignore the
                   ;; invitation because invitations matches exactly; and user can't login with other email and
@@ -113,7 +108,7 @@
                                (assoc profile :is-admin (let [admins (cf/get :admins)]
                                                           (contains? admins (:email profile)))))]
               (-> response
-                  (rph/with-transform (session/create-fn cfg profile))
+                  (rph/with-transform (session/create-fn cfg (:id profile)))
                   (rph/with-meta {::audit/props (audit/profile->props profile)
                                   ::audit/profile-id (:id profile)}))))]
 
@@ -149,24 +144,7 @@
   [cfg params]
   (if (= (:profile-id params)
          (::rpc/profile-id params))
-    (let [{:keys [claims]}
+    (rph/with-transform {} (session/delete-fn cfg))
-          (rph/get-auth-data params)
-
-          provider
-          (some->> (get claims :sso-provider-id)
-                   (oidc/get-provider cfg))
-
-          response
-          (if (and provider (:logout-uri provider))
-            (let [params {"logout_hint" (get claims :sso-session-id)
-                          "client_id" (get provider :client-id)
-                          "post_logout_redirect_uri" (str (cf/get :public-uri))}
-                  uri    (-> (u/uri (:logout-uri provider))
-                             (assoc :query (u/map->query-string params)))]
-              {:redirect-uri uri})
-            {})]
-
-      (rph/with-transform response (session/delete-fn cfg)))
     {}))
 
 ;; ---- COMMAND: Recover Profile
@@ -174,11 +152,11 @@
 (defn recover-profile
   [{:keys [::db/conn] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
-            (let [tdata (tokens/verify cfg {:token token :iss :password-recovery})]
+            (let [tdata (tokens/verify (::setup/props cfg) {:token token :iss :password-recovery})]
               (:profile-id tdata)))
 
           (update-password [conn profile-id]
-            (let [pwd (auth/derive-password password)]
+            (let [pwd (profile/derive-password cfg password)]
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
@@ -213,7 +191,7 @@
               :hint "registration disabled"))
 
   (when (contains? params :invitation-token)
-    (let [invitation (tokens/verify cfg
+    (let [invitation (tokens/verify (::setup/props cfg)
                                     {:token (:invitation-token params)
                                      :iss :team-invitation})]
       (when-not (= (:email params) (:member-email invitation))
@@ -270,7 +248,7 @@
                  :props {:newsletter-updates (or accept-newsletter-updates false)}}
 
         params (d/without-nils params)
-        token  (tokens/generate cfg params)]
+        token  (tokens/generate (::setup/props cfg) params)]
 
     (with-meta {:token token}
       {::audit/profile-id uuid/zero})))
@@ -292,29 +270,11 @@
 
 ;; ---- COMMAND: Register Profile
 
-(defn import-profile-picture
+(defn create-profile!
-  [cfg uri]
-  (try
-    (let [storage (sto/resolve cfg)
-          input   (media/download-image cfg uri)
-          input   (media/run {:cmd :info :input input})
-          hash    (sto/calculate-hash (:path input))
-          content (-> (sto/content (:path input) (:size input))
-                      (sto/wrap-with-hash hash))
-          sobject (sto/put-object! storage {::sto/content content
-                                            ::sto/deduplicate? true
-                                            :bucket "profile"
-                                            :content-type (:mtype input)})]
-      (:id sobject))
-    (catch Throwable cause
-      (l/err :hint "unable to import profile picture"
-             :cause cause)
-      nil)))
-
-(defn create-profile
   "Create the profile entry on the database with limited set of input
   attrs (all the other attrs are filled with default values)."
-  [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
+  [conn {:keys [email] :as params}]
+  (dm/assert! ::sm/email email)
   (let [id        (or (:id params) (uuid/next))
         props     (-> (audit/extract-utm-params params)
                       (merge (:props params))
@@ -322,7 +282,8 @@
                               :viewed-walkthrough? false
                               :nudge {:big 10 :small 1}
                               :v2-info-shown true
-                              :release-notes-viewed (:main cf/version)}))
+                              :release-notes-viewed (:main cf/version)})
+                      (db/tjson))
 
         password  (or (:password params) "!")
 
@@ -337,12 +298,6 @@
         theme     (:theme params nil)
         email     (str/lower email)
 
-        photo-id  (some->> (or (:oidc/picture props)
-                               (:google/picture props)
-                               (:github/picture props)
-                               (:gitlab/picture props))
-                           (import-profile-picture cfg))
-
         params    {:id id
                    :fullname (:fullname params)
                    :email email
@@ -350,26 +305,27 @@
                    :lang locale
                    :password password
                    :deleted-at (:deleted-at params)
-                   :props (db/tjson props)
+                   :props props
                    :theme theme
-                   :photo-id photo-id
                    :is-active is-active
                    :is-muted is-muted
                    :is-demo is-demo}]
-
     (try
       (-> (db/insert! conn :profile params)
           (profile/decode-row))
       (catch org.postgresql.util.PSQLException cause
-        (if (db/duplicate-key-error? cause)
+        (let [state (.getSQLState cause)]
-          (ex/raise :type :validation
+          (if (not= state "23505")
-                    :code :email-already-exists
+            (throw cause)
-                    :hint "email already exists"
-                    :cause cause)
-          (throw cause))))))
 
+            (do
+              (l/error :hint "not an error" :cause cause)
+              (ex/raise :type :validation
+                        :code :email-already-exists
+                        :hint "email already exists"
+                        :cause cause))))))))
 
-(defn create-profile-rels
+(defn create-profile-rels!
   [conn {:keys [id] :as profile}]
   (let [features (cfeat/get-enabled-features cf/flags)
         team     (teams/create-team conn
@@ -386,14 +342,14 @@
 
 (defn send-email-verification!
   [{:keys [::db/conn] :as cfg} profile]
-  (let [vtoken (tokens/generate cfg
+  (let [vtoken (tokens/generate (::setup/props cfg)
                                 {:iss :verify-email
                                  :exp (ct/in-future "72h")
                                  :profile-id (:id profile)
                                  :email (:email profile)})
         ;; NOTE: this token is mainly used for possible complains
         ;; identification on the sns webhook
-        ptoken (tokens/generate cfg
+        ptoken (tokens/generate (::setup/props cfg)
                                 {:iss :profile-identity
                                  :profile-id (:id profile)
                                  :exp (ct/in-future {:days 30})})]
@@ -407,7 +363,7 @@
 
 (defn register-profile
   [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token] :as params}]
-  (let [claims     (tokens/verify cfg {:token token :iss :prepared-register})
+  (let [claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
         params     (into claims params)
 
         profile    (if-let [profile-id (:profile-id claims)]
@@ -419,19 +375,18 @@
                      ;; to detect if the profile is already registered
                      (or (profile/get-profile-by-email conn (:email claims))
                          (let [is-active (or (boolean (:is-active claims))
-                                             (boolean (:email-verified claims))
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
-                                             (update :password auth/derive-password))
+                                             (update :password #(profile/derive-password cfg %)))
-                               profile   (->> (create-profile cfg params)
+                               profile   (->> (create-profile! conn params)
-                                              (create-profile-rels conn))]
+                                              (create-profile-rels! conn))]
                            (vary-meta profile assoc :created true))))
 
         created?   (-> profile meta :created true?)
 
         invitation (when-let [token (:invitation-token params)]
-                     (tokens/verify cfg {:token token :iss :team-invitation}))
+                     (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
 
         props      (-> (audit/profile->props profile)
                        (assoc :from-invitation (some? invitation)))
@@ -463,10 +418,10 @@
       (and (some? invitation)
            (= (:email profile)
               (:member-email invitation)))
-      (let [invitation (assoc invitation :member-id  (:id profile))
+      (let [claims (assoc invitation :member-id  (:id profile))
-            token      (tokens/generate cfg invitation)]
+            token  (tokens/generate (::setup/props cfg) claims)]
         (-> {:invitation-token token}
-            (rph/with-transform (session/create-fn cfg profile claims))
+            (rph/with-transform (session/create-fn cfg (:id profile)))
             (rph/with-meta {::audit/replace-props props
                             ::audit/context {:action "accept-invitation"}
                             ::audit/profile-id (:id profile)})))
@@ -477,7 +432,7 @@
       created?
       (if (:is-active profile)
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg profile claims))
+            (rph/with-transform (session/create-fn cfg (:id profile)))
             (rph/with-defer create-welcome-file-when-needed)
             (rph/with-meta
               {::audit/replace-props props
@@ -538,14 +493,14 @@
 (defn- request-profile-recovery
   [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
   (letfn [(create-recovery-token [{:keys [id] :as profile}]
-            (let [token (tokens/generate cfg
+            (let [token (tokens/generate (::setup/props cfg)
                                          {:iss :password-recovery
                                           :exp (ct/in-future "15m")
                                           :profile-id id})]
               (assoc profile :token token)))
 
           (send-email-notification [conn profile]
-            (let [ptoken (tokens/generate cfg
+            (let [ptoken (tokens/generate (::setup/props cfg)
                                           {:iss :profile-identity
                                            :profile-id (:id profile)
                                            :exp (ct/in-future {:days 30})})]
@@ -606,32 +561,4 @@
   [cfg params]
   (db/tx-run! cfg request-profile-recovery params))
 
-;; --- COMMAND: get-sso-config
 
-(defn- extract-domain
-  "Extract the domain part from email"
-  [email]
-  (let [at (str/last-index-of email "@")]
-    (when (and (>= at 0)
-               (< at (dec (count email))))
-      (-> (subs email (inc at))
-          (str/trim)
-          (str/lower)))))
-
-(def ^:private schema:get-sso-provider
-  [:map {:title "get-sso-config"}
-   [:email ::sm/email]])
-
-(def ^:private schema:get-sso-provider-result
-  [:map {:title "SSOProvider"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::get-sso-provider
-  {::rpc/auth false
-   ::doc/added "2.12"
-   ::sm/params schema:get-sso-provider
-   ::sm/result schema:get-sso-provider-result}
-  [cfg {:keys [email]}]
-  (when-let [domain (extract-domain email)]
-    (when-let [config (db/get* cfg :sso-provider {:domain domain})]
-      (select-keys config [:id]))))

backend/src/app/rpc/commands/binfile.clj

@@ -11,9 +11,9 @@
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
    [app.common.features :as cfeat]
+   [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
-   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
@@ -25,12 +25,11 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
-   [app.storage :as sto]
-   [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.services :as sv]
    [app.worker :as-alias wrk]
-   [datoteka.fs :as fs]))
+   [promesa.exec :as px]
+   [yetti.response :as yres]))
 
 (set! *warn-on-reflection* true)
 
@@ -40,47 +39,62 @@
   schema:export-binfile
   [:map {:title "export-binfile"}
    [:file-id ::sm/uuid]
+   [:version {:optional true} ::sm/int]
    [:include-libraries ::sm/boolean]
    [:embed-assets ::sm/boolean]])
 
-(defn- export-binfile
+(defn stream-export-v1
-  [{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
+  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (let [output  (tmp/tempfile*)]
+  (yres/stream-body
-    (try
+   (fn [_ output-stream]
-      (-> cfg
+     (try
-          (assoc ::bfc/ids #{file-id})
+       (-> cfg
-          (assoc ::bfc/embed-assets embed-assets)
+           (assoc ::bfc/ids #{file-id})
-          (assoc ::bfc/include-libraries include-libraries)
+           (assoc ::bfc/embed-assets embed-assets)
-          (bf.v3/export-files! output))
+           (assoc ::bfc/include-libraries include-libraries)
+           (bf.v1/export-files! output-stream))
+       (catch Throwable cause
+         (l/err :hint "exception on exporting file"
+                :file-id (str file-id)
+                :cause cause))))))
 
-      (let [data   (sto/content output)
+(defn stream-export-v3
-            object (sto/put-object! storage
+  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-                                    {::sto/content data
+  (yres/stream-body
-                                     ::sto/touched-at (ct/in-future {:minutes 60})
+   (fn [_ output-stream]
-                                     :content-type "application/zip"
+     (try
-                                     :bucket "tempfile"})]
+       (-> cfg
-
+           (assoc ::bfc/ids #{file-id})
-        (-> (cf/get :public-uri)
+           (assoc ::bfc/embed-assets embed-assets)
-            (u/join "/assets/by-id/")
+           (assoc ::bfc/include-libraries include-libraries)
-            (u/join (str (:id object)))))
+           (bf.v3/export-files! output-stream))
-
+       (catch Throwable cause
-      (finally
+         (l/err :hint "exception on exporting file"
-        (fs/delete output)))))
+                :file-id (str file-id)
+                :cause cause))))))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
-   ::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
    ::webhooks/event? true
    ::sm/params schema:export-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (sse/response (partial export-binfile cfg params)))
+  (fn [_]
+    (let [version (or version 1)
+          body    (case (int version)
+                    1 (stream-export-v1 cfg params)
+                    2 (throw (ex-info "not-implemented" {}))
+                    3 (stream-export-v3 cfg params))]
+
+      {::yres/status 200
+       ::yres/headers {"content-type" "application/octet-stream"}
+       ::yres/body body})))
 
 ;; --- Command: import-binfile
 
 (defn- import-binfile
-  [{:keys [::db/pool] :as cfg} {:keys [profile-id project-id version name file]}]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
   (let [team   (teams/get-team pool
                                :profile-id profile-id
                                :project-id project-id)
@@ -91,9 +105,13 @@
                    (assoc ::bfc/name name)
                    (assoc ::bfc/input (:path file)))
 
+        ;; NOTE: the importation process performs some operations that are
+        ;; not very friendly with virtual threads, and for avoid
+        ;; unexpected blocking of other concurrent operations we dispatch
+        ;; that operation to a dedicated executor.
         result (case (int version)
-                 1 (bf.v1/import-files! cfg)
+                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
-                 3 (bf.v3/import-files! cfg))]
+                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
 
     (db/update! pool :project
                 {:modified-at (ct/now)}
@@ -109,7 +127,7 @@
    [:project-id ::sm/uuid]
    [:file-id {:optional true} ::sm/uuid]
    [:version {:optional true} ::sm/int]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::import-binfile
   "Import a penpot file in a binary format. If `file-id` is provided,

backend/src/app/rpc/commands/comments.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.comments
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -164,16 +163,34 @@
 (def xf-decode-row
   (map decode-row))
 
+(def ^:private
+  sql:get-file
+  "SELECT f.id, f.modified_at, f.revn, f.features, f.name,
+          f.project_id, p.team_id, f.data,
+          f.data_ref_id, f.data_backend
+     FROM file as f
+    INNER JOIN project as p on (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
+
 (defn- get-file
   "A specialized version of get-file for comments module."
   [cfg file-id page-id]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+  (let [file (db/exec-one! cfg [sql:get-file file-id])]
-    (let [file (bfc/get-file cfg file-id)
+    (when-not file
-          data (get file :data)]
+      (ex/raise :type :not-found
-      (-> file
+                :code :object-not-found
-          (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
+                :hint "file not found"))
-          (assoc :page-id page-id)
+
-          (dissoc :data)))))
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+      (let [file (->> file
+                      (files/decode-row)
+                      (feat.fdata/resolve-file-data cfg))
+            data (get file :data)]
+        (-> file
+            (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
+            (assoc :page-id page-id)
+            (dissoc :data))))))
 
 ;; FIXME: rename
 (defn- get-comment-thread
@@ -234,39 +251,34 @@
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
-(defn- get-comment-threads-sql
+(def ^:private sql:comment-threads
-  [where]
+  "SELECT DISTINCT ON (ct.id)
-  (str/ffmt
+          ct.*,
-   "SELECT DISTINCT ON (ct.id)
+          pf.fullname AS owner_fullname,
-           ct.*,
+          pf.email AS owner_email,
-           pf.fullname AS owner_fullname,
+          pf.photo_id AS owner_photo_id,
-           pf.email AS owner_email,
+          p.team_id AS team_id,
-           pf.photo_id AS owner_photo_id,
+          f.name AS file_name,
-           p.team_id AS team_id,
+          f.project_id AS project_id,
-           f.name AS file_name,
+          first_value(c.content) OVER w AS content,
-           f.project_id AS project_id,
+          (SELECT count(1)
-           first_value(c.content) OVER w AS content,
+             FROM comment AS c
-           (SELECT count(1)
+            WHERE c.thread_id = ct.id) AS count_comments,
-              FROM comment AS c
+          (SELECT count(1)
-             WHERE c.thread_id = ct.id) AS count_comments,
+             FROM comment AS c
-           (SELECT count(1)
+            WHERE c.thread_id = ct.id
-              FROM comment AS c
+              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
-             WHERE c.thread_id = ct.id
+     FROM comment_thread AS ct
-               AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+    INNER JOIN comment AS c ON (c.thread_id = ct.id)
-      FROM comment_thread AS ct
+    INNER JOIN file AS f ON (f.id = ct.file_id)
-     INNER JOIN comment AS c ON (c.thread_id = ct.id)
+    INNER JOIN project AS p ON (p.id = f.project_id)
-     INNER JOIN file AS f ON (f.id = ct.file_id)
+     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
-     INNER JOIN project AS p ON (p.id = f.project_id)
+     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
-      LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
-      LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
-     WHERE f.deleted_at IS NULL
-       AND p.deleted_at IS NULL
-       %1
-    WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)"
-   where))
 
 (def ^:private sql:comment-threads-by-file-id
-  (get-comment-threads-sql "AND ct.file_id = ?"))
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
@@ -275,30 +287,7 @@
 
 ;; --- COMMAND: Get Unread Comment Threads
 
-(def ^:private sql:unread-all-comment-threads-by-team
+(declare ^:private get-unread-comment-threads)
-  (str "WITH threads AS ("
-       (get-comment-threads-sql "AND p.team_id = ?")
-       ")"
-       "SELECT t.* FROM threads AS t
-         WHERE t.count_unread_comments > 0"))
-
-(def ^:private sql:unread-partial-comment-threads-by-team
-  (str "WITH threads AS ("
-       (get-comment-threads-sql "AND p.team_id = ? AND (ct.owner_id = ? OR ? = ANY(ct.mentions))")
-       ")"
-       "SELECT t.* FROM threads AS t
-         WHERE t.count_unread_comments > 0"))
-
-(defn- get-unread-comment-threads
-  [cfg profile-id team-id]
-  (let [profile (-> (db/get cfg :profile {:id profile-id} ::db/remove-deleted false)
-                    (profile/decode-row))
-        notify  (or (-> profile :props :notifications :dashboard-comments) :all)
-        result  (case notify
-                  :all     (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
-                  :partial (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-                  [])]
-    (into [] xf-decode-row result)))
 
 (def ^:private
   schema:get-unread-comment-threads
@@ -309,8 +298,41 @@
   {::doc/added "1.15"
    ::sm/params schema:get-unread-comment-threads}
   [cfg {:keys [::rpc/profile-id team-id] :as params}]
-  (teams/check-read-permissions! cfg profile-id team-id)
+  (db/run!
-  (get-unread-comment-threads cfg profile-id team-id))
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (teams/check-read-permissions! conn profile-id team-id)
+     (get-unread-comment-threads conn profile-id team-id))))
+
+(def sql:unread-all-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
+
+;; The partial configuration will retrieve only comments created by the user and
+;; threads that have a mention to the user.
+(def sql:unread-partial-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads
+         WHERE count_unread_comments > 0
+           AND team_id = ?
+           AND (owner_id = ? OR ? = ANY(mentions))"))
+
+(defn- get-unread-comment-threads
+  [conn profile-id team-id]
+  (let [profile (-> (db/get conn :profile {:id profile-id})
+                    (profile/decode-row))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
+
+    (case notify
+      :all
+      (->> (db/exec! conn [sql:unread-all-comment-threads-by-team profile-id team-id])
+           (into [] xf-decode-row))
+
+      :partial
+      (->> (db/exec! conn [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+           (into [] xf-decode-row))
+
+      [])))
 
 ;; --- COMMAND: Get Single Comment Thread
 
@@ -321,17 +343,16 @@
    [:id ::sm/uuid]
    [:share-id {:optional true} [:maybe ::sm/uuid]]])
 
-(def ^:private sql:get-comment-thread
-  (get-comment-threads-sql "AND ct.file_id = ? AND ct.id = ?"))
-
 (sv/defmethod ::get-comment-thread
   {::doc/added "1.15"
    ::sm/params schema:get-comment-thread}
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
+                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
-                         (decode-row)))))
+                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                   (-> (db/exec-one! conn [sql profile-id id file-id])
+                       (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
 

backend/src/app/rpc/commands/demo.clj

@@ -7,7 +7,6 @@
 (ns app.rpc.commands.demo
   "A demo specific mutations."
   (:require
-   [app.auth :refer [derive-password]]
    [app.common.exceptions :as ex]
    [app.common.time :as ct]
    [app.config :as cf]
@@ -15,6 +14,7 @@
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.auth :as auth]
+   [app.rpc.commands.profile :as profile]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
    [buddy.core.codecs :as bc]
@@ -39,19 +39,18 @@
         fullname (str "Demo User " sem)
 
         password (-> (bn/random-bytes 16)
-                     (bc/bytes->b64 true)
+                     (bc/bytes->b64u)
                      (bc/bytes->str))
 
         params   {:email email
                   :fullname fullname
                   :is-active true
-                  :is-demo true
                   :deleted-at (ct/in-future (cf/get-deletion-delay))
-                  :password (derive-password password)
+                  :password (profile/derive-password cfg password)
                   :props {}}
-        profile  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+        profile  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                                   (->> (auth/create-profile cfg params)
+                                   (->> (auth/create-profile! conn params)
-                                        (auth/create-profile-rels conn))))]
+                                        (auth/create-profile-rels! conn))))]
     (with-meta {:email email
                 :password password}
       {::audit/profile-id (:id profile)})))

backend/src/app/rpc/commands/feedback.clj

@@ -7,7 +7,6 @@
 (ns app.rpc.commands.feedback
   "A general purpose feedback module."
   (:require
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.config :as cf]
@@ -22,11 +21,8 @@
 
 (def ^:private schema:send-user-feedback
   [:map {:title "send-user-feedback"}
-   [:subject [:string {:max 500}]]
+   [:subject [:string {:max 400}]]
-   [:content [:string {:max 2500}]]
+   [:content [:string {:max 2500}]]])
-   [:type {:optional true} :string]
-   [:error-href {:optional true} [:string {:max 2500}]]
-   [:error-report {:optional true} :string]])
 
 (sv/defmethod ::send-user-feedback
   {::doc/added "1.18"
@@ -43,26 +39,16 @@
 
 (defn- send-user-feedback!
   [pool profile params]
-  (let [destination
+  (let [dest (or (cf/get :user-feedback-destination)
-        (or (cf/get :user-feedback-destination)
+                 ;; LEGACY
-            ;; LEGACY
+                 (cf/get :feedback-destination))]
-            (cf/get :feedback-destination))
-
-        attachments
-        (d/without-nils
-         {"error-report.txt" (:error-report params)})]
-
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/user-feedback
-                :from     (cf/get :smtp-default-from)
+                :from     dest
-                :to       destination
+                :to       dest
+                :profile  profile
                 :reply-to (:email profile)
                 :email    (:email profile)
-                :attachments attachments
+                :subject  (:subject params)
-
+                :content  (:content params)})
-                :feedback-subject (:subject params)
-                :feedback-type (:type params "not-specified")
-                :feedback-content (:content params)
-                :feedback-error-href (:error-href params)
-                :profile profile})
     nil))

backend/src/app/rpc/commands/files.clj

@@ -17,7 +17,6 @@
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as-alias smdj]
    [app.common.time :as ct]
-   [app.common.transit :as t]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
    [app.common.uri :as uri]
@@ -25,12 +24,10 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.features.logical-deletion :as ldel]
-   [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
-   [app.msgbus :as mbus]
-   [app.redis :as rds]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
@@ -39,11 +36,11 @@
    [app.rpc.helpers :as rph]
    [app.rpc.permissions :as perms]
    [app.util.blob :as blob]
-   [app.util.events :as events]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
 
 ;; --- FEATURES
 
@@ -58,10 +55,12 @@
   (ct/duration {:days 7}))
 
 (defn decode-row
-  [{:keys [features] :as row}]
+  [{:keys [data changes features] :as row}]
   (when row
     (cond-> row
-      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
 
 (defn check-version!
   [file]
@@ -79,16 +78,13 @@
 
 ;; --- FILE PERMISSIONS
 
-
 (def ^:private sql:file-permissions
   "select fpr.is_owner,
           fpr.is_admin,
           fpr.can_edit
      from file_profile_rel as fpr
-    inner join file as f on (f.id = fpr.file_id)
     where fpr.file_id = ?
       and fpr.profile_id = ?
-      and f.deleted_at is null
    union all
    select tpr.is_owner,
           tpr.is_admin,
@@ -98,7 +94,6 @@
     inner join file as f on (p.id = f.project_id)
     where f.id = ?
       and tpr.profile_id = ?
-      and f.deleted_at is null
    union all
    select ppr.is_owner,
           ppr.is_admin,
@@ -106,8 +101,7 @@
      from project_profile_rel as ppr
     inner join file as f on (f.project_id = ppr.project_id)
     where f.id = ?
-      and ppr.profile_id = ?
+      and ppr.profile_id = ?")
-      and f.deleted_at is null")
 
 (defn get-file-permissions
   [conn profile-id file-id]
@@ -201,7 +195,7 @@
 
 (def schema:permissions-mixin
   [:map {:title "PermissionsMixin"}
-   [:permissions perms/schema:permissions]])
+   [:permissions ::perms/permissions]])
 
 (def schema:file-with-permissions
   [:merge {:title "FileWithPermissions"}
@@ -212,11 +206,93 @@
   schema:get-file
   [:map {:title "get-file"}
    [:features {:optional true} ::cfeat/features]
-   [:id ::sm/uuid]])
+   [:id ::sm/uuid]
+   [:project-id {:optional true} ::sm/uuid]])
+
+(defn- migrate-file
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+            pmap/*tracked* (pmap/create-tracked)]
+    (let [libs (delay (bfc/get-resolved-file-libraries cfg file))
+          ;; For avoid unnecesary overhead of creating multiple pointers and
+          ;; handly internally with objects map in their worst case (when
+          ;; probably all shapes and all pointers will be readed in any
+          ;; case), we just realize/resolve them before applying the
+          ;; migration to the file
+          file (-> file
+                   (update :data feat.fdata/process-pointers deref)
+                   (update :data feat.fdata/process-objects (partial into {}))
+                   (fmg/migrate-file libs))]
+
+      (if (or read-only? (db/read-only? conn))
+        file
+        (let [;; When file is migrated, we break the rule of no perform
+              ;; mutations on get operations and update the file with all
+              ;; migrations applied
+              file (if (contains? (:features file) "fdata/objects-map")
+                     (feat.fdata/enable-objects-map file)
+                     file)
+              file (if (contains? (:features file) "fdata/pointer-map")
+                     (feat.fdata/enable-pointer-map file)
+                     file)]
+
+          (db/update! conn :file
+                      {:data (blob/encode (:data file))
+                       :version (:version file)
+                       :features (db/create-array conn "text" (:features file))}
+                      {:id id}
+                      {::db/return-keys false})
+
+          (when (contains? (:features file) "fdata/pointer-map")
+            (feat.fdata/persist-pointers! cfg id))
+
+          (feat.fmigr/upsert-migrations! conn file)
+          (feat.fmigr/resolve-applied-migrations cfg file))))))
+
+(defn get-file
+  [{:keys [::db/conn ::wrk/executor] :as cfg} id
+   & {:keys [project-id
+             migrate?
+             include-deleted?
+             lock-for-update?
+             preload-pointers?]
+      :or {include-deleted? false
+           lock-for-update? false
+           migrate? true
+           preload-pointers? false}
+      :as options}]
+
+  (assert (db/connection? conn) "expected cfg with valid connection")
+
+  (let [params (merge {:id id}
+                      (when (some? project-id)
+                        {:project-id project-id}))
+        file   (->> (db/get conn :file params
+                            {::db/check-deleted (not include-deleted?)
+                             ::db/remove-deleted (not include-deleted?)
+                             ::sql/for-update lock-for-update?})
+                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (feat.fdata/resolve-file-data cfg))
+
+        ;; NOTE: we perform the file decoding in a separate thread
+        ;; because it has heavy and synchronous operations for
+        ;; decoding file body that are not very friendly with virtual
+        ;; threads.
+        file   (px/invoke! executor #(decode-row file))
+
+        file   (if (and migrate? (fmg/need-migration? file))
+                 (migrate-file cfg file options)
+                 file)]
+
+    (if preload-pointers?
+      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+        (update file :data feat.fdata/process-pointers deref))
+
+      file)))
 
 (defn get-minimal-file
   [cfg id & {:as opts}]
-  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern])]
+  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern :data-ref-id :data-backend])]
     (db/get cfg :file {:id id} opts)))
 
 (defn- get-minimal-file-with-perms
@@ -256,32 +332,23 @@
                                :project-id project-id
                                :file-id id)
 
-          file (-> (bfc/get-file cfg id
+          file (-> (get-file cfg id :project-id project-id)
-                                 :project-id project-id)
                    (assoc :permissions perms)
+                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
           (cfeat/check-client-features! (:features params))
           (cfeat/check-file-features! (:features file)))
 
-      (as-> file file
+      ;; This operation is needed for backward comapatibility with frontends that
-        ;; This operation is needed for backward comapatibility with
+      ;; does not support pointer-map resolution mechanism; this just resolves the
-        ;; frontends that does not support pointer-map resolution
+      ;; pointers on backend and return a complete file.
-        ;; mechanism; this just resolves the pointers on backend and
+      (if (and (contains? (:features file) "fdata/pointer-map")
-        ;; return a complete file
+               (not (contains? (:features params) "fdata/pointer-map")))
-        (if (and (contains? (:features file) "fdata/pointer-map")
+        (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                 (not (contains? (:features params) "fdata/pointer-map")))
+          (update file :data feat.fdata/process-pointers deref))
-          (feat.fdata/realize-pointers cfg file)
+        file))))
-          file)
-
-        ;; This operation is needed for backward comapatibility with
-        ;; frontends that does not support objects-map mechanism; this
-        ;; just converts all objects map instaces to plain maps
-        (if (and (contains? (:features file) "fdata/objects-map")
-                 (not (contains? (:features params) "fdata/objects-map")))
-          (feat.fdata/realize-objects cfg file)
-          file)))))
 
 ;; --- COMMAND QUERY: get-file-fragment (by id)
 
@@ -290,7 +357,7 @@
    [:id ::sm/uuid]
    [:file-id ::sm/uuid]
    [:created-at ::ct/inst]
-   [:content ::sm/any]])
+   [:content any?]])
 
 (def schema:get-file-fragment
   [:map {:title "get-file-fragment"}
@@ -300,8 +367,10 @@
 
 (defn- get-file-fragment
   [cfg file-id fragment-id]
-  (some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"})
+  (let [resolve-file-data (partial feat.fdata/resolve-file-data cfg)]
-          (update :data blob/decode)))
+    (some-> (db/get cfg :file-data-fragment {:file-id file-id :id fragment-id})
+            (resolve-file-data)
+            (update :data blob/decode))))
 
 (sv/defmethod ::get-file-fragment
   "Retrieve a file fragment by its ID. Only authenticated users."
@@ -355,8 +424,9 @@
    ::sm/params schema:get-project-files
    ::sm/result schema:files}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id]}]
-  (projects/check-read-permissions! pool profile-id project-id)
+  (dm/with-open [conn (db/open pool)]
-  (get-project-files pool project-id))
+    (projects/check-read-permissions! conn profile-id project-id)
+    (get-project-files conn project-id)))
 
 ;; --- COMMAND QUERY: has-file-libraries
 
@@ -390,39 +460,6 @@
     (:has-libraries row)))
 
 
-;; --- COMMAND QUERY: get-library-usage
-
-
-(declare get-library-usage)
-
-(def schema:get-library-usage
-  [:map {:title "get-library-usage"}
-   [:file-id ::sm/uuid]])
-:sample
-(sv/defmethod ::get-library-usage
-  "Gets the number of files that use the specified library."
-  {::doc/added "2.10.0"
-   ::sm/params schema:get-library-usage
-   ::sm/result ::sm/int}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! pool profile-id file-id)
-    (get-library-usage conn file-id)))
-
-(def ^:private sql:get-library-usage
-  "SELECT COUNT(*) AS used
-     FROM file_library_rel AS flr
-     JOIN file AS fl ON (flr.library_file_id = fl.id)
-    WHERE flr.library_file_id = ?::uuid
-      AND (fl.deleted_at IS NULL OR
-           fl.deleted_at > now())")
-
-(defn- get-library-usage
-  [conn file-id]
-  (let [row (db/exec-one! conn [sql:get-library-usage file-id])]
-    {:used-in (:used row)}))
-
-
 ;; --- QUERY COMMAND: get-page
 
 (defn- prune-objects
@@ -458,7 +495,7 @@
 
   (let [perms (get-permissions conn profile-id file-id share-id)
 
-        file  (bfc/get-file cfg file-id :read-only? true)
+        file  (get-file cfg file-id :read-only? true)
 
         proj  (db/get conn :project {:id (:project-id file)})
 
@@ -514,136 +551,66 @@
 
 ;; --- COMMAND QUERY: get-team-shared-files
 
-(defn- get-components-with-variants
-  "Return a set with all the variant-ids, and a list of components, but
-  with only one component by variant.
-
-  Returns a vector of unique components and a set of all variant ids"
-  [fdata]
-  (loop [variant-ids #{}
-         components' []
-         components  (ctkl/components-seq fdata)]
-    (if-let [{:keys [variant-id] :as component} (first components)]
-      (cond
-        (nil? variant-id)
-        (recur variant-ids
-               (conj components' component)
-               (rest components))
-
-        (contains? variant-ids variant-id)
-        (recur variant-ids
-               components'
-               (rest components))
-
-        :else
-        (recur (conj variant-ids variant-id)
-               (conj components' component)
-               (rest components)))
-
-      [(d/index-by :id components') variant-ids])))
-
-(defn- sample-assets
-  [assets limit]
-  (let [assets (into [] (map val) assets)]
-    {:count (count assets)
-     :sample (->> assets
-                  (sort-by #(str/lower (:name %)))
-                  (into [] (take limit)))}))
-
-(defn- calculate-library-summary
-  "Calculate the file library summary (counters and samples)"
-  [{:keys [data] :as file}]
-  (let [load-objects
-        (fn [sample]
-          (mapv #(ctf/load-component-objects data %) sample))
-
-        [components variant-ids]
-        (get-components-with-variants data)
-
-        components-sample
-        (-> (sample-assets components 4)
-            (update :sample load-objects))]
-
-    {:components components-sample
-     :variants {:count (count variant-ids)}
-     :colors (sample-assets (:colors data) 3)
-     :typographies (sample-assets (:typographies data) 3)}))
-
-(def ^:private file-summary-cache-key-ttl
-  (ct/duration {:days 30}))
-
-(def file-summary-cache-key-prefix
-  "penpot.library-summary.")
-
-(defn- get-file-with-summary
-  "Get a file without data with a summary of its local library content"
-  [cfg id]
-  (let [get-from-cache
-        (fn [{:keys [::rds/conn]} cache-key]
-          (when-let [result (rds/get conn cache-key)]
-            (let [file    (bfc/get-file cfg id :load-data? false)
-                  summary (t/decode-str result)]
-              (-> (assoc file :library-summary summary)
-                  (dissoc :data)))))
-
-        calculate-from-db
-        (fn []
-          (let [file   (bfc/get-file cfg id)
-                result (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                         (calculate-library-summary file))]
-            (-> file
-                (assoc :library-summary result)
-                (dissoc :legacy-data)
-                (dissoc :data))))
-
-        persist-to-cache
-        (fn [{:keys [::rds/conn]} data cache-key]
-          (rds/set conn cache-key (t/encode-str data)
-                   (rds/build-set-args {:ex file-summary-cache-key-ttl})))]
-
-    (if (contains? cf/flags :redis-cache)
-      (let [cache-key (str file-summary-cache-key-prefix id)]
-        (or (rds/run! cfg get-from-cache cache-key)
-            (let [file (calculate-from-db)]
-              (rds/run! cfg persist-to-cache (:library-summary file) cache-key)
-              file)))
-      (calculate-from-db))))
-
 (def ^:private sql:team-shared-files
-  "WITH file_library_agg AS (
+  "select f.id,
-      SELECT flr.file_id,
+          f.revn,
-             coalesce(array_agg(flr.library_file_id) filter (WHERE flr.library_file_id IS NOT NULL), '{}') AS library_file_ids
+          f.vern,
-        FROM file_library_rel flr
+          f.data,
-       GROUP BY flr.file_id
+          f.project_id,
-   )
+          f.created_at,
+          f.modified_at,
+          f.data_backend,
+          f.data_ref_id,
+          f.name,
+          f.version,
+          f.is_shared,
+          ft.media_id,
+          p.team_id
+     from file as f
+    inner join project as p on (p.id = f.project_id)
+     left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn and ft.deleted_at is null)
+    where f.is_shared = true
+      and f.deleted_at is null
+      and p.deleted_at is null
+      and p.team_id = ?
+    order by f.modified_at desc")
 
-   SELECT f.id,
+(defn- get-library-summary
-          fla.library_file_ids,
+  [cfg {:keys [id data] :as file}]
-          ft.media_id AS thumbnail_id
+  (letfn [(assets-sample [assets limit]
-     FROM file AS f
+            (let [sorted-assets (->> (vals assets)
-    INNER JOIN project AS p ON (p.id = f.project_id)
+                                     (sort-by #(str/lower (:name %))))]
-     LEFT JOIN file_thumbnail AS ft ON (ft.file_id = f.id AND ft.revn = f.revn AND ft.deleted_at IS NULL)
+              {:count (count sorted-assets)
-     LEFT JOIN file_library_agg AS fla ON (fla.file_id = f.id)
+               :sample (into [] (take limit sorted-assets))}))]
-    WHERE f.is_shared = true
+
-      AND f.deleted_at IS NULL
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-      AND p.deleted_at IS NULL
+      (let [load-objects      (fn [component]
-      AND p.team_id = ?
+                                (ctf/load-component-objects data component))
-    ORDER BY f.modified_at DESC")
+            components-sample (-> (assets-sample (ctkl/components data) 4)
+                                  (update :sample #(mapv load-objects %)))]
+        {:components components-sample
+         :media (assets-sample (:media data) 3)
+         :colors (assets-sample (:colors data) 3)
+         :typographies (assets-sample (:typographies data) 3)}))))
 
 (defn- get-team-shared-files
   [{:keys [::db/conn] :as cfg} {:keys [team-id profile-id]}]
   (teams/check-read-permissions! conn profile-id team-id)
-
+  (->> (db/exec! conn [sql:team-shared-files team-id])
-  (let [process-row
+       (into #{} (comp
-        (fn [{:keys [id library-file-ids]}]
+                  ;; NOTE: this decode operation is a workaround for a
-          (let [file (get-file-with-summary cfg id)]
+                  ;; fast fix, this should be approached with a more
-            (assoc file :library-file-ids (db/decode-pgarray library-file-ids #{}))))
+                  ;; efficient implementation, for now it loads all
-
+                  ;; the files in memory.
-        xform
+                  (map (partial bfc/decode-file cfg))
-        (map process-row)]
+                  (map (fn [row]
-
+                         (if-let [media-id (:media-id row)]
-    (->> (db/plan conn [sql:team-shared-files team-id] {:fetch-size 1})
+                           (-> row
-         (transduce xform conj #{}))))
+                               (dissoc :media-id)
+                               (assoc :thumbnail-id media-id))
+                           (dissoc row :media-id))))
+                  (map #(assoc % :library-summary (get-library-summary cfg %)))
+                  (map #(dissoc % :data))))))
 
 (def ^:private schema:get-team-shared-files
   [:map {:title "get-team-shared-files"}
@@ -656,28 +623,6 @@
   [cfg {:keys [::rpc/profile-id] :as params}]
   (db/tx-run! cfg get-team-shared-files (assoc params :profile-id profile-id)))
 
-;; --- COMMAND QUERY: get-file-summary
-
-(defn- get-file-summary
-  [cfg id]
-  (let [file (get-file-with-summary cfg id)]
-    (-> (:library-summary file)
-        (assoc :name (:name file)))))
-
-(def ^:private
-  schema:get-file-summary
-  [:map {:title "get-file-summary"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::get-file-summary
-  "Retrieve a file summary by its ID. Only authenticated users."
-  {::doc/added "1.20"
-   ::sm/params schema:get-file-summary}
-  [cfg {:keys [::rpc/profile-id id] :as params}]
-  (check-read-permissions! cfg profile-id id)
-  (get-file-summary cfg id))
-
-
 ;; --- COMMAND QUERY: get-file-libraries
 
 (def ^:private schema:get-file-libraries
@@ -766,61 +711,46 @@
     (get-team-recent-files conn team-id)))
 
 
-;; --- COMMAND QUERY: get-team-deleted-files
+;; --- COMMAND QUERY: get-file-summary
 
-(def sql:team-deleted-files
+(defn- get-file-summary
-  "WITH deleted_files AS (
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}]
-     SELECT f.id,
+  (check-read-permissions! conn profile-id id)
-            f.revn,
+  (let [team (teams/get-team conn
-            f.vern,
+                             :profile-id profile-id
-            f.project_id,
+                             :project-id project-id
-            f.created_at,
+                             :file-id id)
-            f.modified_at,
-            f.name,
-            f.is_shared,
-            f.deleted_at AS will_be_deleted_at,
-            ft.media_id AS thumbnail_id,
-            row_number() OVER w AS row_num,
-            p.team_id
-       FROM file AS f
-      INNER JOIN project AS p ON (p.id = f.project_id)
-       LEFT JOIN file_thumbnail AS ft on (ft.file_id = f.id
-                                          AND ft.revn = f.revn
-                                          AND ft.deleted_at is null)
-      WHERE p.team_id = ?
-        AND (p.deleted_at > ?::timestamptz OR
-             f.deleted_at > ?::timestamptz)
-     WINDOW w AS (PARTITION BY f.project_id
-                      ORDER BY f.modified_at DESC)
-      ORDER BY f.modified_at DESC
-   )
-   SELECT * FROM deleted_files")
 
-(defn get-team-deleted-files
+        file (get-file cfg id
-  [conn team-id]
+                       :project-id project-id
-  (let [now (ct/now)]
+                       :read-only? true)]
-    (db/exec! conn [sql:team-deleted-files team-id now now])))
 
-(def ^:private schema:get-team-deleted-files
+    (-> (cfeat/get-team-enabled-features cf/flags team)
-  [:map {:title "get-team-deleted-files"}
+        (cfeat/check-client-features! (:features params))
-   [:team-id ::sm/uuid]])
+        (cfeat/check-file-features! (:features file)))
+
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+      {:name             (:name file)
+       :components-count (count (ctkl/components-seq (:data file)))
+       :graphics-count   (count (get-in file [:data :media] []))
+       :colors-count     (count (get-in file [:data :colors] []))
+       :typography-count (count (get-in file [:data :typographies] []))})))
+
+(sv/defmethod ::get-file-summary
+  "Retrieve a file summary by its ID. Only authenticated users."
+  {::doc/added "1.20"
+   ::sm/params schema:get-file}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg get-file-summary (assoc params :profile-id profile-id)))
 
-(sv/defmethod ::get-team-deleted-files
-  {::doc/added "2.12"
-   ::sm/params schema:get-team-deleted-files}
-  [cfg {:keys [::rpc/profile-id team-id]}]
-  (db/run! cfg (fn [{:keys [::db/conn]}]
-                 (teams/check-read-permissions! conn profile-id team-id)
-                 (get-team-deleted-files conn team-id))))
 
 ;; --- COMMAND QUERY: get-file-info
 
-
 (defn- get-file-info
   [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
-  (db/get conn :file
+  (db/get* conn :file
-          {:id id}
+           {:id id}
-          {::sql/columns [:id :deleted-at]}))
+           {::sql/columns [:id]}))
 
 (sv/defmethod ::get-file-info
   "Retrieve minimal file info by its ID."
@@ -880,7 +810,7 @@
 
 ;; --- MUTATION COMMAND: set-file-shared
 
-(def ^:private sql:get-referenced-files
+(def sql:get-referenced-files
   "SELECT f.id
      FROM file_library_rel AS flr
     INNER JOIN file AS f ON (f.id = flr.file_id)
@@ -891,51 +821,56 @@
 (defn- absorb-library-by-file!
   [cfg ldata file-id]
 
-  (assert (db/connection-map? cfg)
+  (dm/assert!
-          "expected cfg with valid connection")
+   "expected cfg with valid connection"
+   (db/connection-map? cfg))
 
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)
             pmap/*tracked* (pmap/create-tracked)]
-    (let [file (-> (bfc/get-file cfg file-id
+    (let [file (-> (get-file cfg file-id
-                                 :include-deleted? true
+                             :include-deleted? true
-                                 :lock-for-update? true)
+                             :lock-for-update? true)
                    (update :data ctf/absorb-assets ldata))]
 
       (l/trc :hint "library absorbed"
              :library-id (str (:id ldata))
              :file-id (str file-id))
 
-      (bfc/update-file! cfg {:id file-id
+      (db/update! cfg :file
-                             :migrations (:migrations file)
+                  {:revn (inc (:revn file))
-                             :revn (inc (:revn file))
+                   :data (blob/encode (:data file))
-                             :data (:data file)
+                   :modified-at (ct/now)
-                             :modified-at (ct/now)
+                   :has-media-trimmed false}
-                             :has-media-trimmed false}))))
+                  {:id file-id})
+
+      (feat.fdata/persist-pointers! cfg file-id))))
 
 (defn- absorb-library
   "Find all files using a shared library, and absorb all library assets
   into the file local libraries"
-  [cfg {:keys [id data] :as library}]
+  [cfg {:keys [id] :as library}]
 
-  (assert (db/connection-map? cfg)
+  (dm/assert!
-          "expected cfg with valid connection")
+   "expected cfg with valid connection"
+   (db/connection-map? cfg))
 
-  (let [ids (->> (db/exec! cfg [sql:get-referenced-files id])
+  (let [ldata (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                 (sequence bfc/xf-map-id))]
+                (-> library :data (feat.fdata/process-pointers deref)))
+        ids   (->> (db/exec! cfg [sql:get-referenced-files id])
+                   (map :id))]
 
     (l/trc :hint "absorbing library"
            :library-id (str id)
            :files (str/join "," (map str ids)))
 
-    (run! (partial absorb-library-by-file! cfg data) ids)
+    (run! (partial absorb-library-by-file! cfg ldata) ids)
     library))
 
 (defn absorb-library!
   [{:keys [::db/conn] :as cfg} id]
-  (let [file (-> (bfc/get-file cfg id
+  (let [file (-> (get-file cfg id
-                               :realize? true
+                           :lock-for-update? true
-                               :lock-for-update? true
+                           :include-deleted? true)
-                               :include-deleted? true)
                  (check-version!))
 
         proj (db/get* conn :project {:id (:project-id file)}
@@ -1032,14 +967,7 @@
   (let [team (teams/get-team conn
                              :profile-id profile-id
                              :file-id id)
-        file (mark-file-deleted conn team id)
+        file (mark-file-deleted conn team id)]
-        msgbus (::mbus/msgbus cfg)]
-
-    (mbus/pub! msgbus
-               :topic id
-               :message {:type :file-deleted
-                         :file-id id
-                         :profile-id profile-id})
 
     (rph/with-meta (rph/wrap)
       {::audit/props {:project-id (:project-id file)
@@ -1072,7 +1000,6 @@
    [:library-id ::sm/uuid]])
 
 (sv/defmethod ::link-file-to-library
-  "Link a file to a library. Returns the recursive list of libraries used by that library"
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:link-file-to-library}
@@ -1086,8 +1013,7 @@
               (fn [{:keys [::db/conn]}]
                 (check-edition-permissions! conn profile-id file-id)
                 (check-edition-permissions! conn profile-id library-id)
-                (link-file-to-library conn params)
+                (link-file-to-library conn params))))
-                (bfc/get-libraries cfg [library-id]))))
 
 ;; --- MUTATION COMMAND: unlink-file-from-library
 
@@ -1161,138 +1087,3 @@
   (check-edition-permissions! conn profile-id file-id)
   (->  (ignore-sync conn params)
        (update :features db/decode-pgarray #{})))
-
-;; --- MUTATION COMMAND: delete-files-immediatelly
-
-(def ^:private sql:delete-team-files
-  "UPDATE file AS uf SET deleted_at = ?::timestamptz
-     FROM (
-        SELECT f.id
-          FROM file AS f
-          JOIN project AS p ON (p.id = f.project_id)
-          JOIN team AS t ON (t.id = p.team_id)
-         WHERE t.deleted_at IS NULL
-           AND t.id = ?
-           AND f.id = ANY(?::uuid[])
-     ) AS subquery
-    WHERE uf.id = subquery.id
-   RETURNING uf.id, uf.deleted_at;")
-
-(def ^:private schema:permanently-delete-team-files
-  [:map {:title "permanently-delete-team-files"}
-   [:team-id ::sm/uuid]
-   [:ids [::sm/set ::sm/uuid]]])
-
-(sv/defmethod ::permanently-delete-team-files
-  "Mark the specified files to be deleted immediatelly on the
-  specified team. The team-id on params will be used to filter and
-  check writable permissons on team."
-
-  {::doc/added "2.12"
-   ::sm/params schema:permanently-delete-team-files
-   ::db/transaction true}
-
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at team-id ids]}]
-  (teams/check-edition-permissions! conn profile-id team-id)
-
-  (reduce (fn [acc {:keys [id deleted-at]}]
-            (wrk/submit! {::db/conn conn
-                          ::wrk/task :delete-object
-                          ::wrk/params {:object :file
-                                        :deleted-at deleted-at
-                                        :id id}})
-            (conj acc id))
-          #{}
-          (db/plan conn [sql:delete-team-files request-at team-id
-                         (db/create-array conn "uuid" ids)])))
-
-;; --- MUTATION COMMAND: restore-files-immediatelly
-
-(def ^:private sql:resolve-editable-files
-  "SELECT f.id, f.project_id
-     FROM file AS f
-     JOIN project AS p ON (p.id = f.project_id)
-     JOIN team AS t ON (t.id = p.team_id)
-    WHERE t.deleted_at IS NULL
-      AND t.id = ?
-      AND f.id = ANY(?::uuid[])")
-
-(defn- restore-file
-  [conn file-id]
-  (db/update! conn :file
-              {:deleted-at nil
-               :has-media-trimmed false}
-              {:id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-media-object
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-change
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-data
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-tagged-object-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false}))
-
-(def ^:private sql:restore-projects
-  "UPDATE project SET deleted_at = null WHERE id = ANY(?::uuid[])")
-
-(defn- restore-projects
-  [conn project-ids]
-  (let [project-ids (db/create-array conn "uuid" project-ids)]
-    (->> (db/exec-one! conn [sql:restore-projects project-ids])
-         (db/get-update-count))))
-
-(defn- restore-deleted-team-files
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id ids]}]
-  (teams/check-edition-permissions! conn profile-id team-id)
-  (let [total-files
-        (count ids)
-
-        {:keys [files projects]}
-        (reduce (fn [result {:keys [id project-id]}]
-                  (let [index (-> result :files count)]
-                    (events/tap :progress {:file-id id :index index :total total-files})
-                    (restore-file conn id)
-
-                    (-> result
-                        (update :files conj id)
-                        (update :projects conj project-id))))
-
-                {:files #{} :projectes #{}}
-                (db/plan conn [sql:resolve-editable-files team-id
-                               (db/create-array conn "uuid" ids)]))]
-
-    (restore-projects conn projects)
-
-    files))
-
-(def ^:private schema:restore-deleted-team-files
-  [:map {:title "restore-deleted-team-files"}
-   [:team-id ::sm/uuid]
-   [:ids [::sm/set ::sm/uuid]]])
-
-(sv/defmethod ::restore-deleted-team-files
-  "Removes the deletion mark from the specified files (and respective
-  projects) on the specified team."
-  {::doc/added "2.12"
-   ::sse/stream? true
-   ::sm/params schema:restore-deleted-team-files}
-  [cfg params]
-  (sse/response #(db/tx-run! cfg restore-deleted-team-files params)))

backend/src/app/rpc/commands/files_create.clj

@@ -8,7 +8,6 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.features :as cfeat]
-   [app.common.files.migrations :as fmg]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.types.file :as ctf]
@@ -46,14 +45,12 @@
 
   (binding [pmap/*tracked* (pmap/create-tracked)
             cfeat/*current* features]
-
     (let [file (ctf/make-file {:id id
                                :project-id project-id
                                :name name
                                :revn revn
                                :is-shared is-shared
                                :features features
-                               :migrations fmg/available-migrations
                                :ignore-sync-until ignore-sync-until
                                :created-at modified-at
                                :deleted-at deleted-at}
@@ -69,7 +66,7 @@
                   {:modified-at (ct/now)}
                   {:id project-id})
 
-      (bfc/get-file cfg (:id file)))))
+      file)))
 
 (def ^:private schema:create-file
   [:map {:title "create-file"}
@@ -115,15 +112,14 @@
     ;; FIXME: IMPORTANT: this code can have race conditions, because
     ;; we have no locks for updating team so, creating two files
     ;; concurrently can lead to lost team features updating
+
     (when-let [features (-> features
                             (set/difference (:features team))
                             (set/difference cfeat/no-team-inheritable-features)
                             (not-empty))]
-      (let [features (-> features
+      (let [features (->> features
-                         (set/union (:features team))
+                          (set/union (:features team))
-                         (set/difference cfeat/no-team-inheritable-features)
+                          (db/create-array conn "text"))]
-                         (into-array))]
-
         (db/update! conn :team
                     {:features features}
                     {:id (:id team)}

backend/src/app/rpc/commands/files_snapshot.clj

@@ -8,20 +8,52 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
+   [app.common.files.migrations :as fmg]
+   [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.file-snapshots :as fsnap]
+   [app.features.fdata :as feat.fdata]
-   [app.features.logical-deletion :as ldel]
+   [app.features.file-migrations :refer [reset-migrations!]]
    [app.main :as-alias main]
    [app.msgbus :as mbus]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.quotes :as quotes]
-   [app.util.services :as sv]))
+   [app.storage :as sto]
+   [app.util.blob :as blob]
+   [app.util.services :as sv]
+   [cuerdas.core :as str]))
+
+(defn decode-row
+  [{:keys [migrations] :as row}]
+  (when row
+    (cond-> row
+      (some? migrations)
+      (assoc :migrations (db/decode-pgarray migrations)))))
+
+(def sql:get-file-snapshots
+  "WITH changes AS (
+      SELECT id, label, revn, created_at, created_by, profile_id, locked_by
+        FROM file_change
+       WHERE file_id = ?
+         AND data IS NOT NULL
+         AND (deleted_at IS NULL OR deleted_at > now())
+   ), versions AS (
+      (SELECT * FROM changes WHERE created_by = 'system' LIMIT 1000)
+      UNION ALL
+      (SELECT * FROM changes WHERE created_by != 'system' LIMIT 1000)
+   )
+   SELECT * FROM versions
+    ORDER BY created_at DESC;")
+
+(defn get-file-snapshots
+  [conn file-id]
+  (db/exec! conn [sql:get-file-snapshots file-id]))
 
 (def ^:private schema:get-file-snapshots
   [:map {:title "get-file-snapshots"}
@@ -33,7 +65,73 @@
   [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-read-permissions! conn profile-id file-id)
-                 (fsnap/get-visible-snapshots conn file-id))))
+                 (get-file-snapshots conn file-id))))
+
+(defn- generate-snapshot-label
+  []
+  (let [ts (-> (ct/now)
+               (ct/format-inst)
+               (str/replace #"[T:\.]" "-")
+               (str/rtrim "Z"))]
+    (str "snapshot-" ts)))
+
+(defn create-file-snapshot!
+  [cfg file & {:keys [label created-by deleted-at profile-id]
+               :or {deleted-at :default
+                    created-by :system}}]
+
+  (assert (#{:system :user :admin} created-by)
+          "expected valid keyword for created-by")
+
+  (let [created-by
+        (name created-by)
+
+        deleted-at
+        (cond
+          (= deleted-at :default)
+          (ct/plus (ct/now) (cf/get-deletion-delay))
+
+          (ct/inst? deleted-at)
+          deleted-at
+
+          :else
+          nil)
+
+        label
+        (or label (generate-snapshot-label))
+
+        snapshot-id
+        (uuid/next)
+
+        data
+        (blob/encode (:data file))
+
+        features
+        (into-array (:features file))
+
+        migrations
+        (into-array (:migrations file))]
+
+    (l/dbg :hint "creating file snapshot"
+           :file-id (str (:id file))
+           :id (str snapshot-id)
+           :label label)
+
+    (db/insert! cfg :file-change
+                {:id snapshot-id
+                 :revn (:revn file)
+                 :data data
+                 :version (:version file)
+                 :features features
+                 :migrations migrations
+                 :profile-id profile-id
+                 :file-id (:id file)
+                 :label label
+                 :deleted-at deleted-at
+                 :created-by created-by}
+                {::db/return-keys false})
+
+    {:id snapshot-id :label label}))
 
 (def ^:private schema:create-file-snapshot
   [:map
@@ -46,7 +144,7 @@
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id label]}]
   (files/check-edition-permissions! conn profile-id file-id)
-  (let [file    (bfc/get-file cfg file-id :realize? true)
+  (let [file    (bfc/get-file cfg file-id)
         project (db/get-by-id cfg :project (:project-id file))]
 
     (-> cfg
@@ -57,10 +155,96 @@
         (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
                        {::quotes/id ::quotes/snapshots-per-team}))
 
-    (fsnap/create! cfg file
+    (create-file-snapshot! cfg file
-                   {:label label
+                           {:label label
-                    :profile-id profile-id
+                            :profile-id profile-id
-                    :created-by "user"})))
+                            :created-by :user})))
+
+(defn restore-file-snapshot!
+  [{:keys [::db/conn ::mbus/msgbus] :as cfg} file-id snapshot-id]
+  (let [storage  (sto/resolve cfg {::db/reuse-conn true})
+        file     (files/get-minimal-file conn file-id {::db/for-update true})
+        vern     (rand-int Integer/MAX_VALUE)
+        snapshot (some->> (db/get* conn :file-change
+                                   {:file-id file-id
+                                    :id snapshot-id}
+                                   {::db/for-share true})
+                          (feat.fdata/resolve-file-data cfg)
+                          (decode-row))
+
+        ;; If snapshot has tracked applied migrations, we reuse them,
+        ;; if not we take a safest set of migrations as starting
+        ;; point. This is because, at the time of implementing
+        ;; snapshots, migrations were not taken into account so we
+        ;; need to make this backward compatible in some way.
+        file     (assoc file :migrations
+                        (or (:migrations snapshot)
+                            (fmg/generate-migrations-from-version 67)))]
+
+    (when-not snapshot
+      (ex/raise :type :not-found
+                :code :snapshot-not-found
+                :hint "unable to find snapshot with the provided label"
+                :snapshot-id snapshot-id
+                :file-id file-id))
+
+    (when-not (:data snapshot)
+      (ex/raise :type :validation
+                :code :snapshot-without-data
+                :hint "snapshot has no data"
+                :label (:label snapshot)
+                :file-id file-id))
+
+    (l/dbg :hint "restoring snapshot"
+           :file-id (str file-id)
+           :label (:label snapshot)
+           :snapshot-id (str (:id snapshot)))
+
+    ;; If the file was already offloaded, on restoring the snapshot we
+    ;; are going to replace the file data, so we need to touch the old
+    ;; referenced storage object and avoid possible leaks
+    (when (feat.fdata/offloaded? file)
+      (sto/touch-object! storage (:data-ref-id file)))
+
+    ;; In the same way, on reseting the file data, we need to restore
+    ;; the applied migrations on the moment of taking the snapshot
+    (reset-migrations! conn file)
+
+    (db/update! conn :file
+                {:data (:data snapshot)
+                 :revn (inc (:revn file))
+                 :vern vern
+                 :version (:version snapshot)
+                 :data-backend nil
+                 :data-ref-id nil
+                 :has-media-trimmed false
+                 :features (:features snapshot)}
+                {:id file-id})
+
+    ;; clean object thumbnails
+    (let [sql (str "update file_tagged_object_thumbnail "
+                   "   set deleted_at = now() "
+                   " where file_id=? returning media_id")
+          res (db/exec! conn [sql file-id])]
+      (doseq [media-id (into #{} (keep :media-id) res)]
+        (sto/touch-object! storage media-id)))
+
+    ;; clean file thumbnails
+    (let [sql (str "update file_thumbnail "
+                   "   set deleted_at = now() "
+                   " where file_id=? returning media_id")
+          res (db/exec! conn [sql file-id])]
+      (doseq [media-id (into #{} (keep :media-id) res)]
+        (sto/touch-object! storage media-id)))
+
+    ;; Send to the clients a notification to reload the file
+    (mbus/pub! msgbus
+               :topic (:id file)
+               :message {:type :file-restore
+                         :file-id (:id file)
+                         :vern vern})
+    {:id (:id snapshot)
+     :label (:label snapshot)}))
 
 (def ^:private schema:restore-file-snapshot
   [:map {:title "restore-file-snapshot"}
@@ -69,76 +253,88 @@
 
 (sv/defmethod ::restore-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:restore-file-snapshot
+   ::sm/params schema:restore-file-snapshot}
-   ::db/transaction true}
+  [cfg {:keys [::rpc/profile-id file-id id] :as params}]
-  [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id file-id id] :as params}]
+  (db/tx-run! cfg
-  (files/check-edition-permissions! conn profile-id file-id)
+              (fn [{:keys [::db/conn] :as cfg}]
-  (let [file  (bfc/get-file cfg file-id)
+                (files/check-edition-permissions! conn profile-id file-id)
-        team  (teams/get-team conn
+                (let [file (bfc/get-file cfg file-id)]
-                              :profile-id profile-id
+                  (create-file-snapshot! cfg file
-                              :file-id file-id)
+                                         {:profile-id profile-id
-        delay (ldel/get-deletion-delay team)]
+                                          :created-by :system})
-
+                  (restore-file-snapshot! cfg file-id id)))))
-    (fsnap/create! cfg file
-                   {:profile-id profile-id
-                    :deleted-at (ct/in-future delay)
-                    :created-by "system"})
-
-    (let [vern (fsnap/restore! cfg file-id id)]
-      ;; Send to the clients a notification to reload the file
-      (mbus/pub! msgbus
-                 :topic (:id file)
-                 :message {:type :file-restore
-                           :file-id (:id file)
-                           :vern vern})
-      nil)))
 
 (def ^:private schema:update-file-snapshot
   [:map {:title "update-file-snapshot"}
    [:id ::sm/uuid]
    [:label ::sm/text]])
 
+(defn- update-file-snapshot!
+  [conn snapshot-id label]
+  (-> (db/update! conn :file-change
+                  {:label label
+                   :created-by "user"
+                   :deleted-at nil}
+                  {:id snapshot-id}
+                  {::db/return-keys true})
+      (dissoc :data :features :migrations)))
+
+(defn- get-snapshot
+  "Get a minimal snapshot from database and lock for update"
+  [conn id]
+  (db/get conn :file-change
+          {:id id}
+          {::sql/columns [:id :file-id :created-by :deleted-at :profile-id :locked-by]
+           ::db/for-update true}))
+
 (sv/defmethod ::update-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:update-file-snapshot
+   ::sm/params schema:update-file-snapshot}
-   ::db/transaction true}
+  [cfg {:keys [::rpc/profile-id id label]}]
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id label]}]
+  (db/tx-run! cfg
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+              (fn [{:keys [::db/conn]}]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                (let [snapshot (get-snapshot conn id)]
-    (fsnap/update! conn (assoc snapshot :label label))))
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                  (update-file-snapshot! conn id label)))))
 
 (def ^:private schema:remove-file-snapshot
   [:map {:title "remove-file-snapshot"}
    [:id ::sm/uuid]])
 
+(defn- delete-file-snapshot!
+  [conn snapshot-id]
+  (db/update! conn :file-change
+              {:deleted-at (ct/now)}
+              {:id snapshot-id}
+              {::db/return-keys false})
+  nil)
+
 (sv/defmethod ::delete-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:remove-file-snapshot
+   ::sm/params schema:remove-file-snapshot}
-   ::db/transaction true}
+  [cfg {:keys [::rpc/profile-id id]}]
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (db/tx-run! cfg
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+              (fn [{:keys [::db/conn]}]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                (let [snapshot (get-snapshot conn id)]
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-    (when (not= (:created-by snapshot) "user")
+                  (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :system-snapshots-cant-be-deleted
+                              :code :system-snapshots-cant-be-deleted
-                :file-id (:file-id snapshot)
+                              :snapshot-id id
-                :snapshot-id id
+                              :profile-id profile-id))
-                :profile-id profile-id))
 
-    (when (and (some? (:locked-by snapshot))
+                  ;; Check if version is locked by someone else
-               (not= (:locked-by snapshot) profile-id))
+                  (when (and (:locked-by snapshot)
-      (ex/raise :type :validation
+                             (not= (:locked-by snapshot) profile-id))
-                :code :snapshot-is-locked
+                    (ex/raise :type :validation
-                :file-id (:file-id snapshot)
+                              :code :snapshot-is-locked
-                :snapshot-id id
+                              :hint "Cannot delete a locked version"
-                :profile-id profile-id))
+                              :snapshot-id id
+                              :profile-id profile-id
+                              :locked-by (:locked-by snapshot)))
 
-    (let [team  (teams/get-team conn
+                  (delete-file-snapshot! conn id)))))
-                                :profile-id profile-id
-                                :file-id (:file-id snapshot))
-          delay (ldel/get-deletion-delay team)]
-      (fsnap/delete! conn (assoc snapshot :deleted-at (ct/in-future delay))))))
 
 ;;; Lock/unlock version endpoints
 
@@ -146,75 +342,93 @@
   [:map {:title "lock-file-snapshot"}
    [:id ::sm/uuid]])
 
+(defn- lock-file-snapshot!
+  [conn snapshot-id profile-id]
+  (db/update! conn :file-change
+              {:locked-by profile-id}
+              {:id snapshot-id}
+              {::db/return-keys false})
+  nil)
+
 (sv/defmethod ::lock-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:lock-file-snapshot
+   ::sm/params schema:lock-file-snapshot}
-   ::db/transaction true}
+  [cfg {:keys [::rpc/profile-id id]}]
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (db/tx-run! cfg
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+              (fn [{:keys [::db/conn]}]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                (let [snapshot (get-snapshot conn id)]
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-    (when (not= (:created-by snapshot) "user")
+                  (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :system-snapshots-cant-be-locked
+                              :code :system-snapshots-cant-be-locked
-                :hint "Only user-created versions can be locked"
+                              :hint "Only user-created versions can be locked"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id))
+                              :profile-id profile-id))
 
-    ;; Only the creator can lock their own version
+                  ;; Only the creator can lock their own version
-    (when (not= (:profile-id snapshot) profile-id)
+                  (when (not= (:profile-id snapshot) profile-id)
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :only-creator-can-lock
+                              :code :only-creator-can-lock
-                :hint "Only the version creator can lock it"
+                              :hint "Only the version creator can lock it"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id
+                              :profile-id profile-id
-                :creator-id (:profile-id snapshot)))
+                              :creator-id (:profile-id snapshot)))
 
-    ;; Check if already locked
+                  ;; Check if already locked
-    (when (:locked-by snapshot)
+                  (when (:locked-by snapshot)
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :snapshot-already-locked
+                              :code :snapshot-already-locked
-                :hint "Version is already locked"
+                              :hint "Version is already locked"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id
+                              :profile-id profile-id
-                :locked-by (:locked-by snapshot)))
+                              :locked-by (:locked-by snapshot)))
 
-    (fsnap/lock-by! conn id profile-id)))
+                  (lock-file-snapshot! conn id profile-id)))))
 
 (def ^:private schema:unlock-file-snapshot
   [:map {:title "unlock-file-snapshot"}
    [:id ::sm/uuid]])
 
+(defn- unlock-file-snapshot!
+  [conn snapshot-id]
+  (db/update! conn :file-change
+              {:locked-by nil}
+              {:id snapshot-id}
+              {::db/return-keys false})
+  nil)
+
 (sv/defmethod ::unlock-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:unlock-file-snapshot
+   ::sm/params schema:unlock-file-snapshot}
-   ::db/transaction  true}
+  [cfg {:keys [::rpc/profile-id id]}]
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
+  (db/tx-run! cfg
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
+              (fn [{:keys [::db/conn]}]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                (let [snapshot (get-snapshot conn id)]
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-    (when (not= (:created-by snapshot) "user")
+                  (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :system-snapshots-cant-be-unlocked
+                              :code :system-snapshots-cant-be-unlocked
-                :hint "Only user-created versions can be unlocked"
+                              :hint "Only user-created versions can be unlocked"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id))
+                              :profile-id profile-id))
 
-    ;; Only the creator can unlock their own version
+                  ;; Only the creator can unlock their own version
-    (when (not= (:profile-id snapshot) profile-id)
+                  (when (not= (:profile-id snapshot) profile-id)
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :only-creator-can-unlock
+                              :code :only-creator-can-unlock
-                :hint "Only the version creator can unlock it"
+                              :hint "Only the version creator can unlock it"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id
+                              :profile-id profile-id
-                :creator-id (:profile-id snapshot)))
+                              :creator-id (:profile-id snapshot)))
 
-    ;; Check if not locked
+                  ;; Check if not locked
-    (when (not (:locked-by snapshot))
+                  (when (not (:locked-by snapshot))
-      (ex/raise :type :validation
+                    (ex/raise :type :validation
-                :code :snapshot-not-locked
+                              :code :snapshot-not-locked
-                :hint "Version is not locked"
+                              :hint "Version is not locked"
-                :snapshot-id id
+                              :snapshot-id id
-                :profile-id profile-id))
+                              :profile-id profile-id))
 
-    (fsnap/unlock! conn id)))
+                  (unlock-file-snapshot! conn id)))))

backend/src/app/rpc/commands/files_temp.clj

@@ -0,0 +1,161 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files-temp
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as cpc]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as sql]
+   [app.features.fdata :as fdata]
+   [app.loggers.audit :as audit]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
+   [app.rpc.commands.files-create :as files.create]
+   [app.rpc.commands.files-update :as-alias files.update]
+   [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
+   [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
+   [app.util.services :as sv]
+   [clojure.set :as set]))
+
+;; --- MUTATION COMMAND: create-temp-file
+
+(def ^:private schema:create-temp-file
+  [:map {:title "create-temp-file"}
+   [:name [:string {:max 250}]]
+   [:project-id ::sm/uuid]
+   [:id {:optional true} ::sm/uuid]
+   [:is-shared ::sm/boolean]
+   [:features ::cfeat/features]
+   [:create-page ::sm/boolean]])
+
+(sv/defmethod ::create-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:create-temp-file
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
+  (projects/check-edition-permissions! conn profile-id project-id)
+  (let [team (teams/get-team conn :profile-id profile-id :project-id project-id)
+        ;; When we create files, we only need to respect the team
+        ;; features, because some features can be enabled
+        ;; globally, but the team is still not migrated properly.
+        input-features
+        (:features params #{})
+
+        ;; If the imported project doesn't contain v2 we need to remove it
+        team-features
+        (cond-> (cfeat/get-team-enabled-features cf/flags team)
+          (not (contains? input-features "components/v2"))
+          (disj "components/v2"))
+
+        ;; We also include all no migration features declared by
+        ;; client; that enables the ability to enable a runtime
+        ;; feature on frontend and make it permanent on file
+        features
+        (-> input-features
+            (set/intersection cfeat/no-migration-features)
+            (set/union team-features))
+
+        params
+        (-> params
+            (assoc :profile-id profile-id)
+            (assoc :deleted-at (ct/in-future {:days 1}))
+            (assoc :features features))]
+
+    (files.create/create-file cfg params)))
+
+;; --- MUTATION COMMAND: update-temp-file
+
+
+(def ^:private schema:update-temp-file
+  [:map {:title "update-temp-file"}
+   [:changes [:vector ::cpc/change]]
+   [:revn [::sm/int {:min 0}]]
+   [:session-id ::sm/uuid]
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::update-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:update-temp-file}
+  [cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (db/insert! conn :file-change
+                                {:id (uuid/next)
+                                 :session-id session-id
+                                 :profile-id profile-id
+                                 :created-at (ct/now)
+                                 :file-id id
+                                 :revn revn
+                                 :data nil
+                                 :changes (blob/encode changes)})
+                    (rph/with-meta (rph/wrap nil)
+                      {::audit/replace-props {:file-id id
+                                              :revn revn}}))))
+
+;; --- MUTATION COMMAND: persist-temp-file
+
+(defn persist-temp-file
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
+  (let [file (files/get-file cfg id
+                             :migrate? false
+                             :lock-for-update? true)]
+
+    (when (nil? (:deleted-at file))
+      (ex/raise :type :validation
+                :code :cant-persist-already-persisted-file))
+
+    (let [changes (->> (db/cursor conn
+                                  (sql/select :file-change {:file-id id}
+                                              {:order-by [[:revn :asc]]})
+                                  {:chunk-size 10})
+                       (sequence (mapcat (comp blob/decode :changes))))
+
+          file    (update file :data cpc/process-changes changes)
+
+          file    (if (contains? (:features file) "fdata/objects-map")
+                    (fdata/enable-objects-map file)
+                    file)
+
+          file    (if (contains? (:features file) "fdata/pointer-map")
+                    (binding [pmap/*tracked* (pmap/create-tracked)]
+                      (let [file (fdata/enable-pointer-map file)]
+                        (fdata/persist-pointers! cfg id)
+                        file))
+                    file)]
+
+      ;; Delete changes from the changes history
+      (db/delete! conn :file-change {:file-id id})
+
+      (db/update! conn :file
+                  {:deleted-at nil
+                   :revn 1
+                   :data (blob/encode (:data file))}
+                  {:id id})
+      nil)))
+
+(def ^:private schema:persist-temp-file
+  [:map {:title "persist-temp-file"}
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::persist-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:persist-temp-file}
+  [cfg {:keys [::rpc/profile-id id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (persist-temp-file cfg params))))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.files-thumbnails
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
@@ -96,7 +95,7 @@
 ;; loading all pages into memory for find the frame set for thumbnail.
 
 (defn get-file-data-for-thumbnail
-  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file} strip-frames-with-thumbnails]
+  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
   (letfn [;; function responsible on finding the frame marked to be
           ;; used as thumbnail; the returned frame always have
           ;; the :page-id set to the page that it belongs.
@@ -173,7 +172,7 @@
 
         ;; Assoc the available thumbnails and prune not visible shapes
         ;; for avoid transfer unnecessary data.
-        strip-frames-with-thumbnails
+        :always
         (update :objects assoc-thumbnails page-id thumbs)))))
 
 (def ^:private
@@ -186,8 +185,7 @@
   [:map {:title "PartialFile"}
    [:id ::sm/uuid]
    [:revn {:min 0} ::sm/int]
-   [:page [:map-of :keyword ::sm/any]]
+   [:page :any]])
-   [:strip-frames-with-thumbnails {:optional true} ::sm/boolean]])
 
 (sv/defmethod ::get-file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
@@ -196,7 +194,7 @@
    ::doc/module :files
    ::sm/params schema:get-file-data-for-thumbnail
    ::sm/result schema:partial-file}
-  [cfg {:keys [::rpc/profile-id file-id strip-frames-with-thumbnails] :as params}]
+  [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
                  (files/check-read-permissions! conn profile-id file-id)
 
@@ -204,20 +202,16 @@
                                             :profile-id profile-id
                                             :file-id file-id)
 
-                       file (bfc/get-file cfg file-id
+                       file (files/get-file cfg file-id
-                                          :realize? true
+                                            :preload-pointers? true
-                                          :read-only? true)
+                                            :read-only? true)]
-
-                       strip-frames-with-thumbnails
-                       (or (nil? strip-frames-with-thumbnails) ;; if not present, default to true
-                           (true? strip-frames-with-thumbnails))]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
                        (cfeat/check-file-features! (:features file)))
 
                    {:file-id file-id
                     :revn (:revn file)
-                    :page (get-file-data-for-thumbnail cfg file strip-frames-with-thumbnails)}))))
+                    :page (get-file-data-for-thumbnail cfg file)}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
@@ -277,7 +271,7 @@
   [:map {:title "create-file-object-thumbnail"}
    [:file-id ::sm/uuid]
    [:object-id [:string {:max 250}]]
-   [:media media/schema:upload]
+   [:media ::media/upload]
    [:tag {:optional true} [:string {:max 50}]]])
 
 (sv/defmethod ::create-file-object-thumbnail
@@ -345,7 +339,6 @@
         data  (-> (sto/content path)
                   (sto/wrap-with-hash hash))
         tnow  (ct/now)
-
         media (sto/put-object! storage
                                {::sto/content data
                                 ::sto/deduplicate? true
@@ -388,7 +381,7 @@
   [:map {:title "create-file-thumbnail"}
    [:file-id ::sm/uuid]
    [:revn ::sm/int]
-   [:media media/schema:upload]])
+   [:media ::media/upload]])
 
 (sv/defmethod ::create-file-thumbnail
   "Creates or updates the file thumbnail. Mainly used for paint the

backend/src/app/rpc/commands/files_update.clj

@@ -19,25 +19,27 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as fdata]
+   [app.features.fdata :as feat.fdata]
-   [app.features.file-snapshots :as fsnap]
+   [app.features.file-migrations :as feat.fmigr]
    [app.features.logical-deletion :as ldel]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
    [app.metrics :as mtx]
    [app.msgbus :as mbus]
-   [app.redis :as rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as climit]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [clojure.set :as set]))
+   [app.worker :as wrk]
+   [clojure.set :as set]
+   [promesa.exec :as px]))
 
 (declare ^:private get-lagged-changes)
 (declare ^:private send-notifications!)
@@ -45,7 +47,6 @@
 (declare ^:private update-file*)
 (declare ^:private process-changes-and-validate)
 (declare ^:private take-snapshot?)
-(declare ^:private invalidate-caches!)
 
 ;; PUBLIC API; intended to be used outside of this module
 (declare update-file!)
@@ -63,10 +64,10 @@
    [:revn {:min 0} ::sm/int]
    [:vern {:min 0} ::sm/int]
    [:features {:optional true} ::cfeat/features]
-   [:changes {:optional true} [:vector cpc/schema:change]]
+   [:changes {:optional true} [:vector ::cpc/change]]
    [:changes-with-metadata {:optional true}
     [:vector [:map
-              [:changes [:vector cpc/schema:change]]
+              [:changes [:vector ::cpc/change]]
               [:hint-origin {:optional true} :keyword]
               [:hint-events {:optional true} [:vector [:string {:max 250}]]]]]]
    [:skip-validate {:optional true} ::sm/boolean]])
@@ -75,7 +76,7 @@
   schema:update-file-result
   [:vector {:title "update-file-result"}
    [:map
-    [:changes [:vector cpc/schema:change]]
+    [:changes [:vector ::cpc/change]]
     [:file-id ::sm/uuid]
     [:id ::sm/uuid]
     [:revn {:min 0} ::sm/int]
@@ -128,78 +129,77 @@
    ::sm/params schema:update-file
    ::sm/result schema:update-file-result
    ::doc/module :files
-   ::doc/added "1.17"
+   ::doc/added "1.17"}
-   ::db/transaction true}
+  [{:keys [::mtx/metrics] :as cfg}
-  [{:keys [::mtx/metrics ::db/conn] :as cfg}
    {:keys [::rpc/profile-id id changes changes-with-metadata] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (db/xact-lock! conn id)
 
-  (files/check-edition-permissions! conn profile-id id)
+                    (let [file     (get-file conn id)
-  (db/xact-lock! conn id)
+                          team     (teams/get-team conn
+                                                   :profile-id profile-id
+                                                   :team-id (:team-id file))
 
-  (let [file     (get-file cfg id)
+                          features (-> (cfeat/get-team-enabled-features cf/flags team)
-        team     (teams/get-team conn
+                                       (cfeat/check-client-features! (:features params))
-                                 :profile-id profile-id
+                                       (cfeat/check-file-features! (:features file)))
-                                 :team-id (:team-id file))
 
-        features (-> (cfeat/get-team-enabled-features cf/flags team)
+                          changes  (if changes-with-metadata
-                     (cfeat/check-client-features! (:features params))
+                                     (->> changes-with-metadata (mapcat :changes) vec)
-                     (cfeat/check-file-features! (:features file)))
+                                     (vec changes))
 
-        changes  (if changes-with-metadata
+                          params   (-> params
-                   (->> changes-with-metadata (mapcat :changes) vec)
+                                       (assoc :profile-id profile-id)
-                   (vec changes))
+                                       (assoc :features (set/difference features cfeat/frontend-only-features))
+                                       (assoc :team team)
+                                       (assoc :file file)
+                                       (assoc :changes changes))
 
-        params   (-> params
+                          cfg      (assoc cfg ::timestamp (ct/now))
-                     (assoc :profile-id profile-id)
-                     (assoc :features (set/difference features cfeat/frontend-only-features))
-                     (assoc :team team)
-                     (assoc :file file)
-                     (assoc :changes changes))
 
-        cfg      (assoc cfg ::timestamp (ct/now))
+                          tpoint   (ct/tpoint)]
-
-        tpoint   (ct/tpoint)]
-
-    (when (not= (:vern params)
-                (:vern file))
-      (ex/raise :type :validation
-                :code :vern-conflict
-                :hint "A different version has been restored for the file."
-                :context {:incoming-revn (:revn params)
-                          :stored-revn (:revn file)}))
-
-    (when (> (:revn params)
-             (:revn file))
-      (ex/raise :type :validation
-                :code :revn-conflict
-                :hint "The incoming revision number is greater that stored version."
-                :context {:incoming-revn (:revn params)
-                          :stored-revn (:revn file)}))
-
-    ;; When newly computed features does not match exactly with the
-    ;; features defined on team row, we update it
-    (when-let [features (-> features
-                            (set/difference (:features team))
-                            (set/difference cfeat/no-team-inheritable-features)
-                            (not-empty))]
-      (let [features (-> features
-                         (set/union (:features team))
-                         (set/difference cfeat/no-team-inheritable-features)
-                         (into-array))]
-        (db/update! conn :team
-                    {:features features}
-                    {:id (:id team)}
-                    {::db/return-keys false})))
 
 
-    (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+                      (when (not= (:vern params)
+                                  (:vern file))
+                        (ex/raise :type :validation
+                                  :code :vern-conflict
+                                  :hint "A different version has been restored for the file."
+                                  :context {:incoming-revn (:revn params)
+                                            :stored-revn (:revn file)}))
 
-    (binding [l/*context* (some-> (meta params)
+                      (when (> (:revn params)
-                                  (get :app.http/request)
+                               (:revn file))
-                                  (errors/request->context))]
+                        (ex/raise :type :validation
-      (-> (update-file* cfg params)
+                                  :code :revn-conflict
-          (rph/with-defer #(let [elapsed (tpoint)]
+                                  :hint "The incoming revision number is greater that stored version."
-                             (l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))
+                                  :context {:incoming-revn (:revn params)
+                                            :stored-revn (:revn file)}))
+
+                      ;; When newly computed features does not match exactly with
+                      ;; the features defined on team row, we update it
+                      (when-let [features (-> features
+                                              (set/difference (:features team))
+                                              (set/difference cfeat/no-team-inheritable-features)
+                                              (not-empty))]
+                        (let [features (->> features
+                                            (set/union (:features team))
+                                            (db/create-array conn "text"))]
+                          (db/update! conn :team
+                                      {:features features}
+                                      {:id (:id team)}
+                                      {::db/return-keys false})))
+
+
+                      (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
+                      (binding [l/*context* (some-> (meta params)
+                                                    (get :app.http/request)
+                                                    (errors/request->context))]
+                        (-> (update-file* cfg params)
+                            (rph/with-defer #(let [elapsed (tpoint)]
+                                               (l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))))
 
 (defn- update-file*
   "Internal function, part of the update-file process, that encapsulates
@@ -209,41 +209,31 @@
   Follow the inner implementation to `update-file-data!` function.
 
   Only intended for internal use on this module."
-  [{:keys [::db/conn ::timestamp] :as cfg}
+  [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
    {:keys [profile-id file team features changes session-id skip-validate] :as params}]
 
-  (binding [pmap/*tracked* (pmap/create-tracked)
+  (let [;; Retrieve the file data
-            pmap/*load-fn* (partial fdata/load-pointer cfg (:id file))]
+        file  (feat.fmigr/resolve-applied-migrations cfg file)
+        file  (feat.fdata/resolve-file-data cfg file)
+        file  (assoc file :features
+                     (-> features
+                         (set/difference cfeat/frontend-only-features)
+                         (set/union (:features file))))]
 
-    (let [file (assoc file :features
+    ;; We create a new lexycal scope for clearly delimit the result of
-                      (-> features
+    ;; executing this update file operation and all its side effects
-                          (set/difference cfeat/frontend-only-features)
+    (let [file (px/invoke! executor
-                          (set/union (:features file))))
+                           (fn []
+                             ;; Process the file data on separated thread for avoid to do
+                             ;; the CPU intensive operation on vthread.
+                             (binding [cfeat/*current*  features
+                                       cfeat/*previous* (:features file)]
+                               (update-file-data! cfg file
+                                                  process-changes-and-validate
+                                                  changes skip-validate))))]
 
-          ;; We need to preserve the original revn for the response
+      (feat.fmigr/upsert-migrations! conn file)
-          revn
+      (persist-file! cfg file)
-          (get file :revn)
-
-          file
-          (binding [cfeat/*current*  features
-                    cfeat/*previous* (:features file)]
-            (update-file-data! cfg file
-                               process-changes-and-validate
-                               changes skip-validate))
-
-          deleted-at
-          (ct/plus timestamp (ct/duration {:hours 1}))]
-
-      (when-let [file (::snapshot file)]
-        (let [deleted-at (ct/plus timestamp (ldel/get-deletion-delay team))
-              label      (str "internal/snapshot/" revn)]
-
-          (fsnap/create! cfg file
-                         {:label label
-                          :created-by "system"
-                          :deleted-at deleted-at
-                          :profile-id profile-id
-                          :session-id session-id})))
 
       ;; Insert change (xlog) with deleted_at in a future data for
       ;; make them automatically eleggible for GC once they expires
@@ -253,71 +243,87 @@
                    :profile-id profile-id
                    :created-at timestamp
                    :updated-at timestamp
-                   :deleted-at deleted-at
+                   :deleted-at (if (::snapshot-data file)
+                                 (ct/plus timestamp (ldel/get-deletion-delay team))
+                                 (ct/plus timestamp (ct/duration {:hours 1})))
                    :file-id (:id file)
                    :revn (:revn file)
                    :version (:version file)
-                   :features (into-array (:features file))
+                   :features (:features file)
+                   :label (::snapshot-label file)
+                   :data (::snapshot-data file)
                    :changes (blob/encode changes)}
                   {::db/return-keys false})
 
-      (persist-file! cfg file)
-
-      (when (contains? cf/flags :redis-cache)
-        (invalidate-caches! cfg file))
-
       ;; Send asynchronous notifications
-      (send-notifications! cfg params file)
+      (send-notifications! cfg params file))
 
-      (with-meta {:revn revn :lagged (get-lagged-changes conn params)}
+    (when (feat.fdata/offloaded? file)
-        {::audit/replace-props
+      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-         {:id         (:id file)
+        (some->> (:data-ref-id file) (sto/touch-object! storage))))
-          :name       (:name file)
+
-          :features   (:features file)
+    (let [response {:revn (:revn file)
-          :project-id (:project-id file)
+                    :lagged (get-lagged-changes conn params)}]
-          :team-id    (:team-id file)}}))))
+      (vary-meta response assoc ::audit/replace-props
+                 {:id         (:id file)
+                  :name       (:name file)
+                  :features   (:features file)
+                  :project-id (:project-id file)
+                  :team-id    (:team-id file)}))))
+
+(defn update-file!
+  "A public api that allows apply a transformation to a file with all context setup."
+  [{:keys [::db/conn] :as cfg} file-id update-fn & args]
+  (let [file (get-file cfg file-id)
+        file (apply update-file-data! cfg file update-fn args)]
+    (feat.fmigr/upsert-migrations! conn file)
+    (persist-file! cfg file)))
+
+(def ^:private sql:get-file
+  "SELECT f.*, p.team_id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR
+           f.deleted_at > now())
+      FOR KEY SHARE")
 
 (defn get-file
   "Get not-decoded file, only decodes the features set."
-  [cfg id]
+  [conn id]
-  (bfc/get-file cfg id :decode? false :lock-for-share? true))
+  (let [file (db/exec-one! conn [sql:get-file id])]
+    (when-not file
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint (format "file with id '%s' does not exists" id)))
+    (update file :features db/decode-pgarray #{})))
 
 (defn persist-file!
   "Function responsible of persisting already encoded file. Should be
   used together with `get-file` and `update-file-data!`.
 
   It also updates the project modified-at attr."
-  [{:keys [::db/conn ::timestamp] :as cfg} file]
+  [{:keys [::db/conn ::timestamp]} file]
   (let [;; The timestamp can be nil because this function is also
         ;; intended to be used outside of this module
-        modified-at
+        modified-at (or timestamp (ct/now))]
-        (or timestamp (ct/now))
-
-        file
-        (-> file
-            (dissoc ::snapshot)
-            (assoc :modified-at modified-at)
-            (assoc :has-media-trimmed false))]
 
     (db/update! conn :project
                 {:modified-at modified-at}
                 {:id (:project-id file)}
                 {::db/return-keys false})
 
-    (bfc/update-file! cfg file)))
+    (db/update! conn :file
-
+                {:revn (:revn file)
-(defn- invalidate-caches!
+                 :data (:data file)
-  [cfg {:keys [id] :as file}]
+                 :version (:version file)
-  (rds/run! cfg (fn [{:keys [::rds/conn]}]
+                 :features (:features file)
-                  (let [key (str files/file-summary-cache-key-prefix id)]
+                 :data-backend nil
-                    (rds/del conn key)))))
+                 :data-ref-id nil
-
+                 :modified-at modified-at
-(defn- attach-snapshot
+                 :has-media-trimmed false}
-  "Attach snapshot data to the file. This should be called before the
+                {:id (:id file)}
-  upcoming file operations are applied to the file."
+                {::db/return-keys false})))
-  [cfg migrated? file]
-  (let [snapshot (if migrated? file (fdata/realize cfg file))]
-    (assoc file ::snapshot snapshot)))
 
 (defn- update-file-data!
   "Perform a file data transformation in with all update context setup.
@@ -329,35 +335,52 @@
   fdata/pointer-map modified fragments."
 
   [cfg {:keys [id] :as file} update-fn & args]
-  (let [file (update file :data (fn [data]
+  (binding [pmap/*tracked* (pmap/create-tracked)
-                                  (-> data
+            pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                                      (blob/decode)
+    (let [file (update file :data (fn [data]
-                                      (assoc :id id))))
+                                    (-> data
-        libs (delay (bfc/get-resolved-file-libraries cfg file))
+                                        (blob/decode)
+                                        (assoc :id (:id file)))))
+          libs (delay (bfc/get-resolved-file-libraries cfg file))
 
-        need-migration?
+          ;; For avoid unnecesary overhead of creating multiple pointers
-        (fmg/need-migration? file)
+          ;; and handly internally with objects map in their worst
+          ;; case (when probably all shapes and all pointers will be
+          ;; readed in any case), we just realize/resolve them before
+          ;; applying the migration to the file
+          file (if (fmg/need-migration? file)
+                 (-> file
+                     (update :data feat.fdata/process-pointers deref)
+                     (update :data feat.fdata/process-objects (partial into {}))
+                     (fmg/migrate-file libs))
+                 file)
 
-        take-snapshot?
+          file (apply update-fn cfg file args)
-        (take-snapshot? file)
 
-        ;; For avoid unnecesary overhead of creating multiple
+          ;; TODO: reuse operations if file is migrated
-        ;; pointers and handly internally with objects map in their
+          ;; TODO: move encoding to a separated thread
-        ;; worst case (when probably all shapes and all pointers
+          file (if (take-snapshot? file)
-        ;; will be readed in any case), we just realize/resolve them
+                 (let [tpoint   (ct/tpoint)
-        ;; before applying the migration to the file
+                       snapshot (-> (:data file)
-        file
+                                    (feat.fdata/process-pointers deref)
-        (cond-> file
+                                    (feat.fdata/process-objects (partial into {}))
-          ;; need-migration?
+                                    (blob/encode))
-          ;; (->> (fdata/realize cfg))
+                       elapsed (tpoint)
+                       label   (str "internal/snapshot/" (:revn file))]
 
-          need-migration?
+                   (l/trc :hint "take snapshot"
-          (fmg/migrate-file libs)
+                          :file-id (str (:id file))
+                          :revn (:revn file)
+                          :label label
+                          :elapsed (ct/format-duration elapsed))
 
-          take-snapshot?
+                   (-> file
-          (->> (attach-snapshot cfg need-migration?)))]
+                       (assoc ::snapshot-data snapshot)
+                       (assoc ::snapshot-label label)))
+                 file)]
+
+      (bfc/encode-file cfg file))))
 
-    (apply update-fn cfg file args)))
 
 (defn- soft-validate-file-schema!
   [file]
@@ -385,6 +408,7 @@
                    (not skip-validate))
           (bfc/get-resolved-file-libraries cfg file))
 
+
         ;; The main purpose of this atom is provide a contextual state
         ;; for the changes subsystem where optionally some hints can
         ;; be provided for the changes processing. Right now we are
@@ -446,9 +470,8 @@
 (defn- get-lagged-changes
   [conn {:keys [id revn] :as params}]
   (->> (db/exec! conn [sql:lagged-changes id revn])
-       (filter :changes)
+       (map files/decode-row)
-       (mapv (fn [row]
+       (vec)))
-               (update row :changes blob/decode)))))
 
 (defn- send-notifications!
   [cfg {:keys [team changes session-id] :as params} file]

backend/src/app/rpc/commands/fonts.clj

@@ -26,7 +26,9 @@
    [app.rpc.helpers :as rph]
    [app.rpc.quotes :as quotes]
    [app.storage :as sto]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 (def valid-weight #{100 200 300 400 500 600 700 800 900 950})
 (def valid-style #{"normal" "italic"})
@@ -35,13 +37,14 @@
 
 (def ^:private
   schema:get-font-variants
-  [:and
+  [:schema {:title "get-font-variants"}
-   [:map {:title "get-font-variants"}
+   [:and
-    [:team-id {:optional true} ::sm/uuid]
+    [:map
-    [:file-id {:optional true} ::sm/uuid]
+     [:team-id {:optional true} ::sm/uuid]
-    [:project-id {:optional true} ::sm/uuid]
+     [:file-id {:optional true} ::sm/uuid]
-    [:share-id {:optional true} ::sm/uuid]]
+     [:project-id {:optional true} ::sm/uuid]
-   [::sm/contains-any #{:team-id :file-id :project-id}]])
+     [:share-id {:optional true} ::sm/uuid]]
+    [::sm/contains-any #{:team-id :file-id :project-id}]]])
 
 (sv/defmethod ::get-font-variants
   {::doc/added "1.18"
@@ -103,7 +106,7 @@
                 (create-font-variant cfg (assoc params :profile-id profile-id)))))
 
 (defn create-font-variant
-  [{:keys [::sto/storage ::db/conn]} {:keys [data] :as params}]
+  [{:keys [::sto/storage ::db/conn ::wrk/executor]} {:keys [data] :as params}]
   (letfn [(generate-missing! [data]
             (let [data (media/run {:cmd :generate-fonts :input data})]
               (when (and (not (contains? data "font/otf"))
@@ -155,7 +158,7 @@
                          :otf-file-id (:id otf)
                          :ttf-file-id (:id ttf)}))]
 
-    (let [data   (generate-missing! data)
+    (let [data   (px/invoke! executor (partial generate-missing! data))
           assets (persist-fonts-files! data)
           result (insert-font-variant! assets)]
       (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))

backend/src/app/rpc/commands/ldap.clj

@@ -38,7 +38,7 @@
    ::doc/added "1.15"
    ::doc/module :auth
    ::sm/params schema:login-with-ldap}
-  [{:keys [::ldap/provider] :as cfg} params]
+  [{:keys [::setup/props ::ldap/provider] :as cfg} params]
   (when-not provider
     (ex/raise :type :restriction
               :code :ldap-not-initialized
@@ -60,18 +60,18 @@
         ;; user comes from team-invitation process; in this case,
         ;; regenerate token and send back to the user a new invitation
         ;; token (and mark current session as logged).
-        (let [claims (tokens/verify cfg {:token token :iss :team-invitation})
+        (let [claims (tokens/verify props {:token token :iss :team-invitation})
               claims (assoc claims
                             :member-id  (:id profile)
                             :member-email (:email profile))
-              token  (tokens/generate cfg claims)]
+              token  (tokens/generate props claims)]
           (-> {:invitation-token token}
-              (rph/with-transform (session/create-fn cfg profile))
+              (rph/with-transform (session/create-fn cfg (:id profile)))
               (rph/with-meta {::audit/props (:props profile)
                               ::audit/profile-id (:id profile)})))
 
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg profile))
+            (rph/with-transform (session/create-fn cfg (:id profile)))
             (rph/with-meta {::audit/props (:props profile)
                             ::audit/profile-id (:id profile)}))))))
 
@@ -83,6 +83,6 @@
                              (profile/clean-email)
                              (profile/get-profile-by-email conn))
                     (->> (assoc info :is-active true :is-demo false)
-                         (auth/create-profile cfg)
+                         (auth/create-profile! conn)
-                         (auth/create-profile-rels conn)
+                         (auth/create-profile-rels! conn)
                          (profile/strip-private-attrs))))))

backend/src/app/rpc/commands/management.clj

@@ -28,14 +28,16 @@
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
    [app.storage.tmp :as tmp]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 ;; --- COMMAND: Duplicate File
 
 (defn duplicate-file
   [{:keys [::db/conn ::bfc/timestamp] :as cfg} {:keys [profile-id file-id name reset-shared-flag] :as params}]
   (let [;; We don't touch the original file on duplication
-        file       (bfc/get-file cfg file-id :realize? true)
+        file       (bfc/get-file cfg file-id)
         project-id (:project-id file)
         file       (-> file
                        (update :id bfc/lookup-index)
@@ -311,14 +313,15 @@
 
     ;; Update the modification date of the all affected projects
     ;; ensuring that the destination project is the most recent one.
-    (loop [project-ids (into (list project-id) source)
+    (doseq [project-id (into (list project-id) source)]
-           modified-at (ct/now)]
+
-      (when-let [project-id (first project-ids)]
+      ;; NOTE: as this is executed on virtual thread, sleeping does
-        (db/update! conn :project
+      ;; not causes major issues, and allows an easy way to set a
-                    {:modified-at modified-at}
+      ;; trully different modification date to each file.
-                    {:id project-id})
+      (px/sleep 10)
-        (recur (rest project-ids)
+      (db/update! conn :project
-               (ct/plus modified-at 10))))
+                  {:modified-at (ct/now)}
+                  {:id project-id}))
 
     nil))
 
@@ -393,7 +396,12 @@
 ;; --- COMMAND: Clone Template
 
 (defn clone-template
-  [{:keys [::db/pool] :as cfg} {:keys [project-id profile-id] :as params} template]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [project-id profile-id] :as params} template]
+
+  ;; NOTE: the importation process performs some operations
+  ;; that are not very friendly with virtual threads, and for
+  ;; avoid unexpected blocking of other concurrent operations
+  ;; we dispatch that operation to a dedicated executor.
   (let [template (tmp/tempfile-from template
                                     :prefix "penpot.template."
                                     :suffix ""
@@ -411,8 +419,8 @@
                      (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
 
         result   (if (= format :binfile-v3)
-                   (bf.v3/import-files! cfg)
+                   (px/invoke! executor (partial bf.v3/import-files! cfg))
-                   (bf.v1/import-files! cfg))]
+                   (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
     (db/tx-run! cfg
                 (fn [{:keys [::db/conn] :as cfg}]

backend/src/app/rpc/commands/media.clj

@@ -7,10 +7,14 @@
 (ns app.rpc.commands.media
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
+   [app.http.client :as http]
    [app.loggers.audit :as-alias audit]
    [app.media :as media]
    [app.rpc :as-alias rpc]
@@ -18,7 +22,15 @@
    [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
    [app.storage :as sto]
-   [app.util.services :as sv]))
+   [app.storage.tmp :as tmp]
+   [app.util.services :as sv]
+   [app.worker :as-alias wrk]
+   [cuerdas.core :as str]
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
+
+(def default-max-file-size
+  (* 1024 1024 10)) ; 10 MiB
 
 (def thumbnail-options
   {:width 100
@@ -36,7 +48,7 @@
    [:file-id ::sm/uuid]
    [:is-local ::sm/boolean]
    [:name [:string {:max 250}]]
-   [:content media/schema:upload]])
+   [:content ::media/upload]])
 
 (sv/defmethod ::upload-file-media-object
   {::doc/added "1.17"
@@ -141,9 +153,9 @@
       (assoc ::image (process-main-image info)))))
 
 (defn- create-file-media-object
-  [{:keys [::sto/storage ::db/conn] :as cfg}
+  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
    {:keys [id file-id is-local name content]}]
-  (let [result (process-image content)
+  (let [result (px/invoke! executor (partial process-image content))
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]
@@ -187,12 +199,56 @@
 
     mobj))
 
+(defn download-image
+  [{:keys [::http/client]} uri]
+  (letfn [(parse-and-validate [{:keys [headers] :as response}]
+            (let [size     (some-> (get headers "content-length") d/parse-integer)
+                  mtype    (get headers "content-type")
+                  format   (cm/mtype->format mtype)
+                  max-size (cf/get :media-max-file-size default-max-file-size)]
+
+              (when-not size
+                (ex/raise :type :validation
+                          :code :unknown-size
+                          :hint "seems like the url points to resource with unknown size"))
+
+              (when (> size max-size)
+                (ex/raise :type :validation
+                          :code :file-too-large
+                          :hint (str/ffmt "the file size % is greater than the maximum %"
+                                          size
+                                          default-max-file-size)))
+
+              (when (nil? format)
+                (ex/raise :type :validation
+                          :code :media-type-not-allowed
+                          :hint "seems like the url points to an invalid media object"))
+
+              {:size size :mtype mtype :format format}))]
+
+    (let [{:keys [body] :as response} (http/req! client
+                                                 {:method :get :uri uri}
+                                                 {:response-type :input-stream :sync? true})
+          {:keys [size mtype]} (parse-and-validate response)
+          path    (tmp/tempfile :prefix "penpot.media.download.")
+          written (io/write* path body :size size)]
+
+      (when (not= written size)
+        (ex/raise :type :internal
+                  :code :mismatch-write-size
+                  :hint "unexpected state: unable to write to file"))
+
+      {:filename "tempfile"
+       :size size
+       :path path
+       :mtype mtype})))
+
 (defn- create-file-media-object-from-url
   [cfg {:keys [url name] :as params}]
-  (let [content (media/download-image cfg url)
+  (let [content (download-image cfg url)
         params  (-> params
                     (assoc :content content)
-                    (assoc :name (d/nilv name "unknown")))]
+                    (assoc :name (or name (:filename content))))]
 
     ;; NOTE: we use the climit here in a dynamic invocation because we
     ;; don't want saturate the process-image limit with IO (download

backend/src/app/rpc/commands/profile.clj

@@ -30,13 +30,16 @@
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
 
 (declare check-profile-existence!)
 (declare decode-row)
+(declare derive-password)
 (declare filter-props)
 (declare get-profile)
 (declare strip-private-attrs)
+(declare verify-password)
 
 (def schema:props-notifications
   [:map {:title "props-notifications"}
@@ -107,9 +110,7 @@
 (defn get-profile
   "Get profile by id. Throws not-found exception if no profile found."
   [conn id & {:as opts}]
-  ;; NOTE: We need to set ::db/remove-deleted to false because demo profiles
+  (-> (db/get-by-id conn :profile id opts)
-  ;; are created with a set deleted-at value
-  (-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false))
       (decode-row)))
 
 ;; --- MUTATION: Update Profile (own)
@@ -130,7 +131,9 @@
   ;; NOTE: we need to retrieve the profile independently if we use
   ;; it or not for explicit locking and avoid concurrent updates of
   ;; the same row/object.
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
+                    (decode-row))
+
         ;; Update the profile map with direct params
         profile (-> profile
                     (assoc :fullname fullname)
@@ -140,9 +143,9 @@
     (db/update! conn :profile
                 {:fullname fullname
                  :lang lang
-                 :theme theme}
+                 :theme theme
-                {:id profile-id}
+                 :props (db/tjson (:props profile))}
-                {::db/return-keys false})
+                {:id profile-id})
 
     (-> profile
         (strip-private-attrs)
@@ -154,6 +157,7 @@
 
 (declare validate-password!)
 (declare update-profile-password!)
+(declare invalidate-profile-session!)
 
 (def ^:private
   schema:update-profile-password
@@ -168,7 +172,8 @@
    ::climit/id :auth/global
    ::db/transaction true}
   [cfg {:keys [::rpc/profile-id password] :as params}]
-  (let [profile (validate-password! cfg (assoc params :profile-id profile-id))]
+  (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
+        session-id (::session/id params)]
 
     (when (= (:email profile) (str/lower (:password params)))
       (ex/raise :type :validation
@@ -176,18 +181,20 @@
                 :hint "you can't use your email as password"))
 
     (update-profile-password! cfg (assoc profile :password password))
-
+    (invalidate-profile-session! cfg profile-id session-id)
-    (->> (rph/get-request params)
-         (session/get-session)
-         (session/invalidate-others cfg))
-
     nil))
 
+(defn- invalidate-profile-session!
+  "Removes all sessions except the current one."
+  [{:keys [::db/conn]} profile-id session-id]
+  (let [sql "delete from http_session where profile_id = ? and id != ?"]
+    (:next.jdbc/update-count (db/exec-one! conn [sql profile-id session-id]))))
+
 (defn- validate-password!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
   (let [profile (db/get-by-id conn :profile profile-id ::sql/for-update true)]
     (when (and (not= (:password profile) "!")
-               (not (:valid (auth/verify-password old-password (:password profile)))))
+               (not (:valid (verify-password cfg old-password (:password profile)))))
       (ex/raise :type :validation
                 :code :old-password-not-match))
     profile))
@@ -196,7 +203,7 @@
   [{:keys [::db/conn] :as cfg} {:keys [id password] :as profile}]
   (when-not (db/read-only? conn)
     (db/update! conn :profile
-                {:password (auth/derive-password password)}
+                {:password (derive-password cfg password)}
                 {:id id})
     nil))
 
@@ -221,22 +228,21 @@
 
 (defn- update-notifications!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
-  (let [profile
+  (let [profile (get-profile conn profile-id)
-        (get-profile conn profile-id ::db/for-update true)
 
         notifications
         {:dashboard-comments dashboard-comments
          :email-comments email-comments
-         :email-invites email-invites}
+         :email-invites email-invites}]
 
-        props
+    (db/update!
-        (-> (get profile :props)
+     conn :profile
-            (assoc :notifications notifications))]
+     {:props
+      (-> (:props profile)
+          (assoc :notifications notifications)
+          (db/tjson))}
+     {:id (:id profile)})
 
-    (db/update! conn :profile
-                {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
     nil))
 
 ;; --- MUTATION: Update Photo
@@ -247,7 +253,7 @@
 (def ^:private
   schema:update-profile-photo
   [:map {:title "update-profile-photo"}
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-profile-photo
   {:doc/added "1.1"
@@ -280,9 +286,9 @@
                          :file-path (str (:path file))
                          :file-mtype (:mtype file)}}))))
 
-(defn- generate-thumbnail
+(defn- generate-thumbnail!
-  [_ input]
+  [_ file]
-  (let [input   (media/run {:cmd :info :input input})
+  (let [input   (media/run {:cmd :info :input file})
         thumb   (media/run {:cmd :profile-thumbnail
                             :format :jpeg
                             :quality 85
@@ -298,12 +304,13 @@
      :content-type (:mtype thumb)}))
 
 (defn upload-photo
-  [{:keys [::sto/storage] :as cfg} {:keys [file] :as params}]
+  [{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file] :as params}]
   (let [params (-> cfg
                    (assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
                                        [:process-image/global]])
                    (assoc ::climit/label "upload-photo")
-                   (climit/invoke! generate-thumbnail file))]
+                   (assoc ::climit/executor executor)
+                   (climit/invoke! generate-thumbnail! file))]
     (sto/put-object! storage params)))
 
 ;; --- MUTATION: Request Email Change
@@ -343,12 +350,12 @@
 
 (defn- request-email-change!
   [{:keys [::db/conn] :as cfg} {:keys [profile email] :as params}]
-  (let [token   (tokens/generate cfg
+  (let [token   (tokens/generate (::setup/props cfg)
                                  {:iss :change-email
                                   :exp (ct/in-future "15m")
                                   :profile-id (:id profile)
                                   :email email})
-        ptoken  (tokens/generate cfg
+        ptoken  (tokens/generate (::setup/props cfg)
                                  {:iss :profile-identity
                                   :profile-id (:id profile)
                                   :exp (ct/in-future {:days 30})})]
@@ -404,7 +411,7 @@
 
 (defn update-profile-props
   [{:keys [::db/conn] :as cfg} profile-id props]
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (get-profile conn profile-id ::sql/for-update true)
         props   (reduce-kv (fn [props k v]
                              ;; We don't accept namespaced keys
                              (if (simple-ident? k)
@@ -417,17 +424,16 @@
 
     (db/update! conn :profile
                 {:props (db/tjson props)}
-                {:id profile-id}
+                {:id profile-id})
-                {::db/return-keys false})
 
     (filter-props props)))
 
 (sv/defmethod ::update-profile-props
   {::doc/added "1.0"
-   ::sm/params schema:update-profile-props
+   ::sm/params schema:update-profile-props}
-   ::db/transaction true}
   [cfg {:keys [::rpc/profile-id props]}]
-  (update-profile-props cfg profile-id props))
+  (db/tx-run! cfg (fn [cfg]
+                    (update-profile-props cfg profile-id props))))
 
 ;; --- MUTATION: Delete Profile
 
@@ -465,29 +471,6 @@
     (-> (rph/wrap nil)
         (rph/with-transform (session/delete-fn cfg)))))
 
-(def sql:get-subscription-editors
-  "SELECT DISTINCT
-          p.id,
-          p.fullname AS name,
-          p.email AS email
-     FROM team_profile_rel AS tpr1
-     JOIN team as t
-       ON tpr1.team_id = t.id
-     JOIN team_profile_rel AS tpr2
-       ON (tpr1.team_id = tpr2.team_id)
-     JOIN profile AS p
-       ON (tpr2.profile_id = p.id)
-    WHERE tpr1.profile_id = ?
-      AND tpr1.is_owner IS true
-      AND tpr2.can_edit IS true
-      AND t.deleted_at IS NULL")
-
-(sv/defmethod ::get-subscription-usage
-  {::doc/added "2.9"}
-  [cfg {:keys [::rpc/profile-id]}]
-  (let [editors (db/exec! cfg [sql:get-subscription-editors profile-id])]
-    {:editors editors}))
-
 ;; --- HELPERS
 
 (def sql:owned-teams
@@ -545,6 +528,15 @@
   [props]
   (into {} (filter (fn [[k _]] (simple-ident? k))) props))
 
+(defn derive-password
+  [{:keys [::wrk/executor]} password]
+  (when password
+    (px/invoke! executor (partial auth/derive-password password))))
+
+(defn verify-password
+  [{:keys [::wrk/executor]} password password-data]
+  (px/invoke! executor (partial auth/verify-password password password-data)))
+
 (defn decode-row
   [{:keys [props] :as row}]
   (cond-> row

backend/src/app/rpc/commands/projects.clj

@@ -70,27 +70,7 @@
 
 ;; --- QUERY: Get projects
 
-(def ^:private sql:projects
+(declare get-projects)
-  "SELECT p.*,
-          coalesce(tpp.is_pinned, false) as is_pinned,
-          (SELECT count(*) FROM file AS f
-            WHERE f.project_id = p.id
-              AND f.deleted_at is null) AS count,
-          (SELECT count(*) FROM file AS f
-            WHERE f.project_id = p.id) AS total_count
-     FROM project AS p
-    INNER JOIN team AS t ON (t.id = p.team_id)
-     LEFT JOIN team_project_profile_rel AS tpp
-            ON (tpp.project_id = p.id AND
-                tpp.team_id = p.team_id AND
-                tpp.profile_id = ?)
-    WHERE p.team_id = ?
-      AND t.deleted_at is null
-    ORDER BY p.modified_at DESC")
-
-(defn get-projects
-  [conn profile-id team-id]
-  (db/exec! conn [sql:projects profile-id team-id]))
 
 (def ^:private schema:get-projects
   [:map {:title "get-projects"}
@@ -98,11 +78,32 @@
 
 (sv/defmethod ::get-projects
   {::doc/added "1.18"
-   ::doc/changes [["2.12" "This endpoint now return deleted but recoverable projects"]]
    ::sm/params schema:get-projects}
-  [cfg {:keys [::rpc/profile-id team-id]}]
+  [{:keys [::db/pool]} {:keys [::rpc/profile-id team-id]}]
-  (teams/check-read-permissions! cfg profile-id team-id)
+  (dm/with-open [conn (db/open pool)]
-  (get-projects cfg profile-id team-id))
+    (teams/check-read-permissions! conn profile-id team-id)
+    (get-projects conn profile-id team-id)))
+
+(def sql:projects
+  "select p.*,
+          coalesce(tpp.is_pinned, false) as is_pinned,
+          (select count(*) from file as f
+            where f.project_id = p.id
+              and deleted_at is null) as count
+     from project as p
+    inner join team as t on (t.id = p.team_id)
+     left join team_project_profile_rel as tpp
+            on (tpp.project_id = p.id and
+                tpp.team_id = p.team_id and
+                tpp.profile_id = ?)
+    where p.team_id = ?
+      and p.deleted_at is null
+      and t.deleted_at is null
+    order by p.modified_at desc")
+
+(defn get-projects
+  [conn profile-id team-id]
+  (db/exec! conn [sql:projects profile-id team-id]))
 
 ;; --- QUERY: Get all projects
 
@@ -169,19 +170,12 @@
 ;; --- MUTATION: Create Project
 
 (defn- create-project
-  [{:keys [::db/conn] :as cfg} {:keys [::rpc/request-at profile-id team-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id team-id] :as params}]
-  (assert (ct/inst? request-at) "expect request-at assigned")
+  (let [project (teams/create-project conn params)]
-  (let [params    (-> params
-                      (assoc :created-at request-at)
-                      (assoc :modified-at request-at))
-        project   (teams/create-project conn params)
-        timestamp (::rpc/request-at params)]
     (teams/create-project-role conn profile-id (:id project) :owner)
     (db/insert! conn :team-project-profile-rel
                 {:project-id (:id project)
                  :profile-id profile-id
-                 :created-at timestamp
-                 :modified-at timestamp
                  :team-id team-id
                  :is-pinned false})
     (assoc project :is-pinned false)))

backend/src/app/rpc/commands/teams.clj

@@ -12,7 +12,7 @@
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
    [app.common.time :as ct]
-   [app.common.types.team :as types.team]
+   [app.common.types.team :as tt]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -37,14 +37,14 @@
 ;; --- Helpers & Specs
 
 (def ^:private sql:team-permissions
-  "SELECT tpr.is_owner,
+  "select tpr.is_owner,
           tpr.is_admin,
           tpr.can_edit
-     FROM team_profile_rel AS tpr
+     from team_profile_rel as tpr
-     JOIN team AS t ON (t.id = tpr.team_id)
+     join team as t on (t.id = tpr.team_id)
-    WHERE tpr.profile_id = ?
+    where tpr.profile_id = ?
-      AND tpr.team_id = ?
+      and tpr.team_id = ?
-      AND t.deleted_at IS NULL")
+      and t.deleted_at is null")
 
 (defn get-permissions
   [conn profile-id team-id]
@@ -443,18 +443,13 @@
    [:team-id ::sm/uuid]])
 
 (def sql:team-invitations
-  "SELECT email_to AS email,
+  "select email_to as email, role, (valid_until < now()) as expired
-          role,
+   from team_invitation where team_id = ? order by valid_until desc, created_at desc")
-          (valid_until < ?::timestamptz) AS expired
-     FROM team_invitation
-    WHERE team_id = ?
-    ORDER BY valid_until DESC, created_at DESC")
 
 (defn get-team-invitations
   [conn team-id]
-  (let [now (ct/now)]
+  (->> (db/exec! conn [sql:team-invitations team-id])
-    (->> (db/exec! conn [sql:team-invitations now team-id])
+       (mapv #(update % :role keyword))))
-         (mapv #(update % :role keyword)))))
 
 (sv/defmethod ::get-team-invitations
   {::doc/added "1.17"
@@ -508,7 +503,7 @@
 
   (let [features (-> (cfeat/get-enabled-features cf/flags)
                      (set/difference cfeat/frontend-only-features)
-                     (set/difference cfeat/no-team-inheritable-features))
+                     (cfeat/check-client-features! (:features params)))
         params   (-> params
                      (assoc :profile-id profile-id)
                      (assoc :features features))
@@ -634,7 +629,7 @@
 
         ;; assign owner role to new profile
         (db/update! conn :team-profile-rel
-                    (get types.team/permissions-for-role :owner)
+                    (get tt/permissions-for-role :owner)
                     {:team-id id :profile-id reassign-to}))
 
       ;; and finally, if all other conditions does not match and the
@@ -747,7 +742,7 @@
                          :team-id team-id
                          :role role})
 
-    (let [params (get types.team/permissions-for-role role)]
+    (let [params (get tt/permissions-for-role role)]
       ;; Only allow single owner on team
       (when (= role :owner)
         (db/update! conn :team-profile-rel
@@ -765,7 +760,7 @@
   [:map {:title "update-team-member-role"}
    [:team-id ::sm/uuid]
    [:member-id ::sm/uuid]
-   [:role types.team/schema:role]])
+   [:role ::tt/role]])
 
 (sv/defmethod ::update-team-member-role
   {::doc/added "1.17"
@@ -815,7 +810,7 @@
 (def ^:private schema:update-team-photo
   [:map {:title "update-team-photo"}
    [:team-id ::sm/uuid]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-team-photo
   {::doc/added "1.17"

backend/src/app/rpc/commands/teams_invitations.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.teams-invitations
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -22,6 +21,7 @@
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
@@ -43,7 +43,7 @@
 
 (defn- create-invitation-token
   [cfg {:keys [profile-id valid-until team-id member-id member-email role]}]
-  (tokens/generate cfg
+  (tokens/generate (::setup/props cfg)
                    {:iss :team-invitation
                     :exp valid-until
                     :profile-id profile-id
@@ -54,8 +54,12 @@
 
 (defn- create-profile-identity-token
   [cfg profile-id]
-  (assert (uuid? profile-id) "expected valid uuid for profile-id")
+
-  (tokens/generate cfg
+  (dm/assert!
+   "expected valid uuid for profile-id"
+   (uuid? profile-id))
+
+  (tokens/generate (::setup/props cfg)
                    {:iss :profile-identity
                     :profile-id profile-id
                     :exp (ct/in-future {:days 30})}))
@@ -71,7 +75,7 @@
     [:map
      [:id ::sm/uuid]
      [:fullname :string]]]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:email ::sm/email]])
 
 (def ^:private check-create-invitation-params
@@ -220,112 +224,62 @@
 (def ^:private xf:map-email (map :email))
 
 (defn- create-team-invitations
-  "Unified function to handle both create and resend team invitations.
+  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails] :as params}]
-  Accepts either:
+  (let [emails           (set emails)
-  - emails (set) + role (single role for all emails)
-  - invitations (vector of {:email :role} maps)"
-  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails invitations] :as params}]
-  (let [;; Normalize input to a consistent format: [{:email :role}]
-        invitation-data  (cond
-                           ;; Case 1: emails + single role (create invitations style)
-                           (and emails role)
-                           (map (fn [email] {:email email :role role}) emails)
 
-                           ;; Case 2: invitations with individual roles (resend invitations style)
+        join-requests    (->> (get-valid-access-request-profiles conn (:id team))
-                           (some? invitations)
+                              (d/index-by :email))
-                           invitations
 
-                           :else
+        team-members     (into #{} xf:map-email
-                           (throw (ex-info "Invalid parameters: must provide either emails+role or invitations" {})))
+                               (teams/get-team-members conn (:id team)))
 
-        invitation-emails (into #{} (map :email) invitation-data)
+        invitations      (into #{}
-
+                               (comp
-        join-requests     (->> (get-valid-access-request-profiles conn (:id team))
+                                ;;  We don't re-send inviation to
-                               (d/index-by :email))
+                                ;;  already existing members
-
+                                (remove team-members)
-        team-members      (into #{} xf:map-email
-                                (teams/get-team-members conn (:id team)))
-
-        invitations       (into #{}
-                                (comp
-                                ;; We don't re-send invitations to
-                                ;; already existing members
-                                 (remove #(contains? team-members (:email %)))
                                 ;; We don't send invitations to
                                 ;; join-requested members
-                                 (remove #(contains? join-requests (:email %)))
+                                (remove join-requests)
-                                 (map (fn [{:keys [email role]}]
+                                (map (fn [email] (assoc params :email email)))
-                                        (create-invitation cfg
+                                (keep (partial create-invitation cfg)))
-                                                           (-> params
+                               emails)]
-                                                               (assoc :email email)
-                                                               (assoc :role role)))))
-                                 (remove nil?))
-                                invitation-data)]
 
     ;; For requested invitations, do not send invitation emails, add
     ;; the user directly to the team
     (->> join-requests
-         (filter #(contains? invitation-emails (key %)))
+         (filter #(contains? emails (key %)))
-         (map (fn [[email member]]
+         (map val)
-                (let [role (:role (first (filter #(= (:email %) email) invitation-data)))]
+         (run! (partial add-member-to-team conn profile team role)))
-                  (add-member-to-team conn profile team role member))))
-         (doall))
 
     invitations))
 
 (def ^:private schema:create-team-invitations
-  [:and
+  [:map {:title "create-team-invitations"}
-   [:map {:title "create-team-invitations"}
+   [:team-id ::sm/uuid]
-    [:team-id ::sm/uuid]
+   [:role ::types.team/role]
-    ;; Support both formats:
+   [:emails [::sm/set ::sm/email]]])
-    ;; 1. emails (set) + role (single role for all)
-    ;; 2. invitations (vector of {:email :role} maps)
-    [:emails {:optional true} [::sm/set ::sm/email]]
-    [:role {:optional true} types.team/schema:role]
-    [:invitations {:optional true} [:vector [:map
-                                             [:email ::sm/email]
-                                             [:role types.team/schema:role]]]]]
-
-   ;; Ensure exactly one format is provided
-   [:fn (fn [params]
-          (let [has-emails-role (and (contains? params :emails)
-                                     (contains? params :role))
-                has-invitations (contains? params :invitations)]
-            (and (or has-emails-role has-invitations)
-                 (not (and has-emails-role has-invitations)))))]])
 
 (def ^:private max-invitations-by-request-threshold
   "The number of invitations can be sent in a single rpc request"
   25)
 
 (sv/defmethod ::create-team-invitations
-  "A rpc call that allows to send single or multiple invitations to join the team.
+  "A rpc call that allow to send a single or multiple invitations to
-
+  join the team."
-  Supports two parameter formats:
-  1. emails (set) + role (single role for all emails)
-  2. invitations (vector of {:email :role} maps for individual roles)"
   {::doc/added "1.17"
    ::doc/module :teams
    ::sm/params schema:create-team-invitations}
-  [cfg {:keys [::rpc/profile-id team-id role emails] :as params}]
+  [cfg {:keys [::rpc/profile-id team-id emails] :as params}]
   (let [perms    (teams/get-permissions cfg profile-id team-id)
         profile  (db/get-by-id cfg :profile profile-id)
-        ;; Determine which format is being used
+        emails   (into #{} (map profile/clean-email) emails)]
-        using-emails-format? (and emails role)
-        ;; Handle both parameter formats
-        emails   (if using-emails-format?
-                   (into #{} (map profile/clean-email) emails)
-                   #{})
-        ;; Calculate total invitation count for both formats
-        invitation-count (if using-emails-format?
-                           (count emails)
-                           (count (:invitations params)))]
 
     (when-not (:is-admin perms)
       (ex/raise :type :validation
                 :code :insufficient-permissions))
 
-    (when (> invitation-count max-invitations-by-request-threshold)
+    (when (> (count emails) max-invitations-by-request-threshold)
       (ex/raise :type :validation
                 :code :max-invitations-by-request
                 :hint "the maximum of invitation on single request is reached"
@@ -334,7 +288,7 @@
     (-> cfg
         (assoc ::quotes/profile-id profile-id)
         (assoc ::quotes/team-id team-id)
-        (assoc ::quotes/incr invitation-count)
+        (assoc ::quotes/incr (count emails))
         (quotes/check! {::quotes/id ::quotes/invitations-per-team}
                        {::quotes/id ::quotes/profiles-per-team}))
 
@@ -350,12 +304,7 @@
                                   (-> params
                                       (assoc :profile profile)
                                       (assoc :team team)
-                                      ;; Pass parameters in the correct format for the unified function
+                                      (assoc :emails emails)))]
-                                      (cond-> using-emails-format?
-                                        ;; If using emails+role format, ensure both are present
-                                        (assoc :emails emails :role role)
-                                        ;; If using invitations format, the :invitations key is already in params
-                                        (not using-emails-format?) identity)))]
 
       (with-meta {:total (count invitations)
                   :invitations invitations}
@@ -369,7 +318,7 @@
    [:features {:optional true} ::cfeat/features]
    [:id {:optional true} ::sm/uuid]
    [:emails [::sm/set ::sm/email]]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::create-team-with-invitations
   {::doc/added "1.17"
@@ -454,7 +403,7 @@
   [:map {:title "update-team-invitation-role"}
    [:team-id ::sm/uuid]
    [:email ::sm/email]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::update-team-invitation-role
   {::doc/added "1.17"
@@ -518,7 +467,7 @@
 
 (defn- check-existing-team-access-request
   "Checks if an existing team access request is still valid"
-  [{:keys [::db/conn]} team-id profile-id]
+  [conn team-id profile-id]
   (when-let [request (db/get* conn :team-access-request
                               {:team-id team-id
                                :requester-id profile-id})]
@@ -536,8 +485,8 @@
 
 (defn- upsert-team-access-request
   "Create or update team access request for provided team and profile-id"
-  [{:keys [::db/conn] :as cfg} team-id requester-id]
+  [conn team-id requester-id]
-  (check-existing-team-access-request cfg team-id requester-id)
+  (check-existing-team-access-request conn team-id requester-id)
   (let [valid-until     (ct/in-future {:hours 24})
         auto-join-until (ct/in-future {:days 7})
         request-id      (uuid/next)]
@@ -550,7 +499,7 @@
   "A specific method for obtain a file with name and page-id used for
   team request access procediment"
   [cfg file-id]
-  (let [file (bfc/get-file cfg file-id :migrate? false)]
+  (let [file (files/get-file cfg file-id :migrate? false)]
     (-> file
         (dissoc :data)
         (dissoc :deleted-at)
@@ -599,7 +548,7 @@
     (teams/check-email-bounce conn (:email team-owner) false)
     (teams/check-email-spam conn (:email team-owner) true)
 
-    (let [request (upsert-team-access-request cfg team-id profile-id)
+    (let [request (upsert-team-access-request conn team-id profile-id)
           factory (cond
                     (and (some? file) (:is-default team) is-viewer)
                     eml/request-file-access-yourpenpot-view

backend/src/app/rpc/commands/verify_token.clj

@@ -38,7 +38,7 @@
    ::doc/module :auth
    ::sm/params schema:verify-token}
   [cfg {:keys [token] :as params}]
-  (let [claims (tokens/verify cfg {:token token})]
+  (let [claims (tokens/verify (::setup/props cfg) {:token token})]
     (db/tx-run! cfg process-token params claims)))
 
 (defmethod process-token :change-email
@@ -73,7 +73,7 @@
                   {:id (:id profile)}))
 
     (-> claims
-        (rph/with-transform (session/create-fn cfg profile))
+        (rph/with-transform (session/create-fn cfg profile-id))
         (rph/with-meta {::audit/name "verify-profile-email"
                         ::audit/props (audit/profile->props profile)
                         ::audit/profile-id (:id profile)}))))
@@ -128,7 +128,7 @@
    [:iss :keyword]
    [:exp ::ct/inst]
    [:profile-id ::sm/uuid]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:team-id ::sm/uuid]
    [:member-email ::sm/email]
    [:member-id {:optional true} ::sm/uuid]])

backend/src/app/rpc/commands/viewer.clj

@@ -51,7 +51,7 @@
 
 (defn- get-view-only-bundle
   [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
-  (let [file    (bfc/get-file cfg file-id)
+  (let [file    (files/get-file cfg file-id)
 
         project (db/get conn :project
                         {:id (:project-id file)}
@@ -81,7 +81,7 @@
 
         libs    (->> (bfc/get-file-libraries conn file-id)
                      (mapv (fn [{:keys [id] :as lib}]
-                             (merge lib (bfc/get-file cfg id)))))
+                             (merge lib (files/get-file cfg id)))))
 
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]

backend/src/app/rpc/cond.clj

@@ -39,8 +39,9 @@
 (defn- encode
   [s]
   (-> s
-      (bh/blake2b-256)
+      bh/blake2b-256
-      (bc/bytes->b64-str true)))
+      bc/bytes->b64u
+      bc/bytes->str))
 
 (defn- fmt-key
   [s]

backend/src/app/rpc/doc.clj

@@ -16,7 +16,6 @@
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.openapi :as oapi]
    [app.common.schema.registry :as sr]
-   [app.common.uri :as u]
    [app.config :as cf]
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
@@ -26,6 +25,7 @@
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
+   [integrant.core :as ig]
    [pretty-spec.core :as ps]
    [yetti.response :as-alias yres]))
 
@@ -33,8 +33,8 @@
 ;; DOC (human readable)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- context
+(defn- prepare-doc-context
-  [{:keys [methods entrypoint label openapi]}]
+  [methods]
   (letfn [(fmt-spec [mdata]
             (when-let [spec (ex/ignoring (s/spec (::sv/spec mdata)))]
               (with-out-str
@@ -62,10 +62,8 @@
              :added (::added mdata)
              :changes (some->> (::changes mdata) (partition-all 2) (map vec))
              :spec (fmt-spec mdata)
-             :entrypoint (-> entrypoint
+             :entrypoint (str (cf/get :public-uri) "/api/rpc/command/" (::sv/name mdata))
-                             (u/ensure-path-slash)
+
-                             (u/join (::sv/name mdata))
-                             (str))
              :params-schema-js   (fmt-schema :js mdata ::sm/params)
              :result-schema-js   (fmt-schema :js mdata ::sm/result)
              :webhook-schema-js  (fmt-schema :js mdata ::sm/webhook)
@@ -74,9 +72,6 @@
              :webhook-schema-clj (fmt-schema :clj mdata ::sm/webhook)})]
 
     {:version (:main cf/version)
-     :label label
-     :entrypoint (str entrypoint)
-     :openapi (str openapi)
      :methods
      (->> methods
           (map val)
@@ -85,19 +80,17 @@
           (map get-context)
           (sort-by (juxt :module :name)))}))
 
-(defn- handler
+(defn- doc-handler
-  [& {:keys [template] :as options}]
+  [context]
   (if (contains? cf/flags :backend-api-doc)
-    (let [context  (delay (context options))
+    (fn [request]
-          template (or template "app/templates/api-doc.tmpl")]
+      (let [params  (:query-params request)
-      (fn [request]
+            pstyle  (:type params "js")
-        (let [params  (:query-params request)
+            context (assoc @context :param-style pstyle)]
-              pstyle  (:type params "js")
-              context (assoc @context :param-style pstyle)]
 
-          {::yres/status 200
+        {::yres/status 200
-           ::yres/body (-> (io/resource template)
+         ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
-                           (tmpl/render context))})))
+                         (tmpl/render context))}))
     (fn [_]
       {::yres/status 404})))
 
@@ -105,8 +98,8 @@
 ;; OPENAPI / SWAGGER (v3.1)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- openapi-context
+(defn prepare-openapi-context
-  [{:keys [methods entrypoint description]}]
+  [methods]
   (let [definitions (atom {})
         options {:registry sr/default-registry
                  ::oapi/definitions-path "#/components/schemas/"
@@ -119,9 +112,7 @@
         (fn [tsx schema]
           (let [schema  (sm/schema schema)
                 example (sm/generate schema)
-                example (sm/encode schema example output-transformer)
+                example (sm/encode schema example output-transformer)]
-                example (json/encode example :key-fn json/write-camel-key)]
-
             {:default
              {:description "A default response"
               :content
@@ -132,9 +123,7 @@
         gen-params-doc
         (fn [tsx schema]
           (let [example (sm/generate schema)
-                example (sm/encode schema example output-transformer)
+                example (sm/encode schema example output-transformer)]
-                example (json/encode example :key-fn json/write-camel-key)]
-
             {:required true
              :content
              {"application/json"
@@ -169,35 +158,37 @@
                (map gen-method-doc)
                (sort-by (juxt :module :name))
                (map (fn [doc]
-                      [(:name doc) (:repr doc)]))
+                      [(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
                (into {})))]
 
     {:openapi "3.0.0"
      :info {:version (:main cf/version)}
-     :servers [{:url (str entrypoint)
+     :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
-                :description (or description "")}]
+                ;; :description "penpot backend"
+                }]
+     :security
+     {:api_key []}
+
      :paths paths
      :components {:schemas @definitions}}))
 
-(defn- openapi-json-handler
+(defn openapi-json-handler
-  [& {:as options}]
+  [context]
   (if (contains? cf/flags :backend-openapi-doc)
-    (let [context (delay (openapi-context options))]
+    (fn [_]
-      (fn [_]
+      {::yres/status 200
-        {::yres/status 200
+       ::yres/headers {"content-type" "application/json; charset=utf-8"}
-         ::yres/headers {"content-type" "application/json; charset=utf-8"}
+       ::yres/body (json/encode @context)})
-         ::yres/body (json/encode @context)}))
     (fn [_]
       {::yres/status 404})))
 
-(defn- openapi-handler
+(defn openapi-handler
-  [& {:keys [uri label]}]
+  []
   (if (contains? cf/flags :backend-openapi-doc)
     (fn [_]
       (let [swagger-js (slurp (io/resource "app/assets/swagger-ui-4.18.3.js"))
             swagger-cs (slurp (io/resource "app/assets/swagger-ui-4.18.3.css"))
-            context    {:uri (str uri)
+            context    {:public-uri (cf/get :public-uri)
-                        :label label
                         :swagger-js swagger-js
                         :swagger-css swagger-cs}]
         {::yres/status 200
@@ -208,43 +199,27 @@
       {::yres/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ROUTES HELPER
+;; MODULE INIT
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn routes
+(defmethod ig/assert-key ::routes
-  [& {:keys [label base-uri description methods]}]
+  [_ params]
-  (let [entrypoint
+  (assert (sm/valid? ::rpc/methods (::rpc/methods params)) "expected valid methods"))
-        (-> base-uri
-            (u/ensure-path-slash)
-            (u/join "methods"))
 
-        openapi
+(defmethod ig/init-key ::routes
-        (-> base-uri
+  [_ {:keys [::rpc/methods] :as cfg}]
-            (u/ensure-path-slash)
+  [(let [context (delay (prepare-doc-context methods))]
-            (u/join "doc/openapi"))
+     [["/_doc"
+       {:handler (doc-handler context)
+        :allowed-methods #{:get}}]
+      ["/doc"
+       {:handler (doc-handler context)
+        :allowed-methods #{:get}}]])
 
-        template
+   (let [context (delay (prepare-openapi-context methods))]
-        (case label
+     [["/openapi"
-          "management" "app/templates/management-api-doc.tmpl"
+       {:handler (openapi-handler)
-          "main"       "app/templates/main-api-doc.tmpl")]
+        :allowed-methods #{:get}}]
-
+      ["/openapi.json"
-    ["/doc"
+       {:handler (openapi-json-handler context)
-     ["" {:handler (handler :methods methods
+        :allowed-methods #{:get}}]])])
-                            :label label
-                            :entrypoint entrypoint
-                            :openapi openapi
-                            :template template)
-          :allowed-methods #{:get}}]
-
-     ["/openapi"
-      {:handler (openapi-handler
-                 :uri (u/join openapi "openapi.json")
-                 :label label)
-       :allowed-methods #{:get}}]
-
-     ["/openapi.json"
-      {:handler (openapi-json-handler {:entrypoint entrypoint
-                                       :description description
-                                       :methods methods})
-
-       :allowed-methods #{:get}}]]))

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [yetti.response :as yres]))
+   [yetti.response :as-alias yres]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -78,21 +78,3 @@
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
                  (update response ::yres/headers assoc "cache-control" val)))))
-
-(defn stream
-  "A convenience allias for yetti.response/stream-body"
-  [f]
-  (yres/stream-body f))
-
-(defn get-request
-  "Get http request from RPC params"
-  [params]
-  (assert (contains? params ::rpc/request-at) "rpc params required")
-  (-> (meta params)
-      (get ::http/request)))
-
-(defn get-auth-data
-  "Get http auth-data from RPC params"
-  [params]
-  (-> (get-request params)
-      (get ::http/auth-data)))

backend/src/app/rpc/management/exporter.clj

@@ -1,49 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.rpc.management.exporter
-  (:require
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.uri :as u]
-   [app.config :as cf]
-   [app.media :refer [schema:upload]]
-   [app.rpc :as-alias rpc]
-   [app.rpc.doc :as doc]
-   [app.storage :as sto]
-   [app.util.services :as sv]))
-
-;; ---- RPC METHOD: UPLOAD-TEMPFILE
-
-(def ^:private
-  schema:upload-tempfile-params
-  [:map {:title "upload-templfile-params"}
-   [:content schema:upload]])
-
-(def ^:private
-  schema:upload-tempfile-result
-  [:map {:title "upload-templfile-result"}])
-
-(sv/defmethod ::upload-tempfile
-  {::doc/added "2.12"
-   ::sm/params schema:upload-tempfile-params
-   ::sm/result schema:upload-tempfile-result}
-  [cfg {:keys [::rpc/profile-id content]}]
-  (let [storage (sto/resolve cfg)
-        hash    (sto/calculate-hash (:path content))
-        data    (-> (sto/content (:path content))
-                    (sto/wrap-with-hash hash))
-        content {::sto/content data
-                 ::sto/deduplicate? true
-                 ::sto/touched-at (ct/in-future {:minutes 10})
-                 :profile-id profile-id
-                 :content-type (:mtype content)
-                 :bucket "tempfile"}
-        object (sto/put-object! storage content)]
-    {:id (:id object)
-     :uri (-> (cf/get :public-uri)
-              (u/join "/assets/by-id/")
-              (u/join (str (:id object))))}))

backend/src/app/rpc/management/subscription.clj

@@ -1,183 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.rpc.management.subscription
-  (:require
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.schema.generators :as sg]
-   [app.common.time :as ct]
-   [app.db :as db]
-   [app.rpc :as-alias rpc]
-   [app.rpc.commands.profile :as profile]
-   [app.rpc.doc :as doc]
-   [app.util.services :as sv]))
-
-;; ---- RPC METHOD: AUTHENTICATE
-
-(def ^:private
-  schema:authenticate-params
-  [:map {:title "authenticate-params"}])
-
-(def ^:private
-  schema:authenticate-result
-  [:map {:title "authenticate-result"}
-   [:profile-id ::sm/uuid]])
-
-(sv/defmethod ::auth
-  {::doc/added "2.12"
-   ::sm/params schema:authenticate-params
-   ::sm/result schema:authenticate-result}
-  [_ {:keys [::rpc/profile-id]}]
-  {:profile-id profile-id})
-
-;; ---- RPC METHOD: GET-CUSTOMER
-
-;; FIXME: move to app.common.time
-(def ^:private schema:timestamp
-  (sm/type-schema
-   {:type ::timestamp
-    :pred ct/inst?
-    :type-properties
-    {:title "inst"
-     :description "The same as :app.common.time/inst but encodes to epoch"
-     :error/message "should be an instant"
-     :gen/gen (->> (sg/small-int)
-                   (sg/fmap (fn [v] (ct/inst v))))
-     :decode/string #(some-> % ct/inst)
-     :encode/string #(some-> % inst-ms)
-     :decode/json #(some-> % ct/inst)
-     :encode/json #(some-> % inst-ms)}}))
-
-(def ^:private schema:subscription
-  [:map {:title "Subscription"}
-   [:id ::sm/text]
-   [:customer-id ::sm/text]
-   [:type [:enum
-           "unlimited"
-           "professional"
-           "enterprise"]]
-   [:status [:enum
-             "active"
-             "canceled"
-             "incomplete"
-             "incomplete_expired"
-             "past_due"
-             "paused"
-             "trialing"
-             "unpaid"]]
-
-   [:billing-period [:enum
-                     "month"
-                     "day"
-                     "week"
-                     "year"]]
-   [:quantity :int]
-   [:description [:maybe ::sm/text]]
-   [:created-at schema:timestamp]
-   [:start-date [:maybe schema:timestamp]]
-   [:ended-at [:maybe schema:timestamp]]
-   [:trial-end [:maybe schema:timestamp]]
-   [:trial-start [:maybe schema:timestamp]]
-   [:cancel-at [:maybe schema:timestamp]]
-   [:canceled-at [:maybe schema:timestamp]]
-   [:current-period-end [:maybe schema:timestamp]]
-   [:current-period-start [:maybe schema:timestamp]]
-   [:cancel-at-period-end :boolean]
-
-   [:cancellation-details
-    [:map {:title "CancellationDetails"}
-     [:comment [:maybe ::sm/text]]
-     [:reason [:maybe ::sm/text]]
-     [:feedback [:maybe
-                 [:enum
-                  "customer_service"
-                  "low_quality"
-                  "missing_feature"
-                  "other"
-                  "switched_service"
-                  "too_complex"
-                  "too_expensive"
-                  "unused"]]]]]])
-
-(def ^:private sql:get-customer-slots
-  "WITH teams AS (
-    SELECT tpr.team_id AS id,
-           tpr.profile_id AS profile_id
-      FROM team_profile_rel AS tpr
-     WHERE tpr.is_owner IS true
-       AND tpr.profile_id = ?
-  ), teams_with_slots AS (
-    SELECT tpr.team_id AS id,
-           count(*) AS total
-      FROM team_profile_rel AS tpr
-     WHERE tpr.team_id IN (SELECT id FROM teams)
-       AND tpr.can_edit IS true
-     GROUP BY 1
-     ORDER BY 2
-  )
-  SELECT max(total) AS total FROM teams_with_slots;")
-
-(defn- get-customer-slots
-  [cfg profile-id]
-  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
-    (:total result)))
-
-(def ^:private schema:get-customer-params
-  [:map])
-
-(def ^:private schema:get-customer-result
-  [:map
-   [:id ::sm/uuid]
-   [:name :string]
-   [:num-editors ::sm/int]
-   [:subscription {:optional true} schema:subscription]])
-
-(sv/defmethod ::get-customer
-  {::doc/added "2.12"
-   ::sm/params schema:get-customer-params
-   ::sm/result schema:get-customer-result}
-  [cfg {:keys [::rpc/profile-id]}]
-  (let [profile (profile/get-profile cfg profile-id)]
-    {:id (get profile :id)
-     :name (get profile :fullname)
-     :email (get profile :email)
-     :num-editors (get-customer-slots cfg profile-id)
-     :subscription (-> profile :props :subscription)}))
-
-
-;; ---- RPC METHOD: GET-CUSTOMER
-
-(def ^:private schema:update-customer-params
-  [:map
-   [:subscription [:maybe schema:subscription]]])
-
-(def ^:private schema:update-customer-result
-  [:map])
-
-(sv/defmethod ::update-customer
-  {::doc/added "2.12"
-   ::sm/params schema:update-customer-params
-   ::sm/result schema:update-customer-result}
-  [cfg {:keys [::rpc/profile-id subscription]}]
-  (let [{:keys [props] :as profile}
-        (profile/get-profile cfg profile-id ::db/for-update true)
-
-        props
-        (assoc props :subscription subscription)]
-
-    (l/dbg :hint "update customer"
-           :profile-id (str profile-id)
-           :subscription-type (get subscription :type)
-           :subscription-status (get subscription :status)
-           :subscription-quantity (get subscription :quantity))
-
-    (db/update! cfg :profile
-                {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
-
-    nil))