Revert "add new configurations"

add new configurations

.circleci/config.yml

@@ -1,53 +1,5 @@
 version: 2.1
-
 jobs:
-  lint:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    steps:
-      - checkout
-
-      - run:
-          name: "fmt check"
-          working_directory: "."
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-
-      - run:
-          name: "lint clj common"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:common
-
-      - run:
-          name: "lint clj frontend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:frontend
-
-      - run:
-          name: "lint clj backend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:backend
-
-      - run:
-          name: "lint clj exporter"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:exporter
-
-      - run:
-          name: "lint clj library"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:library
-
   test-common:
     docker:
       - image: penpotapp/devenv:latest
@@ -65,7 +17,15 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
+            - v1-dependencies-{{ checksum "common/deps.edn"}}
+
+      - run:
+          name: "fmt check & linter"
+          working_directory: "./common"
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+            yarn run lint:clj
 
       - run:
           name: "JVM tests"
@@ -77,16 +37,12 @@ jobs:
           name: "NODE tests"
           working_directory: "./common"
           command: |
-            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
+          key: v1-dependencies-{{ checksum "common/deps.edn"}}
 
   test-frontend:
     docker:
@@ -105,68 +61,36 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
 
       - run:
-          name: "install dependencies"
-          working_directory: "./frontend"
-          # We install playwright here because the dependent tasks
-          # uses the same cache as this task so we prepopulate it
+          name: "prepopulate linter cache"
+          working_directory: "./common"
           command: |
             yarn install
-            yarn run playwright install chromium
+            yarn run lint:clj
 
       - run:
-          name: "lint scss on frontend"
+          name: "fmt check & linter"
           working_directory: "./frontend"
           command: |
+            yarn install
+            yarn run fmt:clj:check
+            yarn run fmt:js:check
             yarn run lint:scss
+            yarn run lint:clj
 
       - run:
           name: "unit tests"
           working_directory: "./frontend"
           command: |
+            yarn install
             yarn run test
 
       - save_cache:
           paths:
             - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-  test-library:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx6g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: Install dependencies and build
-          working_directory: "./library"
-          command: |
-            yarn install
-
-      - run:
-          name: Build and Test
-          working_directory: "./library"
-          command: |
-            ./scripts/build
-            yarn run test
+          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}
 
   test-components:
     docker:
@@ -185,14 +109,14 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
 
       - run:
           name: Install dependencies
           working_directory: "./frontend"
           command: |
-            yarn install
-            yarn run playwright install chromium
+            yarn
+            npx playwright install --with-deps
 
       - run:
           name: Build Storybook
@@ -224,32 +148,17 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
+            - v1-dependencies-{{ checksum "frontend/deps.edn"}}
 
-      # Build frontend
       - run:
-          name: "frontend build"
+          name: "integration tests"
           working_directory: "./frontend"
           command: |
             yarn install
             yarn run build:app:assets
             yarn run build:app
             yarn run build:app:libs
-
-          # Build the wasm bundle
-      - run:
-          name: "wasm build"
-          working_directory: "./render-wasm"
-          command: |
-            EMSDK_QUIET=1 . /opt/emsdk/emsdk_env.sh
-            ./build release
-
-      # Run integration tests
-      - run:
-          name: "integration tests"
-          working_directory: "./frontend"
-          command: |
-            yarn run playwright install chromium
+            yarn run playwright install --with-deps chromium
             yarn run test:e2e -x --workers=4
 
   test-backend:
@@ -276,6 +185,21 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "backend/deps.edn" }}
 
+      - run:
+          name: "prepopulate linter cache"
+          working_directory: "./common"
+          command: |
+            yarn install
+            yarn run lint:clj
+
+      - run:
+          name: "fmt check & linter"
+          working_directory: "./backend"
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+            yarn run lint:clj
+
       - run:
           name: "tests"
           working_directory: "./backend"
@@ -291,9 +215,37 @@ jobs:
       - save_cache:
           paths:
             - ~/.m2
-            - ~/.gitlibs
           key: v1-dependencies-{{ checksum "backend/deps.edn" }}
 
+  test-exporter:
+    docker:
+      - image: penpotapp/devenv:latest
+
+    working_directory: ~/repo
+    resource_class: medium+
+
+    environment:
+      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    steps:
+      - checkout
+
+      - run:
+          name: "prepopulate linter cache"
+          working_directory: "./common"
+          command: |
+            yarn install
+            yarn run lint:clj
+
+      - run:
+          name: "fmt check & linter"
+          working_directory: "./exporter"
+          command: |
+            yarn install
+            yarn run fmt:clj:check
+            yarn run lint:clj
+
   test-render-wasm:
     docker:
       - image: penpotapp/devenv:latest
@@ -326,26 +278,10 @@ jobs:
 workflows:
   penpot:
     jobs:
-      - test-frontend:
-          requires:
-            - lint: success
-
-      - test-library:
-          requires:
-            - lint: success
-
-      - test-components:
-          requires:
-            - lint: success
-
-      - test-backend:
-          requires:
-            - lint: success
-
-      - test-common:
-          requires:
-            - lint: success
-
-      - lint
+      - test-frontend
+      - test-components
       - test-integration
+      - test-backend
+      - test-common
+      - test-exporter
       - test-render-wasm

.clj-kondo/config.edn

@@ -45,16 +45,10 @@
   :potok/reify-type
   {:level :error}
 
-  :missing-protocol-method
-  {:level :off}
-
   :unresolved-namespace
   {:level :warning
    :exclude [data_readers]}
 
-  :unused-value
-  {:level :off}
-
   :single-key-in
   {:level :warning}
 
@@ -70,9 +64,6 @@
   :redundant-nested-call
   {:level :off}
 
-  :redundant-str-call
-  {:level :off}
-
   :earmuffed-var-not-dynamic
   {:level :off}
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,7 +13,6 @@
 - [ ] Add a detailed explanation of how to reproduce the issue and/or verify the fix, if applicable.
 - [ ] Include screenshots or videos, if applicable.
 - [ ] Add or modify existing integration tests in case of bugs or new features, if applicable.
-- [ ] Refactor any modified SCSS files following the refactor guide.
 - [ ] Check CI passes successfully.
 - [ ] Update the `CHANGES.md` file, referencing the related GitHub issue, if applicable.
 

.github/workflows/build-bundle.yml

@@ -1,89 +0,0 @@
-name: Bundles Builder
-
-on:
-  # Create bundle from manual action
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-      build_wasm:
-        description: 'BUILD_WASM. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-      build_storybook:
-        description: 'BUILD_STORYBOOK. Valid values: yes, no'
-        type: string
-        required: false
-        default: 'yes'
-
-jobs:
-  build-bundle:
-    name: Build and Upload Penpot Bundle
-    runs-on: ubuntu-24.04
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.gh_ref }}
-
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Build bundle
-        env:
-          BUILD_WASM: ${{ inputs.build_wasm }}
-          BUILD_STORYBOOK: ${{ inputs.build_storybook }}
-        run: ./manage.sh build-bundle
-
-      - name: Prepare directories for zipping
-        run: |
-          mkdir zips
-          mv bundles penpot
-
-      - name: Create zip bundle
-        run: |
-          echo "📦 Packaging Penpot bundle..."
-          zip -r zips/penpot.zip penpot
-
-      - name: Upload Penpot bundle to S3
-        run: |
-          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          TEXT: |
-            ❌ *[PENPOT] Error during the execution of the job*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/build-develop.yml

@@ -1,21 +0,0 @@
-name: _DEVELOP
-
-on:
-  schedule:
-    - cron: '16 5-20 * * 1-5'
-
-jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: "develop"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
-    secrets: inherit
-    with:
-      gh_ref: "develop"

.github/workflows/build-docker-devenv.yml

@@ -1,36 +0,0 @@
-name: DevEnv Docker Image Builder
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build-and-push:
-    name: Build and push DevEnv Docker image
-    environment: release-admins
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.PUB_DOCKER_USERNAME }}
-          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      - name: Build and push DevEnv Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'penpotapp/devenv'
-        with:
-          context: ./docker/devenv/
-          file: ./docker/devenv/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ env.DOCKER_IMAGE }}:latest
-          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache,mode=max

.github/workflows/build-docker.yml

@@ -1,128 +0,0 @@
-name: Docker Images Builder
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Name of the branch or ref'
-        type: string
-        required: true
-        default: 'develop'
-
-jobs:
-  build-and-push:
-    name: Build and Push Penpot Docker Images
-    runs-on: ubuntu-24.04-arm
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ inputs.gh_ref }}
-
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Download Penpot Bundles
-        env:
-          FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        run: |
-          pushd docker/images
-          aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
-          unzip $FILE_NAME > /dev/null
-          mv penpot/backend bundle-backend
-          mv penpot/frontend bundle-frontend
-          mv penpot/exporter bundle-exporter
-          mv penpot/storybook bundle-storybook
-          popd
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ secrets.DOCKER_REGISTRY }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build and push Backend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'backend'
-          BUNDLE_PATH: './bundle-backend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.backend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Frontend Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'frontend'
-          BUNDLE_PATH: './bundle-frontend'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.frontend
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Exporter Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'exporter'
-          BUNDLE_PATH: './bundle-exporter'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.exporter
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Build and push Storybook Docker image
-        uses: docker/build-push-action@v6
-        env:
-          DOCKER_IMAGE: 'storybook'
-          BUNDLE_PATH: './bundle-storybook'
-        with:
-          context: ./docker/images/
-          file: ./docker/images/Dockerfile.storybook
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
-          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            ❌ 🐳 *[PENPOT] Error building penpot docker images.*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra

.github/workflows/build-staging.yml

@@ -1,21 +0,0 @@
-name: _STAGING
-
-on:
-  schedule:
-    - cron: '36 5-20 * * 1-5'
-
-jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: "staging"
-      build_wasm: "yes"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
-    secrets: inherit
-    with:
-      gh_ref: "staging"

.github/workflows/build-tag.yml

@@ -1,30 +0,0 @@
-name: _TAG
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-bundle:
-    uses: ./.github/workflows/build-bundle.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-      build_wasm: "no"
-      build_storybook: "yes"
-
-  build-docker:
-    needs: build-bundle
-    uses: ./.github/workflows/build-docker.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}
-
-  publish-final-tag:
-    if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta')  && contains(github.ref_name, '.') }}
-    needs: build-docker
-    uses: ./.github/workflows/release.yml
-    secrets: inherit
-    with:
-      gh_ref: ${{ github.ref_name }}

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s["A-Z].*[^.]$'
+          pattern: '^:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):\s[A-Z].*[^.]$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/release.yml

@@ -1,114 +0,0 @@
-name: Release Publisher
-
-on:
-  workflow_dispatch:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-  workflow_call:
-    inputs:
-      gh_ref:
-        description: 'Tag to release'
-        type: string
-        required: true
-
-permissions:
-  contents: write
-
-jobs:
-  release:
-    environment: release-admins
-    runs-on: ubuntu-24.04
-    outputs:
-      version: ${{ steps.vars.outputs.gh_ref }}
-      release_notes: ${{ steps.extract_release_notes.outputs.release_notes }}
-    steps:
-      - name: Extract some useful variables
-        id: vars
-        run: |
-          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ steps.vars.outputs.gh_ref }}
-
-      # --- Publicly release the docker images ---
-      - name: Configure ECR credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
-          aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
-          aws-region: ${{ secrets.AWS_REGION }}
-
-      - name: Install Skopeo
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y skopeo
-
-      - name: Copy images from AWS ECR to Docker Hub
-        env:
-          AWS_REGION: ${{ secrets.AWS_REGION }}
-          DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
-          PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
-          PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-        run: |
-          aws ecr get-login-password --region $AWS_REGION | \
-            skopeo login --username AWS --password-stdin \
-            $DOCKER_REGISTRY
-
-          echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io
-
-          IMAGES=("frontend" "backend" "exporter" "storybook")
-
-          for image in "${IMAGES[@]}"; do
-            skopeo copy --all \
-              docker://$DOCKER_REGISTRY/$image:$TAG \
-              docker://docker.io/$PUB_DOCKER_USERNAME/$image:$TAG
-
-            for alias in main latest; do
-              skopeo copy --all \
-                docker://$DOCKER_REGISTRY/$image:$TAG \
-                docker://docker.io/$PUB_DOCKER_USERNAME/$image:$alias
-            done
-          done
-
-      # --- Release notes extraction ---
-      - name: Extract release notes from CHANGES.md
-        id: extract_release_notes
-        env:
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-        run: |
-          RELEASE_NOTES=$(awk "/^## $TAG$/{flag=1; next} /^## /{flag=0} flag" CHANGES.md | awk '{$1=$1};1')
-          if [ -z "$RELEASE_NOTES" ]; then
-            RELEASE_NOTES="No changes for $TAG according to CHANGES.md"
-          fi
-          echo "release_notes<<EOF" >> $GITHUB_OUTPUT
-          echo "$RELEASE_NOTES" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-
-      # --- Create GitHub release ---
-      - name: Create GitHub release
-        uses: softprops/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ steps.vars.outputs.gh_ref }}
-          name: ${{ steps.vars.outputs.gh_ref }}
-          body: ${{ steps.extract_release_notes.outputs.release_notes }}
-
-      - name: Notify Mattermost
-        if: failure()
-        uses: mattermost/action-mattermost-notify@master
-        with:
-          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
-          MATTERMOST_CHANNEL: bot-alerts-cicd
-          TEXT: |
-            ❌ 🚀 *[PENPOT] Error releasing penpot.*
-            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
-            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-            @infra

.gitignore

@@ -30,8 +30,6 @@
 /*.zip
 /.clj-kondo/.cache
 /_dump
-/notes
-/playground/
 /backend/*.md
 /backend/*.sql
 /backend/*.txt
@@ -42,7 +40,6 @@
 /backend/resources/public/assets
 /backend/resources/public/media
 /backend/target/
-/backend/experiments
 /bundle*
 /cd.md
 /clj-profiler/
@@ -53,6 +50,9 @@
 /exporter/target
 /frontend/.storybook/preview-body.html
 /frontend/.storybook/preview-head.html
+/frontend/cypress/fixtures/validuser.json
+/frontend/cypress/videos/*/
+/frontend/cypress/videos/*/
 /frontend/dist/
 /frontend/npm-debug.log
 /frontend/out/
@@ -68,10 +68,6 @@
 /vendor/**/target
 /vendor/svgclean/bundle*.js
 /web
-/library/target/
-/library/*.zip
-/external
-
 clj-profiler/
 node_modules
 /test-results/

.nvmrc

@@ -1 +1 @@
-v22.19.0
+v22.13.1

CHANGES.md

@@ -1,314 +1,51 @@
 # CHANGELOG
 
-## 2.11.1
-
-- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
-
-## 2.11.0
-
-### :boom: Breaking changes & Deprecations
-
-- Deprecated configuration variables with the prefix `PENPOT_ASSETS_*`, and will be
-  removed in future versions:
-
-  - The `PENPOT_ASSETS_STORAGE_BACKEND` becomes `PENPOT_OBJECTS_STORAGE_BACKEND` and its
-    values passes from (`assets-fs` or `assets-s3`) to (`fs` or `s3`)
-  - The `PENPOT_STORAGE_ASSETS_FS_DIRECTORY` becomes `PENPOT_OBJECTS_STORAGE_FS_DIRECTORY`
-  - The `PENPOT_STORAGE_ASSETS_S3_BUCKET` becomes `PENPOT_OBJECTS_STORAGE_S3_BUCKET`
-  - The `PENPOT_STORAGE_ASSETS_S3_REGION` becomes `PENPOT_OBJECTS_STORAGE_S3_REGION`
-  - The `PENPOT_STORAGE_ASSETS_S3_ENDPOINT` becomes `PENPOT_OBJECTS_STORAGE_S3_ENDPOINT`
-  - The `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` replaced (see below)
-
-- Add `PENPOT_NETTY_IO_THREADS` and `PENPOT_EXECUTOR_THREADS` variables to provide the
-  control over concurrency of the shared resources used by netty. Penpot uses the netty IO
-  threads for AWS S3 SDK and Redis/Valkey communication, and the EXEC threads to perform
-  out of HTTP serving threads tasks such that cache invalidation, S3 response completion,
-  configuration reloading and many other auxiliar tasks. By default they use a half number
-  if available cpus with a minumum of 2 for both executors. You should not touch that
-  variables unless you are know what you are doing.
-
-- Replace the `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` with a more general configuration
-  `PENPOT_NETTY_IO_THREADS` used to configure a shared netty resources across different
-  services which use netty internally (redis connection, S3 SDK client). This
-  configuration is not very commonly used so don't expected real impact on any user.
-
-### :sparkles: New features & Enhancements
-
-- New composite token: Typography [Taiga #10200](https://tree.taiga.io/project/penpot/us/10200)
-- Show current Penpot version [Taiga #11603](https://tree.taiga.io/project/penpot/us/11603)
-- Switch several variant copies at the same time [Taiga #11411](https://tree.taiga.io/project/penpot/us/11411)
-- Invitations management improvements [Taiga #3479](https://tree.taiga.io/project/penpot/us/3479)
-- Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
-- Reorder properties for a component [Taiga #10225](https://tree.taiga.io/project/penpot/us/10225)
-- File Data storage layout refactor [Github #7345](https://github.com/penpot/penpot/pull/7345)
-- Make several queries optimization on comment threads [Github #7506](https://github.com/penpot/penpot/pull/7506)
-
-### :bug: Bugs fixed
-
-- Fix selection problems when devtools open [Taiga #11950](https://tree.taiga.io/project/penpot/issue/11950)
-- Fix long font names overlap [Taiga #11844](https://tree.taiga.io/project/penpot/issue/11844)
-- Fix paste behavior according to the selected element [Taiga #11979](https://tree.taiga.io/project/penpot/issue/11979)
-- Fix problem with export size [Github #7160](https://github.com/penpot/penpot/issues/7160)
-- Fix multi level library dependencies [Taiga #12155](https://tree.taiga.io/project/penpot/issue/12155)
-- Fix component context menu options order in assets tab [Taiga #11941](https://tree.taiga.io/project/penpot/issue/11941)
-- Fix error updating library [Taiga #12218](https://tree.taiga.io/project/penpot/issue/12218)
-- Fix restoring a variant in another file makes it overlap the existing variant [Taiga #12049](https://tree.taiga.io/project/penpot/issue/12049)
-- Fix auto-width changes to fixed when switching variants [Taiga #12172](https://tree.taiga.io/project/penpot/issue/12172)
-- Fix component number has no singular translation string [Taiga #12106](https://tree.taiga.io/project/penpot/issue/12106)
-- Fix adding/removing identical text fills [Taiga #12287](https://tree.taiga.io/project/penpot/issue/12287)
-- Fix scroll on the inspect tab [Taiga #12293](https://tree.taiga.io/project/penpot/issue/12293)
-- Fix lock proportion tooltip [Taiga #12326](https://tree.taiga.io/project/penpot/issue/12326)
-- Fix internal Error when selecting a set by name in the token theme editor [Taiga #12310](https://tree.taiga.io/project/penpot/issue/12310)
-- Fix drag & drop functionality is swapping instead or reordering [Taiga #12254](https://tree.taiga.io/project/penpot/issue/12254)
-- Fix variants not syncronizing tokens on switch [Taiga #12290](https://tree.taiga.io/project/penpot/issue/12290)
-- Fix incorrect behavior of Alt + Drag for variants [Taiga #12309](https://tree.taiga.io/project/penpot/issue/12309)
-- Fix text override is lost after switch [Taiga #12269](https://tree.taiga.io/project/penpot/issue/12269)
-- Fix exporting a board crashing the app [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12384)
-- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
-- Fix selected colors not showing colors from children shapes in multiple selection [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12385)
-- Fix scrollbar issue in design tab [Taiga #12367](https://tree.taiga.io/project/penpot/issue/12367)
-- Fix library update notificacions showing when they should not [Taiga #12397](https://tree.taiga.io/project/penpot/issue/12397)
-- Fix remove flex button doesn’t work within variant [Taiga #12314](https://tree.taiga.io/project/penpot/issue/12314)
-- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
-- Fix problem with certain text input in some editable labels (pages, components, tokens...) being in conflict with the drag/drop functionality [Taiga #12316](https://tree.taiga.io/project/penpot/issue/12316)
-- Fix not controlled theme renaming [Taiga #12411](https://tree.taiga.io/project/penpot/issue/12411)
-- Fix paste without selection sends the new element in the back [Taiga #12382](https://tree.taiga.io/project/penpot/issue/12382)
-- Fix options button does not work for comments created in the lower part of the screen [Taiga #12422](https://tree.taiga.io/project/penpot/issue/12422)
-- Fix problem when checking usage with removed teams [Taiga #12442](https://tree.taiga.io/project/penpot/issue/12442)
-- Fix focus mode persisting across page/file navigation [Taiga #12469](https://tree.taiga.io/project/penpot/issue/12469)
-- Fix shadow color validation [Github #7705](https://github.com/penpot/penpot/pull/7705)
-- Fix exception on selection blend-mode using keyboard [Github #7710](https://github.com/penpot/penpot/pull/7710)
-- Fix crash when using decimal (floating-point) values for X/Y or width/height [Taiga #12543](https://tree.taiga.io/project/penpot/issue/12543)
-
-## 2.10.1
-
-### :sparkles: New features & Enhancements
-
-- Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
-
-### :bug: Bugs fixed
-
-- Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
-
-## 2.10.0
-
-### :rocket: Epics and highlights
-
-- Variants
-
-### :boom: Breaking changes & Deprecations
-
-### :heart: Community contributions (Thank you!)
-
-### :sparkles: New features & Enhancements
-
-- Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
-- Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
-- Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
-- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
-- New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
-- New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
-- New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
-- New letter spacing token [Taiga #10940](https://tree.taiga.io/project/penpot/us/10940)
-- New font weight token [Taiga #10939](https://tree.taiga.io/project/penpot/us/10939)
-- Upgrade Node to v22.18.0 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Upgrade the base docker image for penpot frontend to v1.29.1 [Github #7283](https://github.com/penpot/penpot/pull/7283)
-- Create variant from an existing component [Taiga #2088](https://tree.taiga.io/project/penpot/us/2088)
-- Create variant from an existing variant [Taiga #8282](https://tree.taiga.io/project/penpot/us/8282)
-- Actions over a component with variants [Taiga #10503](https://tree.taiga.io/project/penpot/us/10503)
-- Create a variant by dragging a component into a component with variants [Taiga #8134](https://tree.taiga.io/project/penpot/us/8134)
-- Transform a variant into an individual component [Taiga #8141](https://tree.taiga.io/project/penpot/us/8141)
-- Delete variant [Taiga #6890](https://tree.taiga.io/project/penpot/us/6890)
-- Restore an orphaned copy of a variant [Taiga #10446](https://tree.taiga.io/project/penpot/us/10446)
-- Add, Edit & Delete variant properties name and value [Taiga #6892](https://tree.taiga.io/project/penpot/us/6892)
-- Retrieve variants [Taiga #6888](https://tree.taiga.io/project/penpot/us/6888)
-- Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
-- Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
-- Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
-- Fix problem with component swapping panel [Taiga #12175](https://tree.taiga.io/project/penpot/issue/12175)
-
-### :bug: Bugs fixed
-
-- Display strokes information in inspect tab [Taiga #11154](https://tree.taiga.io/project/penpot/issue/11154)
-- Fix problem with booleans selection [Taiga #11627](https://tree.taiga.io/project/penpot/issue/11627)
-- Fix missing font when copy&paste a chunk of text [Taiga #11522](https://tree.taiga.io/project/penpot/issue/11522)
-- Fix bad swap slot after two swaps [Taiga #11659](https://tree.taiga.io/project/penpot/issue/11659)
-- Fix missing package for the `penpot_exporter` Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
-- Fix issue where multiple dropdown menus could be opened simultaneously on the dashboard page [Taiga #11500](https://tree.taiga.io/project/penpot/issue/11500)
-- Fix font size/variant not updated when editing a text [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11552)
-- Fix issue where Alt + arrow keys shortcut interferes with letter-spacing when moving text layers [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11771)
-- Fix consistency issues on how font variants are visualized [Taiga #11499](https://tree.taiga.io/project/penpot/us/11499)
-- Fix parsing rx and ry SVG values for rect radius [Taiga #11861](https://tree.taiga.io/project/penpot/issue/11861)
-- Fix misleading affordance in saved versions [Taiga #11887](https://tree.taiga.io/project/penpot/issue/11887)
-- Fix pasting RTF text crashes penpot [Taiga #11717](https://tree.taiga.io/project/penpot/issue/11717)
-- Fix navigation arrows in Libraries & Templates carousel [Taiga #10609](https://tree.taiga.io/project/penpot/issue/10609)
-- Fix applying tokens with zero value to size [Taiga #11618](https://tree.taiga.io/project/penpot/issue/11618)
-- Fix typo [Taiga #11969](https://tree.taiga.io/project/penpot/issue/11969)
-- Fix typo [Taiga #11970](https://tree.taiga.io/project/penpot/issue/11970)
-- Fix typos [Taiga #11971](https://tree.taiga.io/project/penpot/issue/11971)
-- Fix inconsistent naming for "Flatten" [Taiga #8371](https://tree.taiga.io/project/penpot/issue/8371)
-- Layout item tokens should be unapplied when moving out of a layout [Taiga #11012](https://tree.taiga.io/project/penpot/issue/11012)
-- Fix incorrect date displayed for support plan [Taiga #11986](https://tree.taiga.io/project/penpot/issue/11986)
-- Fix can't import 'borderWidth' type token [#132](https://github.com/tokens-studio/penpot/issues/132)
-- Fix moving elements up or down while pressing alt [Taiga Issue #11992](https://tree.taiga.io/project/penpot/issue/11992)
-- Fix conflicting shortcuts (remove dec/inc line height and letter spacing) [Taiga #12102](https://tree.taiga.io/project/penpot/issue/12102)
-- Fix conflicting shortcuts (remove text-align shortcuts) [Taiga #12047](https://tree.taiga.io/project/penpot/issue/12047)
-- Fix export file with empty tokens library [Taiga #12137](https://tree.taiga.io/project/penpot/issue/12137)
-- Fix context menu on spacing tokens [Taiga #12141](https://tree.taiga.io/project/penpot/issue/12141)
-
-## 2.9.0
+## 2.8.0 (Next / Unreleased)
 
 ### :rocket: Epics and highlights
 
 ### :boom: Breaking changes & Deprecations
 
-### :heart: Community contributions (Thank you!)
+**Breaking changes on penpot library:**
 
-- Clarify message when inviting existing team members to make it more user-friendly and clear which invitations will be sent. [Taiga #11441](https://tree.taiga.io/project/penpot/issue/11441) by [@iprithvitharun](https://github.com/iprithvitharun)
-- Update email change confirmation message for clarity and correct grammar. [GitHub #6786](https://github.com/penpot/penpot/issues/6786) by [@iprithvitharun](https://github.com/iprithvitharun)
+- Change the signature of the `addPage` method: it now accepts an object (as a single argument) where you can pass `id`,
+  `name`, and `background` props (instead of the previous positional arguments)
+- Rename the `file.createRect` method to `file.addRect`
+- Rename the `file.createCircle` method to `file.addCircle`
+- Rename the `file.createPath` method to `file.addPath`
+- Rename the `file.createText` method to `file.addText`
+- Rename `file.startComponent` to `file.addComponent` (to preserve the naming style)
+- Rename `file.createComponentInstance` to `file.addComponentInstance` (to preserve the naming style)
+- Rename `file.lookupShape` to `file.getShape`
+- Rename `file.asMap` to `file.toMap`
+- Remove `file.updateLibraryColor` (use `file.addLibraryColor` if you just need to replace a color)
+- Remove `file.deleteLibraryColor` (this library is intended to build files)
+- Remove `file.updateLibraryTypography` (use `file.addLibraryTypography` if you just need to replace a typography)
+- Remove `file.deleteLibraryTypography` (this library is intended to build files)
+- Remove `file.add/update/deleteLibraryMedia` (they are no longer supported by Penpot and have been replaced by components)
+- Remove `file.deleteObject` (this library is intended to build files)
+- Remove `file.updateObject` (this library is intended to build files)
+- Remove `file.finishComponent` (it is no longer necessary; see below for more details on component creation changes)
+- Change the `file.getCurrentPageId` function to a read-only `file.currentPageId` property
+- Add `file.currentFrameId` read-only property
+- Add `file.lastId` read-only property
 
-### :sparkles: New features & Enhancements
-
-- Add visual indicator for new comments in the workspace [Taiga #11328](https://tree.taiga.io/project/penpot/issue/11328)
-- On components overrides, separate the content of the text from the rest of properties [Taiga #7434](https://tree.taiga.io/project/penpot/us/7434)
-- Improve dashboard's sidebar [Taiga #10700](https://tree.taiga.io/project/penpot/us/10700)
-- Change "Save color" button to primary button [Taiga #9410](https://tree.taiga.io/project/penpot/issue/9410)
-- Support for exif rotated images [GitHub #6767](https://github.com/penpot/penpot/issues/6767)
-- Display Blend Mode and Layer Opacity properties in the Inspect tab [Taiga #11283](https://tree.taiga.io/project/penpot/issue/11283)
-- Provide CSS `mix-blend-mode` property in code editor when present on shape [Taiga #11282](https://tree.taiga.io/project/penpot/issue/11282)
-- Add the option to import tokens in a .zip file. [Taiga #11378](https://tree.taiga.io/project/penpot/us/11378)
-- New typography token type - font size token [Taiga #10938](https://tree.taiga.io/project/penpot/us/10938)
-- Hide bounding box while editing visual effects [Taiga #11576](https://tree.taiga.io/project/penpot/issue/11576)
-- Improved text layer resizing: Allow double-click on text bounding box to set auto-width/auto-height [Taiga #11577](https://tree.taiga.io/project/penpot/issue/11577)
-- Improve text layer auto-resize: auto-width switches to auto-height on horizontal resize, and only switches to fixed on vertical resize [Taiga #11578](https://tree.taiga.io/project/penpot/issue/11578)
-- Add the ability to show login dialog on profile settings [Github #6871](https://github.com/penpot/penpot/pull/6871)
-- Improve the application of tokens with object specific tokens [Taiga #10209](https://tree.taiga.io/project/penpot/us/10209)
-- Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
-- Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
-
-### :bug: Bugs fixed
-
-- Copying font size does not copy the unit [Taiga #11143](https://tree.taiga.io/project/penpot/issue/11143)
-- Fix text-decoration line-through that displays a wrong property value [Taiga #11145](https://tree.taiga.io/project/penpot/issue/11145)
-- Fix display error message on register form [Taiga #11444](https://tree.taiga.io/project/penpot/issue/11444)
-- Fix toggle focus mode did not restore viewport and selection upon exit [GitHub #6280](https://github.com/penpot/penpot/issues/6820)
-- Fix problem when creating a layout from an existing layout [Taiga #11554](https://tree.taiga.io/project/penpot/issue/11554)
-- Fix title button from Title Case to Capitalize [Taiga #11476](https://tree.taiga.io/project/penpot/issue/11476)
-- Fix touchpad swipe leading to navigating back/forth [GitHub #4246](https://github.com/penpot/penpot/issues/4246)
-- Keep color data when copying from info tab into CSS [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
-- Update HSL values to modern syntax as defined in W3C CSS Color Module Level 4 [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
-- Fix main component receives focus and is selected when using 'Show Main Component' [Taiga #11402](https://tree.taiga.io/project/penpot/issue/11402)
-- Fix UI theme selection from main menu [Taiga #11567](https://tree.taiga.io/project/penpot/issue/11567)
-- Fix duplicating pages with mainInstance shapes nested inside groups [Taiga #10774](https://tree.taiga.io/project/penpot/issue/10774)
-- Fix ESC key not closing Add/Manage Libraries modal [Taiga #11523](https://tree.taiga.io/project/penpot/issue/11523)
-- Fix copying a shadow color from info tab [Taiga #11211](https://tree.taiga.io/project/penpot/issue/11211)
-- Fix remove color button in the gradient editor [Taiga #11623](https://tree.taiga.io/project/penpot/issue/11623)
-- Fix "Copy as SVG" generates different code from the Inspect panel [Taiga #11519](https://tree.taiga.io/project/penpot/issue/11519)
-- Fix overriden tokens in text copies are not preserved [Taiga #11486](https://tree.taiga.io/project/penpot/issue/11486)
-- Fix problem when changing between flex/grid layout [Taiga #11625](https://tree.taiga.io/project/penpot/issue/11625)
-- Fix opacity on stroke gradients [Taiga #11646](https://tree.taiga.io/project/penpot/issue/11646)
-- Fix change from gradient to solid color [Taiga #11648](https://tree.taiga.io/project/penpot/issue/11648)
-- Fix the context menu always closes after any action [Taiga #11624](https://tree.taiga.io/project/penpot/issue/11624)
-- Fix X & Y position do not sincronize with tokens [Taiga #11617](https://tree.taiga.io/project/penpot/issue/11617)
-- Fix tooltip position after first time [Taiga #11688](https://tree.taiga.io/project/penpot/issue/11688)
-- Fix inconsistent ordering of pinned projects on dashboard sidebar [Taiga #11674](https://tree.taiga.io/project/penpot/issue/11674)
-- Fix export button width on inspect tab [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
-- Fix stroke width token application [Taiga #11724](https://tree.taiga.io/project/penpot/issue/11724)
-- Fix number token application on shape [Taiga #11331](https://tree.taiga.io/project/penpot/task/11331)
-- Fix auto height is fixed in the HTML inspect tab for text elements [Taiga #11680](https://tree.taiga.io/project/penpot/task/11680)
-
-## 2.8.1
-
-### :bug: Bugs fixed
-
-- Fix unexpected exception on processing old texts [Github #6889](https://github.com/penpot/penpot/pull/6889)
-- Fix error on inspect tab when selecting multiple shapes [Taiga #11655](https://tree.taiga.io/project/penpot/issue/11655)
-- Fix missing package for the penport_exporter Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
-
-## 2.8.0
-
-### :rocket: Epics and highlights
-
-### :boom: Breaking changes & Deprecations
-
-**Penpot Library**
-
-The initial prototype is completly reworked to provide a more consistent API
-and to have proper validation and params decoding. All the details can be found
-on [its own changelog](library/CHANGES.md)
-
-**Penpot migrate from Redis to Valkey**
-
-As [Valkey](https://valkey.io/) is an opne-souce fork of [Redis](https://redis.io/)
-version 7.2.4, this version of Penpot will be compatible with Redis but may diverge
-in future versions. Therefore, **migration from Redis to ValKey is recommended for all
-on-premises instances** that want to keep up to date.
+There are also relevant semantic changes in how components should be created: this refactor removes
+all notions of the old components (v1). Since v2, the shapes that are part of a component live on a
+page. So, from now on, to create a component, you should first create a frame, then add shapes
+and/or groups to that frame, and then create a component by declaring that frame as the component
+root.
 
 ### :heart: Community contributions (Thank you!)
 
-- Add Serbian language [GitHub #5002](https://github.com/penpot/penpot/issues/5002) by [crnobog69](https://github.com/crnobog69)
-
-### :sparkles: New features & Enhancements
+### :sparkles: New features
 
 - Optimize profile setup flow for better user experience [Taiga #10028](https://tree.taiga.io/project/penpot/us/10028)
-- Rewrite path shape data PathData encoding [Taiga #8542](https://tree.taiga.io/project/penpot/us/8542?milestone=441308)
-- Update base image for Docker Backend and Exporter to Ubuntu 24.04
-- Update base image for Docker Frontend to Nginx 1.28.0
-- Allow multi file token import [Github #27](https://github.com/tokens-studio/penpot/issues/27)
-- Create `input*` wrapper component, and `label*`, `input-field*` and `hint-message*` components [Taiga #10713](https://tree.taiga.io/project/penpot/us/10713)
-- Deselect layers (and path nodes) with Ctrl+Shift+Drag [Github #2509](https://github.com/penpot/penpot/issues/2509)
-- Copy to SVG from contextual menu [Github #838](https://github.com/penpot/penpot/issues/838)
-- Add styles for Inkeep Chat at workspace [Taiga #10708](https://tree.taiga.io/project/penpot/us/10708)
-- Add configuration for air gapped installations with Docker
-- Support system color scheme [Github #5030](https://github.com/penpot/penpot/issues/5030)
-- Persist ruler visibility across files and reloads [GitHub #4586](https://github.com/penpot/penpot/issues/4586)
-- Update google fonts (at 2025/05/19) [Taiga 10792](https://tree.taiga.io/project/penpot/us/10792)
-- Add tooltip component to DS [Taiga 9220](https://tree.taiga.io/project/penpot/us/9220)
-- Allow multi file token export [Taiga #10144](https://tree.taiga.io/project/penpot/us/10144)
-- Fix problem when double click on hidden shapes [Taiga #11314](https://tree.taiga.io/project/penpot/issue/11314)
 
 ### :bug: Bugs fixed
 
-- Fix getCurrentUser for plugins api [Taiga #11057](https://tree.taiga.io/project/penpot/issue/11057)
-- Fix spacing / sizes of different elements in the measurements section of the design tab [Taiga #11076](https://tree.taiga.io/project/penpot/issue/11076)
-- Fix selection of short paths [Github #4472](https://github.com/penpot/penpot/issues/4472)
-- Fix element positioning on the right side to adjust to grid [#11073](https://tree.taiga.io/project/penpot/issue/11073)
-- Fix palette is over sidebar [#11160](https://tree.taiga.io/project/penpot/issue/11160)
-- Fix font size input not displaying "mixed" when multiple texts are selected [Taiga #11177](https://tree.taiga.io/project/penpot/issue/11177)
-- Misalignments at Create account [Taiga #11315](https://tree.taiga.io/project/penpot/issue/11315)
-- Fix issue with importing files where flex/grid is used [Taiga #11334](https://tree.taiga.io/project/penpot/issue/11334)
-- Fix wrong color in the export progress bar [Taiga #11299](https://tree.taiga.io/project/penpot/issue/11299)
-- Fix right sidebar width overflow on long layer names [Taiga #11212](https://tree.taiga.io/project/penpot/issue/11212)
-- Fix comment icon fill [Taiga #11388](https://tree.taiga.io/project/penpot/issue/11388)
-- Fix gap on radio-buttons component [Taiga #11360](https://tree.taiga.io/project/penpot/issue/11360)
-- Fix button width [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
-- Fix mixed letter spacing and line height [Taiga #11178](https://tree.taiga.io/project/penpot/issue/11178)
-- Fix snap nodes shortcut [Taiga #11054](https://tree.taiga.io/project/penpot/issue/11054)
-- Fix changing a text property in a text layer does not unapply the previously applied token in the same property [Taiga #11337](https://tree.taiga.io/project/penpot/issue/11337)
-- Fix shortcut error pressing G+W from the View Mode [Taiga #11061](https://tree.taiga.io/project/penpot/issue/11061)
-- Fix entering long project name [Taiga #11417](https://tree.taiga.io/project/penpot/issue/11417)
-- Fix slow color picker [Taiga #11019](https://tree.taiga.io/project/penpot/issue/11019)
-- Fix tooltip position after click [Taiga #11405](https://tree.taiga.io/project/penpot/issue/11405)
-- Fix incorrect media translation on paste text with fill images [Github #6845](https://github.com/penpot/penpot/pull/6845)
 
-## 2.7.2
-
-### :bug: Bugs fixed
-
-- Update plugins runtime [Github #6604](https://github.com/penpot/penpot/pull/6604)
-- Backport from develop a minor fix that enables import of files
-  generated by penpot library [Github #6614](https://github.com/penpot/penpot/pull/6614)
-- Fix copy in error message [GitHub #6615](https://github.com/penpot/penpot/pull/6615)
-- Fix url on invitation link [Taiga #11284](https://tree.taiga.io/project/penpot/issue/11284)
-
-## 2.7.1
-
-### :bug: Bugs fixed
-
-- Fix incorrect handling of strokes with images on importing files
-- Fix tokens disappearing after manual additions [Taiga #11063](https://tree.taiga.io/project/penpot/issue/11063)
-
-## 2.7.0
+## 2.7.0 (Unreleased)
 
 ### :rocket: Epics and highlights
 
@@ -326,10 +63,10 @@ on-premises instances** that want to keep up to date.
 - Duplicate token sets [Taiga #10694](https://tree.taiga.io/project/penpot/issue/10694)
 - Add set selection in create Token themes flow [Taiga #10746](https://tree.taiga.io/project/penpot/issue/10746)
 - Display indicator on not active sets [Taiga #10668](https://tree.taiga.io/project/penpot/issue/10668)
+- Create `input*` wrapper component, and `label*`, `input-field*` and `hint-message*` components [Taiga #10713](https://tree.taiga.io/project/penpot/us/10713)
 
 ### :bug: Bugs fixed
 
-- Fix "at" icon to match all icons on app [Taiga #11136](https://tree.taiga.io/project/penpot/issue/11136)
 - Fix problem in viewer with the back button [Taiga #10907](https://tree.taiga.io/project/penpot/issue/10907)
 - Fix resize bar background on tokens panel [Taiga #10811](https://tree.taiga.io/project/penpot/issue/10811)
 - Fix shortcut for history version panel [Taiga #11006](https://tree.taiga.io/project/penpot/issue/11006)
@@ -356,13 +93,6 @@ on-premises instances** that want to keep up to date.
 - Fix cannot rename Design Token Sets when group of same name exists [Taiga Issue #10773](https://tree.taiga.io/project/penpot/issue/10773)
 - Fix problem when duplicating grid layout [Github #6391](https://github.com/penpot/penpot/issues/6391)
 - Fix issue that makes workspace shortcuts stop working [Taiga #11062](https://tree.taiga.io/project/penpot/issue/11062)
-- Fix problem while syncing library colors and typographies [Taiga #11068](https://tree.taiga.io/project/penpot/issue/11068)
-- Fix problem with path edition of shapes [Taiga #9496](https://tree.taiga.io/project/penpot/issue/9496)
-- Fix exception on paste invalid html [Taiga #11047](https://tree.taiga.io/project/penpot/issue/11047)
-- Fix share button being displayed with no permissions [Taiga #11086](https://tree.taiga.io/project/penpot/issue/11086)
-- Fix inline styles in code tab [Taiga Issue #7583](https://tree.taiga.io/project/penpot/issue/7583)
-- Fix exception on returning openapi.json
-- Fix json encoding of TokensLib [Taiga #10994](https://tree.taiga.io/project/penpot/issue/10994)
 
 ## 2.6.2
 
@@ -439,6 +169,7 @@ on-premises instances** that want to keep up to date.
 - Add character limitation to asset inputs [Taiga #10669](https://tree.taiga.io/project/penpot/issue/10669)
 - Fix Storybook link 'list of all available icons' wrong path [Taiga #10705](https://tree.taiga.io/project/penpot/issue/10705)
 
+
 ## 2.5.4
 
 ### :heart: Community contributions (Thank you!)
@@ -483,7 +214,7 @@ on-premises instances** that want to keep up to date.
 
 ### :boom: Breaking changes & Deprecations
 
-Although this is not a breaking change, we believe it's important to highlight it in this
+Although this is not a breaking change, we believe it’s important to highlight it in this
 section:
 
 This release includes a fix for an internal bug in Penpot that caused incorrect handling
@@ -491,9 +222,9 @@ of media assets (e.g., fill images). The issue has been resolved since version 2
 no new incorrect references will be generated. However, existing files may still contain
 incorrect references.
 
-To address this, we've provided a script to correct these references in existing files.
+To address this, we’ve provided a script to correct these references in existing files.
 
-While having incorrect references generally doesn't result in visible issues, there are
+While having incorrect references generally doesn’t result in visible issues, there are
 rare cases where it can cause problems. For example, if a component library (containing
 images) is deleted, and that library is being used in other files, running the FileGC task
 (responsible for freeing up space and performing logical deletions) could leave those
@@ -568,6 +299,7 @@ is a number of cores)
 - Fix missing methods reference on API Docs
 - Fix memory usage issue on file-gc asynchronous task (related to snapshots feature)
 
+
 ## 2.4.1
 
 ### :bug: Bugs fixed
@@ -575,6 +307,7 @@ is a number of cores)
 - Fix error when importing files with touched components [Taiga #9625](https://tree.taiga.io/project/penpot/issue/9625)
 - Fix problem when changing color libraries [Plugins #184](https://github.com/penpot/penpot-plugins/issues/184)
 
+
 ## 2.4.0
 
 ### :rocket: Epics and highlights
@@ -628,6 +361,7 @@ is a number of cores)
 
 - Add initial documentation for Kubernetes
 
+
 ## 2.3.1
 
 ### :bug: Bugs fixed
@@ -635,6 +369,7 @@ is a number of cores)
 - Fix unexpected issue on interaction between plugins sandbox and
   internal impl of promise
 
+
 ## 2.3.0
 
 ### :rocket: Epics and highlights
@@ -660,6 +395,7 @@ is a number of cores)
 
   You can enable it with the `enable-feature-text-editor-v2` configuration flag.
 
+
 ### :bug: Bugs fixed
 
 - Fix problem with constraints buttons [Taiga #8465](https://tree.taiga.io/project/penpot/issue/8465)
@@ -699,8 +435,8 @@ is a number of cores)
 ### :boom: Breaking changes & Deprecations
 
 - Removed "merge assets" option when exporting ".svg + .json" files. After the components changes the option wasn't
-  working properly and we're planning to change the format soon. We think it's better to deprecate the option for the
-  time being.
+working properly and we're planning to change the format soon. We think it's better to deprecate the option for the
+time being.
 
 ### :heart: Community contributions (Thank you!)
 
@@ -716,7 +452,7 @@ is a number of cores)
   freeing up space in the database. It can be enabled with the
   `enable-enable-tiered-file-data-storage` flag.
 
-  _(On-Premise feature, EXPERIMENTAL)._
+  *(On-Premise feature, EXPERIMENTAL).*
 
 - **JSON Interoperability for HTTP API** [Taiga #8372](https://tree.taiga.io/project/penpot/us/8372)
 
@@ -759,7 +495,7 @@ is a number of cores)
 
 - **Design System**
 
-  We implemented and subbed in new components from our Design System: `loader*` ([Taiga #8355](https://tree.taiga.io/project/penpot/task/8355)) and `tab-switcher*` ([Taiga #8518](https://tree.taiga.io/project/penpot/task/8518)).
+  We implemented and subbed in new components from our Design System: `loader*` ([Taiga #8355](https://tree.taiga.io/project/penpot/task/8355))  and `tab-switcher*` ([Taiga #8518](https://tree.taiga.io/project/penpot/task/8518)).
 
 - **Storybook** [Taiga #6329](https://tree.taiga.io/project/penpot/us/6329)
 
@@ -814,11 +550,11 @@ is a number of cores)
 
 ### :sparkles: New features
 
-- Consolidate templates new order and naming [Taiga #8392](https://tree.taiga.io/project/penpot/task/8392)
+- Consolidate templates new order and naming  [Taiga #8392](https://tree.taiga.io/project/penpot/task/8392)
 
 ### :bug: Bugs fixed
 
-- Fix the "search" label in translations [Taiga #8402](https://tree.taiga.io/project/penpot/issue/8402)
+- Fix the “search” label in translations [Taiga #8402](https://tree.taiga.io/project/penpot/issue/8402)
 - Fix pencil loader [Taiga #8348](https://tree.taiga.io/project/penpot/issue/8348)
 - Fix several issues on the OIDC.
 - Fix regression on the `email-verification` flag [Taiga #8398](https://tree.taiga.io/project/penpot/issue/8398)
@@ -898,21 +634,22 @@ is a number of cores)
 - Fix color palette sorting [Taiga #7458](https://tree.taiga.io/project/penpot/issue/7458)
 - Fix style scoping problem with imported SVG [Taiga #7671](https://tree.taiga.io/project/penpot/issue/7671)
 
+
 ## 2.0.1
 
 ### :bug: Bugs fixed
 
 - Fix different issues related to components v2 migrations including [Github #4443](https://github.com/penpot/penpot/issues/4443)
 
+
 ## 2.0.0 - I Just Can't Get Enough
 
 ### :rocket: Epics and highlights
-
 - Grid CSS layout [Taiga #4915](https://tree.taiga.io/project/penpot/epic/4915)
 - UI redesign [Taiga #4958](https://tree.taiga.io/project/penpot/epic/4958)
 - New components System [Taiga #2662](https://tree.taiga.io/project/penpot/epic/2662)
 - Swap components [Taiga #1331](https://tree.taiga.io/project/penpot/us/1331)
-- Images as fill [Taiga #2983](https://tree.taiga.io/project/penpot/us/2983)
+- Images as fill  [Taiga #2983](https://tree.taiga.io/project/penpot/us/2983)
 - HTML code generation [Taiga #5277](https://tree.taiga.io/project/penpot/us/5277)
 - Light and dark themes [Taiga #2287](https://tree.taiga.io/project/penpot/us/2287)
 
@@ -921,9 +658,9 @@ is a number of cores)
 - New strokes default to inside border [Taiga #6847](https://tree.taiga.io/project/penpot/issue/6847)
 - Change default z ordering on layers in flex layout. The previous behavior was inconsistent with how HTML works and we changed it to be more consistent. Previous layers that overlapped could be hidden, the fastest way to fix this is changing the z-index property but a better way is to change the order of your layers.
 
-### :heart: Community contributions (Thank you!)
 
-- New Hausa, Yoruba and Igbo translations and update translation files (by All For Tech Empowerment Foundation) [Taiga #6950](https://tree.taiga.io/project/penpot/us/6950), [Taiga #6534](https://tree.taiga.io/project/penpot/us/6534)
+### :heart: Community contributions (Thank you!)
+- New Hausa, Yoruba and Igbo translations and  update translation files (by All For Tech Empowerment Foundation) [Taiga #6950](https://tree.taiga.io/project/penpot/us/6950), [Taiga #6534](https://tree.taiga.io/project/penpot/us/6534)
 - Hide bounding-box when editing shape (by @VasilevsVV) [#3930](https://github.com/penpot/penpot/pull/3930)
 - CTRL + "+" to zoom into canvas instead of browser (by @audriu) [#3848](https://github.com/penpot/penpot/pull/3848)
 - Add dev deps.edn in the project root (by @PEZ) [#3794](https://github.com/penpot/penpot/pull/3794)
@@ -932,7 +669,6 @@ is a number of cores)
 - Typo (by StephanEggermont) [#157](https://github.com/penpot/penpot-docs/pull/157)
 
 ### :sparkles: New features
-
 - Send comments with Ctrl+Enter / Cmd + Enter [Taiga #6085](https://tree.taiga.io/project/penpot/issue/6085)
 - Select through stroke only rectangle [Taiga #5484](https://tree.taiga.io/project/penpot/issue/5484)
 - Stroke default position [Taiga #6847](https://tree.taiga.io/project/penpot/issue/6847)
@@ -1000,7 +736,6 @@ is a number of cores)
 - [REDESIGN] Onboarding slides [Taiga #6678](https://tree.taiga.io/project/penpot/us/6678)
 
 ### :bug: Bugs fixed
-
 - Fix pixelated thumbnails [Github #3681](https://github.com/penpot/penpot/issues/3681), [Github #3661](https://github.com/penpot/penpot/issues/3661)
 - Fix problem with not applying colors to boards [Github #3941](https://github.com/penpot/penpot/issues/3941)
 - Fix problem with path editor undoing changes [Github #3998](https://github.com/penpot/penpot/issues/3998)
@@ -1009,7 +744,7 @@ is a number of cores)
 - Selecting from Color Palette does not work for board when there is no existing fill [Taiga #6464](https://tree.taiga.io/project/penpot/issue/6464)
 - Color thumbnails are consistently rounded in the inspect code mode [Taiga #5886](https://tree.taiga.io/project/penpot/issue/5886)
 - Adding vector path points before first point of existing open path not working [Taiga #6593](https://tree.taiga.io/project/penpot/issue/6593)
-- Some image formats include the extension when importing [Taiga #5485](https://tree.taiga.io/project/penpot/issue/5485)
+- Some image formats include the extension when importing  [Taiga #5485](https://tree.taiga.io/project/penpot/issue/5485)
 - Gradient color tool doesn't work properly with flipped items [Taiga #6485](https://tree.taiga.io/project/penpot/issue/6485)
 - [TEXT] Align options are not shown when several text are selected [Taiga #5948](https://tree.taiga.io/project/penpot/issue/5948)
 - [VIEW MODE] Comments not working properly on multiple pages [Taiga #6281](https://tree.taiga.io/project/penpot/issue/6281)
@@ -1053,7 +788,7 @@ is a number of cores)
 
 ### :sparkles: New features
 
-- Improve selected colors [Taiga #5805](https://tree.taiga.io/project/penpot/us/5805)
+- Improve selected colors [Taiga #5805]( https://tree.taiga.io/project/penpot/us/5805)
 
 ### :bug: Bugs fixed
 
@@ -1088,6 +823,7 @@ is a number of cores)
 - Fix deleted pages comments shown in right sidebar [Taiga #5648](https://tree.taiga.io/project/penpot/us/5648)
 - Fix tooltip on toggle visibility and toggle lock buttons [Taiga #5141](https://tree.taiga.io/project/penpot/issue/5141)
 
+
 ## 1.19.1
 
 ### :bug: Bugs fixed
@@ -1201,6 +937,7 @@ is a number of cores)
 
 - Update google fonts catalog (at 2023/07/06) [Taiga #5592](https://tree.taiga.io/project/penpot/issue/5592)
 
+
 ### :heart: Community contributions by (Thank you!)
 
 - Update Typography palette order (by @akshay-gupta7) [Github #3156](https://github.com/penpot/penpot/pull/3156)
@@ -1354,14 +1091,12 @@ is a number of cores)
 - Fix problem with opacity in imported SVG's [Taiga #4923](https://tree.taiga.io/project/penpot/issue/4923)
 
 ### :heart: Community contributions by (Thank you!)
-
 - To @ondrejkonec: for contributing to the code with:
 - Refactor CSS variables [Github #2948](https://github.com/penpot/penpot/pull/2948)
 
 ## 1.17.3
 
 ### :bug: Bugs fixed
-
 - Fix copy and paste very nested inside itself [Taiga #4848](https://tree.taiga.io/project/penpot/issue/4848)
 - Fix custom fonts not rendered correctly [Taiga #4874](https://tree.taiga.io/project/penpot/issue/4874)
 - Fix problem with shadows and blur on multiple selection
@@ -1394,7 +1129,6 @@ is a number of cores)
 ## 1.17.1
 
 ### :bug: Bugs fixed
-
 - Fix components groups items show the component name in list mode [Taiga #4770](https://tree.taiga.io/project/penpot/issue/4770)
 - Fix typing CMD+Z on MacOS turns the cursor into a Zoom cursor [Taiga #4778](https://tree.taiga.io/project/penpot/issue/4778)
 - Fix white space on small screens [Taiga #4774](https://tree.taiga.io/project/penpot/issue/4774)
@@ -1509,7 +1243,7 @@ is a number of cores)
 
 ### :boom: Breaking changes & Deprecations
 
-- Removed the support for v2 internal file data blob format. This
+- Removed the support for v2 internal file data blob format.  This
   version has never been documented nor set as default value so
   technically this is not a breaking change because we are removing
   a "private API".
@@ -1614,6 +1348,7 @@ is a number of cores)
 - Fix when ungrouping, the items previously grouped should ALWAYS remain selected [Taiga #4064](https://tree.taiga.io/project/penpot/issue/4064)
 - Change shortcut for "Clear undo" [#2219](https://github.com/penpot/penpot/issues/2219)
 
+
 ## 1.15.2-beta
 
 ### :bug: Bugs fixed
@@ -1697,7 +1432,6 @@ is a number of cores)
 - Fix bringing complete file data when launching the export dialog [Taiga #4006](https://tree.taiga.io/project/penpot/issue/4006)
 
 ### :arrow_up: Deps updates
-
 ### :heart: Community contributions by (Thank you!)
 
 ## 1.14.2-beta
@@ -1738,10 +1472,10 @@ is a number of cores)
 - Prototype connection should be under the rules [Taiga #3384](https://tree.taiga.io/project/penpot/issue/3384)
 - Fix problem with empty text boxes events [Taiga #3627](https://tree.taiga.io/project/penpot/issue/3627)
 
+
 ## 1.13.5-beta
 
 ### :bug: Bugs fixed
-
 - Fix orientation artboard preset not working with differently sized artboards [Taiga #3548](https://tree.taiga.io/project/penpot/issue/3548)
 - Fix background on export arboards [Taiga #1991](https://tree.taiga.io/project/penpot/issue/1991)
 
@@ -1885,7 +1619,6 @@ is a number of cores)
 - Fix problem when resizing a group with texts with auto-width/height [#3171](https://tree.taiga.io/project/penpot/issue/3171)
 
 ### :arrow_up: Deps updates
-
 ### :heart: Community contributions by (Thank you!)
 
 ## 1.12.4-beta
@@ -1903,7 +1636,7 @@ is a number of cores)
 ### :bug: Bugs fixed
 
 - Fix issue with shift+select to deselect shapes [Taiga #3154](https://tree.taiga.io/project/penpot/issue/3154)
-- Fix issue with drag-select shapes [Taiga #3165](https://tree.taiga.io/project/penpot/issue/3165)
+- Fix issue with drag-select shapes  [Taiga #3165](https://tree.taiga.io/project/penpot/issue/3165)
 - Fix issue on password persistence after registration process on private instances
 
 ## 1.12.2-beta
@@ -1921,6 +1654,7 @@ is a number of cores)
 - Fix length of names in sidebar [Taiga #2962](https://tree.taiga.io/project/penpot/issue/2962)
 - Fix issues on loki integration
 
+
 ## 1.12.0-beta
 
 ### :boom: Breaking changes

CODE_OF_CONDUCT.md

@@ -1,3 +0,0 @@
-# Penpot's Code of Conduct
-
-Check it at: https://help.penpot.app/contributing-guide/coc/

CONTRIBUTING.md

@@ -1,59 +1,62 @@
 # Contributing Guide #
 
 Thank you for your interest in contributing to Penpot. This is a
-generic guide that details how to contribute to the project in a way that
-is efficient for everyone. If you are looking for specific documentation on
-different parts of the platform, please refer to the `docs/` directory,
-or the rendered version at the [Help Center](https://help.penpot.app/).
+generic guide that details how to contribute to Penpot in a way that
+is efficient for everyone. If you want a specific documentation for
+different parts of the platform, please refer to `docs/` directory.
+
 
 ## Reporting Bugs ##
 
 We are using [GitHub Issues](https://github.com/penpot/penpot/issues)
-for our public bugs. We keep a close eye on them and try to make it
+for our public bugs. We keep a close eye on this and try to make it
 clear when we have an internal fix in progress. Before filing a new
 task, try to make sure your problem doesn't already exist.
 
-If you found a bug, please report it, as far as possible, with:
+If you found a bug, please report it, as far as possible with:
 
 - a detailed explanation of steps to reproduce the error
-- the browser and browser version used
-- a dev tools console exception stack trace (if available)
+- a browser and the browser version used
+- a dev tools console exception stack trace (if it is available)
 
-If you found a bug which you think is better to discuss in private (for
-example, security bugs), consider first sending an email to
+If you found a bug that you consider better discuss in private (for
+example: security bugs), consider first send an email to
 `support@penpot.app`.
 
-**We don't have a formal bug bounty program for security reports; this
-is an open source application, and your contribution will be recognized
+**We don't have formal bug bounty program for security reports; this
+is an open source application and your contribution will be recognized
 in the changelog.**
 
 
-## Pull Requests ##
+## Pull requests ##
 
-If you want to propose a change or bug fix via a pull request (PR),
-you should first carefully read the section **Developer's Certificate of
-Origin**. You must also format your code and commits according to the
-instructions below.
+If you want propose a change or bug fix with the Pull-Request system
+firstly you should carefully read the **DCO** section and format your
+commits accordingly.
 
-If you intend to fix a bug, it's fine to submit a pull request right
-away, but we still recommend filing an issue detailing what you're
+If you intend to fix a bug it's fine to submit a pull request right
+away but we still recommend to file an issue detailing what you're
 fixing. This is helpful in case we don't accept that specific fix but
 want to keep track of the issue.
 
-If you want to implement or start working on a new feature, please
-open a **question*- / **discussion*- issue for it. No PR
-will be accepted without a prior discussion about the changes,
-whether it is a new feature, an already planned one, or a quick win.
+If you want to implement or start working in a new feature, please
+open a **question** / **discussion** issue for it. No pull-request
+will be accepted without previous chat about the changes,
+independently if it is a new feature, already planned feature or small
+quick win.
 
-If it is your first PR, you can learn how to proceed from
-[this free video
-series](https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github)
+If is going to be your first pull request, You can learn how from this
+free video series:
+
+https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github
+
+We will use the `easy fix` mark for tag for indicate issues that are
+easy for beginners.
 
-We use the `easy fix` tag to indicate issues that are appropriate for beginners.
 
 ## Commit Guidelines ##
 
-We have very precise rules on how our git commit messages must be formatted.
+We have very precise rules over how our git commit messages can be formatted.
 
 The commit message format is:
 
@@ -68,37 +71,34 @@ The commit message format is:
 Where type is:
 
 - :bug: `:bug:` a commit that fixes a bug
-- :sparkles: `:sparkles:` a commit that adds an improvement
-- :tada: `:tada:` a commit with a new feature
+- :sparkles: `:sparkles:` a commit that an improvement
+- :tada: `:tada:` a commit with new feature
 - :recycle: `:recycle:` a commit that introduces a refactor
 - :lipstick: `:lipstick:` a commit with cosmetic changes
-- :ambulance: `:ambulance:` a commit that fixes a critical bug
+- :ambulance: `:ambulance:` a commit that fixes critical bug
 - :books: `:books:` a commit that improves or adds documentation
-- :construction: `:construction:` a WIP commit
+- :construction: `:construction:`: a wip commit
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
-- :whale: `:whale:` a commit for Docker-related stuff
-- :paperclip: `:paperclip:` a commit with other non-relevant changes
-- :arrow_up: `:arrow_up:` a commit with dependency updates
-- :arrow_down: `:arrow_down:` a commit with dependency downgrades
+- :whale: `:whale:` a commit for docker related stuff
+- :paperclip: `:paperclip:` a commit with other not relevant changes
+- :arrow_up: `:arrow_up:` a commit with dependencies updates
+- :arrow_down: `:arrow_down:` a commit with dependencies downgrades
 - :fire: `:fire:` a commit that removes files or code
-- :globe_with_meridians: `:globe_with_meridians:` a commit that adds or updates
-  translations
 
 More info:
-
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
 Each commit should have:
 
-- A concise subject using the imperative mood.
-- The subject should capitalize the first letter, omit the period
-  at the end, and be no longer than 65 characters.
+- A concise subject using imperative mood.
+- The subject should have capitalized the first letter, without period
+  at the end and no larger than 65 characters.
 - A blank line between the subject line and the body.
-- An entry in the CHANGES.md file if applicable, referencing the
-  GitHub or Taiga issue/user story using these same rules.
+- An entry on the CHANGES.md file if applicable, referencing the
+  github or taiga issue/user-story using the these same rules.
 
 Examples of good commit messages:
 
@@ -111,30 +111,8 @@ Examples of good commit messages:
 - `:ambulance: Fix critical bug on user registration process`
 - `:tada: Add new approach for user registration`
 
-## Formatting and Linting ##
 
-You will want to make sure your code is formatted and linted before submitting
-a PR. We use [cljfmt](https://github.com/weavejester/cljfmt) and
-[clj-kondo](https://github.com/clj-kondo/clj-kondo) for this. After installing
-them on your system, you can run them with:
-
-```bash
-# Check formatting
-yarn fmt:clj:check
-
-# Check and fix formatting
-yarn fmt:clj
-
-# Run the linter
-yarn lint:clj
-```
-
-There are more choices in `package.json`.
-
-Ideally, you should run these commands as git pre-commit hooks. A convenient way
-of defining them is to use [Husky](https://typicode.github.io/husky/#/).
-
-## Code of Conduct ##
+## Code of conduct ##
 
 As contributors and maintainers of this project, we pledge to respect
 all people who contribute through reporting issues, posting feature
@@ -154,11 +132,11 @@ unprofessional conduct.
 
 Project maintainers have the right and responsibility to remove, edit,
 or reject comments, commits, code, wiki edits, issues, and other
-contributions that are not aligned with this Code of Conduct. Project
+contributions that are not aligned to this Code of Conduct. Project
 maintainers who do not follow the Code of Conduct may be removed from
 the project team.
 
-This Code of Conduct applies both within project spaces and in public
+This code of conduct applies both within project spaces and in public
 spaces when an individual is representing the project or its
 community.
 
@@ -167,11 +145,12 @@ may be reported by opening an issue or contacting one or more of the
 project maintainers.
 
 This Code of Conduct is adapted from the Contributor Covenant, version
-1.1.0, available from [http://contributor-covenant.org/version/1/1/0/](http://contributor-covenant.org/version/1/1/0/)
+1.1.0, available from http://contributor-covenant.org/version/1/1/0/
 
-## Developer's Certificate of Origin (DCO)
 
-By submitting code you agree to and can certify the following:
+## Developer's Certificate of Origin (DCO) ##
+
+By submitting code you are agree and can certify the below:
 
     Developer's Certificate of Origin 1.1
 
@@ -199,15 +178,13 @@ By submitting code you agree to and can certify the following:
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
 
-Then, all your code patches (**documentation is excluded**) should
+Then, all your code patches (**documentation are excluded**) should
 contain a sign-off at the end of the patch/commit description body. It
-can be automatically added by adding the `-s` parameter to `git commit`.
+can be automatically added on adding `-s` parameter to `git commit`.
 
-This is an example of what the line should look like:
+This is an example of the aspect of the line:
 
-```
-Signed-off-by: Andrey Antukh <niwi@niwi.nz>
-```
+	Signed-off-by: Andrey Antukh <niwi@niwi.nz>
 
 Please, use your real name (sorry, no pseudonyms or anonymous
 contributions are allowed).

README.md

@@ -34,8 +34,7 @@
 
 <br />
 
-[Penpot video](https://github.com/user-attachments/assets/7c67fd7c-04d3-4c9b-88ec-b6f5e23f8332
-)
+[Penpot video](https://github.com/penpot/penpot/assets/5446186/b8ad0764-585e-4ddc-b098-9b4090d337cc)
 
 <br />
 
@@ -77,23 +76,27 @@ Provide your team or organization with a completely owned collaborative design t
 ### Integrations ###
 Penpot offers integration into the development toolchain, thanks to its support for webhooks and an API accessible through access tokens.
 
-###  Building Design Systems: design tokens, components and variants  ###
-Penpot brings design systems to code-minded teams: a single source of truth with native Design Tokens, Components, and Variants for scalable, reusable, and consistent UI across projects and platforms.
+### What’s great for design ###
+With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
+
+### Design Tokens ###
+With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
 
 
 <br />
 
 <p align="center">
- <img src="https://github.com/user-attachments/assets/cce75ad6-f783-473f-8803-da9eb8255fef">
+  <img src="https://img.plasmic.app/img-optimizer/v1/img?src=https%3A%2F%2Fimg.plasmic.app%2Fimg-optimizer%2Fv1%2Fimg%2F9dd677c36afb477e9666ccd1d3f009ad.png" alt="Open Source" style="width: 65%;">
 </p>
 
 <br />
 
 ## Getting started ##
 
-Penpot is the only design & prototype platform that is deployment agnostic. You can use it in our [SAAS](https://design.penpot.app) or deploy it anywhere.
+### Install with Elestio ###
+Penpot is the only design & prototype platform that is deployment agnostic. You can use it or deploy it anywhere.
 
-Learn how to install it with Docker, Kubernetes, Elestio or other options on [our website](https://penpot.app/self-host).
+Learn how to install it with Elestio and Docker, or other options on [our website](https://penpot.app/self-host).
 <br />
 
 <p align="center">
@@ -125,12 +128,6 @@ You will find the following categories:
 </p>
 <br />
 
-### Code of Conduct ###
-
-Anyone who contributes to Penpot, whether through code, in the community, or at an event, must adhere to the
-[code of conduct](https://help.penpot.app/contributing-guide/coc/) and foster a positive and safe environment.
-
-
 ## Contributing ##
 
 Any contribution will make a difference to improve Penpot. How can you get involved?

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.2"}
+  org.clojure/clojure {:mvn/version "1.12.0"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.7-4"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.6-9"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,41 +17,34 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.8.1.RELEASE"}
-  ;; Minimal dependencies required by lettuce, we need to include them
-  ;; explicitly because clojure dependency management does not support
-  ;; yet the BOM format.
-  io.micrometer/micrometer-core {:mvn/version "1.14.2"}
-  io.micrometer/micrometer-observation {:mvn/version "1.14.2"}
-
+  io.lettuce/lettuce-core {:mvn/version "6.5.2.RELEASE"}
   java-http-clj/java-http-clj {:mvn/version "0.4.3"}
-  com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.8"
-   :git/sha "1d1b33f"
+  {:git/tag "v11.4"
+   :git/sha "ce50d42"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
   com.github.seancorfield/next.jdbc
-  {:mvn/version "1.3.1070"}
+  {:mvn/version "1.3.994"}
+  metosin/reitit-core {:mvn/version "0.7.2"}
+  nrepl/nrepl {:mvn/version "1.3.1"}
+  cider/cider-nrepl {:mvn/version "0.52.0"}
 
-  metosin/reitit-core {:mvn/version "0.9.1"}
-  nrepl/nrepl {:mvn/version "1.4.0"}
+  org.postgresql/postgresql {:mvn/version "42.7.5"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.48.0.0"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.7"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}
-
-  com.zaxxer/HikariCP {:mvn/version "7.0.2"}
+  com.zaxxer/HikariCP {:mvn/version "6.2.1"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.2"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
 
-  org.jsoup/jsoup {:mvn/version "1.21.2"}
+  org.jsoup/jsoup {:mvn/version "1.18.3"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -61,12 +54,12 @@
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
-  dawran6/emoji {:mvn/version "0.2.0"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.4"}
+  dawran6/emoji {:mvn/version "0.1.5"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.2"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.33.10"}}
+  software.amazon.awssdk/s3 {:mvn/version "2.28.26"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -81,14 +74,12 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:mvn/version "0.10.10"}}
+   {io.github.clojure/tools.build {:git/tag "v0.10.6" :git/sha "52cf7d6"}}
    :ns-default build}
 
   :test
   {:main-opts ["-m" "kaocha.runner"]
-   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
-              "--sun-misc-unsafe-memory-access=allow"
-              "--enable-native-access=ALL-UNNAMED"]
+   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"]
    :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
 
   :outdated

backend/dev/user.clj

@@ -6,14 +6,12 @@
 
 (ns user
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.debug :as debug]
    [app.common.exceptions :as ex]
    [app.common.files.helpers :as cfh]
    [app.common.fressian :as fres]
    [app.common.geom.matrix :as gmt]
-   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.perf :as perf]
    [app.common.pprint :as pp]
@@ -21,21 +19,20 @@
    [app.common.schema.desc-js-like :as smdj]
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.generators :as sg]
-   [app.common.schema.openapi :as oapi]
    [app.common.spec :as us]
-   [app.common.time :as ct]
+   [app.common.json :as json]
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]
-   [app.srepl.helpers :as h]
-   [app.srepl.main :refer :all]
+   [app.srepl.helpers :as srepl.helpers]
+   [app.srepl.main :as srepl]
    [app.util.blob :as blob]
+   [app.util.time :as dt]
    [clj-async-profiler.core :as prof]
    [clojure.contrib.humanize :as hum]
-   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]

backend/package.json

@@ -4,7 +4,7 @@
   "license": "MPL-2.0",
   "author": "Kaleidos INC",
   "private": true,
-  "packageManager": "yarn@4.9.2+sha512.1fc009bc09d13cfd0e19efa44cbfc2b9cf6ca61482725eb35bbc5e257e093ebf4130db6dfe15d604ff4b79efd8e1e8e99b25fa7d0a6197c9f9826358d4d65c3c",
+  "packageManager": "yarn@4.8.1+sha512.bc946f2a022d7a1a38adfc15b36a66a3807a67629789496c3714dd1703d2e6c6b1c69ff9ec3b43141ac7a1dd853b7685638eb0074300386a59c18df351ef8ff6",
   "repository": {
     "type": "git",
     "url": "https://github.com/penpot/penpot"

backend/resources/app/email/change-email/en.html

@@ -193,7 +193,7 @@
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
                       <div
                         style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
-                        Click the link below to confirm the change.</div>
+                        Click to the link below to confirm the change:</div>
                     </td>
                   </tr>
                   <tr>
@@ -217,7 +217,8 @@
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
                       <div
                         style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
-                        If you did not request this change, consider changing your password for security reasons.</div>
+                        If you received this email by mistake, please consider changing your password for security
+                        reasons.</div>
                     </td>
                   </tr>
                   <tr>

backend/resources/app/email/change-email/en.txt

@@ -2,11 +2,12 @@ Hello {{name|abbreviate:25}}!
 
 We received a request to change your current email to {{ pending-email }}.
 
-Click the link below to confirm the change.
+Click to the link below to confirm the change:
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
-If you did not request this change, consider changing your password for security reasons.
+If you received this email by mistake, please consider changing your password
+for security reasons.
 
 Enjoy!
 The Penpot team.

backend/resources/app/email/invite-to-team/en.subj

@@ -1 +1 @@
-{{invited-by|abbreviate:25}} has invited you to join the team “{{ team|abbreviate:25 }}”
+Invitation to join {{team}}

backend/resources/app/email/request-file-access/en.txt

@@ -13,7 +13,7 @@ This will automatically include {{requested-by|abbreviate:25}} in the team, so t
 
 Click the link below to provide team access:
 
-{{ public-uri }}/#/dashboard/members?team-id={{team-id}}&invite-email={{requested-by-email|urlescape}}
+{{ public-uri }}/#/dashboard/members?team-id{{team-id}}&invite-email={{requested-by-email|urlescape}}
 
 
 

backend/resources/app/onboarding.edn

@@ -1,10 +1,4 @@
-[{:id "tokens-starter-kit"
-  :name "Design tokens starter kit"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"}
- {:id "penpot-design-system"
-  :name "Penpot Design System | Pencil"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/penpot-app.penpot"}
- {:id "wireframing-kit"
+[{:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}
  {:id "prototype-examples"
@@ -13,6 +7,9 @@
  {:id "plants-app"
   :name "UI mockup example"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/Plants-app.penpot"}
+ {:id "penpot-design-system"
+  :name "Design system example"
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Penpot%20-%20Design%20System%20v2.1.penpot"}
  {:id "tutorial-for-beginners"
   :name "Tutorial for beginners"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/tutorial-for-beginners.penpot"}

backend/resources/app/templates/debug.tmpl

@@ -17,6 +17,38 @@ Debug Main Page
       <desc><a href="/dbg/error">CLICK HERE TO SEE THE ERROR REPORTS</a> </desc>
     </fieldset>
 
+    <fieldset>
+      <legend>Download file data:</legend>
+      <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
+      <form method="get" action="/dbg/file/data">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="submit" name="download" value="Download" />
+          <input type="submit" name="clone" value="Clone" />
+        </div>
+      </form>
+    </fieldset>
+
+    <fieldset>
+      <legend>Upload File Data:</legend>
+      <desc>Create a new file on your draft projects using the file downloaded from the previous section.</desc>
+      <form method="post" enctype="multipart/form-data" action="/dbg/file/data">
+        <div class="row">
+          <input type="file" name="file" value="" />
+        </div>
+        <div class="row">
+          <label>Import with same id?</label>
+          <input type="checkbox" name="reuseid" />
+        </div>
+
+        <div class="row">
+          <input type="submit" value="Upload" />
+        </div>
+      </form>
+    </fieldset>
+
     <fieldset>
       <legend>Profile Management</legend>
       <form method="post" action="/dbg/actions/resend-email-verification">
@@ -45,88 +77,10 @@ Debug Main Page
       </form>
     </fieldset>
 
-    <fieldset>
-      <legend>VIRTUAL CLOCK</legend>
 
-      <desc>
-        <p>
-          CURRENT CLOCK: <b>{{current-clock}}</b>
-          <br />
-          CURRENT OFFSET: <b>{{current-offset}}</b>
-          <br />
-          CURRENT TIME: <b>{{current-time}}</b>
-        </p>
-
-        <p>Examples: 3h, -7h, 24h (allowed suffixes: h, s)</p>
-      </desc>
-
-      <form method="post" action="/dbg/actions/set-virtual-clock">
-        <div class="row">
-          <input type="text" name="offset" placeholder="3h" value="" />
-        </div>
-
-        <div class="row">
-          <label for="force-verify">Are you sure?</label>
-          <input id="force-verify" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-        <div class="row">
-          <input type="submit" name="submit" value="Submit" />
-          <input type="submit" name="reset" value="Reset" />
-        </div>
-      </form>
-    </fieldset>
 
   </section>
 
-  <section class="widget">
-
-    <fieldset>
-      <legend>Download RAW file data:</legend>
-      <desc>Given an FILE-ID, downloads the file AS-IS (no validation
-        checks, just exports the file data and related objects in raw)
-
-        <br/>
-        <br/>
-        <b>WARNING: this operation does not performs any checks</b>
-      </desc>
-      <form method="get" action="/dbg/actions/file-raw-export-import">
-        <div class="row">
-          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
-        </div>
-        <div class="row">
-          <input type="submit" name="download" value="Download" />
-          <input type="submit" name="clone" value="Clone" />
-        </div>
-      </form>
-    </fieldset>
-
-    <fieldset>
-      <legend>Upload File Data:</legend>
-      <desc>Create a new file on your draft projects using the file downloaded from the previous section.
-        <br/>
-        <br/>
-        <b>WARNING: this operation does not performs any checks</b>
-      </desc>
-      <form method="post" enctype="multipart/form-data" action="/dbg/actions/file-raw-export-import">
-        <div class="row">
-          <input type="file" name="file" value="" />
-        </div>
-        <div class="row">
-          <label>Import with same id?</label>
-          <input type="checkbox" name="reuseid" />
-        </div>
-
-        <div class="row">
-          <input type="submit" value="Upload" />
-        </div>
-      </form>
-    </fieldset>
-  </section>
   <section class="widget">
     <fieldset>
       <legend>Export binfile:</legend>
@@ -134,7 +88,7 @@ Debug Main Page
       the related libraries in a single custom formatted binary
       file.</desc>
 
-      <form method="get" action="/dbg/actions/file-export">
+      <form method="get" action="/dbg/file/export">
         <div class="row set-of-inputs">
           <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
           <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
@@ -162,7 +116,7 @@ Debug Main Page
       <legend>Import binfile:</legend>
       <desc>Import penpot file in binary format.</desc>
 
-      <form method="post" enctype="multipart/form-data" action="/dbg/actions/file-import">
+      <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
         <div class="row">
           <input type="file" name="file" value="" />
         </div>
@@ -176,27 +130,79 @@ Debug Main Page
 
   <section class="widget">
     <fieldset>
-      <legend>Feature Flags for Team</legend>
+      <legend>Reset file version</legend>
+      <desc>Allows reset file data version to a specific number/</desc>
+
+      <form method="post" action="/dbg/actions/reset-file-version">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="number" style="width:100px" name="version" placeholder="version" value="32" />
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
+
+  <section class="widget">
+    <h2>Feature Flags</h2>
+    <fieldset>
+      <legend>Enable</legend>
       <desc>Add a feature flag to a team</desc>
-      <form method="post" action="/dbg/actions/handle-team-features">
+      <form method="post" action="/dbg/actions/add-team-feature">
         <div class="row">
           <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
         </div>
         <div class="row">
-          <select type="text" style="width:100px" name="feature">
-            {% for feature in supported-features %}
-              <option value="{{feature}}">{{feature}}</option>
-            {% endfor %}
-          </select>
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
         </div>
 
         <div class="row">
-          <select style="width:100px" name="action">
-            <option value="">Action...</option>
-            <option value="show">Show</option>
-            <option value="enable">Enable</option>
-            <option value="disable">Disable</option>
-          </select>
+          <label for="check-feature">Skip feature check</label>
+          <input id="check-feature" type="checkbox" name="skip-check" />
+          <br />
+          <small>
+            Do not check if the feature is supported
+          </small>
+        </div>
+
+        <div class="row">
+          <label for="force-version">Are you sure?</label>
+          <input id="force-version" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" value="Submit" />
+        </div>
+      </form>
+    </fieldset>
+    <fieldset>
+      <legend>Disable</legend>
+      <desc>Remove a feature flag from a team</desc>
+      <form method="post" action="/dbg/actions/remove-team-feature">
+        <div class="row">
+          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
+        </div>
+        <div class="row">
+          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
         </div>
 
         <div class="row">

backend/resources/app/templates/error-list.tmpl

@@ -7,9 +7,7 @@ penpot - error list
 {% block content %}
 <nav>
   <div class="title">
-    <h1>Error reports (last 200)
-      <a href="/dbg">[GO BACK]</a>
-    </h1>
+    <h1>Error reports (last 200)</h1>
   </div>
 </nav>
 <main class="horizontal-list">

backend/scripts/_env

@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-
-
-export PENPOT_MANAGEMENT_API_SHARED_KEY=super-secret-management-api-key
-export PENPOT_SECRET_KEY=super-secret-devenv-key
-export PENPOT_HOST=devenv
-
-export PENPOT_FLAGS="\
-       $PENPOT_FLAGS \
-       enable-login-with-ldap \
-       enable-login-with-password
-       enable-login-with-oidc \
-       enable-login-with-google \
-       enable-login-with-github \
-       enable-login-with-gitlab \
-       enable-backend-worker \
-       enable-backend-asserts \
-       disable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
-       enable-audit-log \
-       enable-transit-readable-response \
-       enable-demo-users \
-       disable-secure-session-cookies \
-       enable-smtp \
-       enable-prepl-server \
-       enable-urepl-server \
-       enable-rpc-climit \
-       enable-rpc-rlimit \
-       enable-quotes \
-       enable-soft-rpc-rlimit \
-       enable-auto-file-snapshot \
-       enable-webhooks \
-       enable-access-tokens \
-       disable-tiered-file-data-storage \
-       enable-file-validation \
-       enable-file-schema-validation \
-       enable-redis-cache \
-       enable-subscriptions";
-
-# Default deletion delay for devenv
-export PENPOT_DELETION_DELAY="24h"
-
-# Setup default upload media file size to 100MiB
-export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
-
-# Setup default multipart upload size to 300MiB
-export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-export PENPOT_OBJECTS_STORAGE_BACKEND=s3
-export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
-export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
-
-export JAVA_OPTS="\
-       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -Djdk.attach.allowAttachSelf \
-       -Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -Djdk.tracePinnedThreads=full \
-       -Dim4java.useV7=true \
-       -XX:+UnlockExperimentalVMOptions \
-       -XX:+UseShenandoahGC \
-       -XX:+UseCompactObjectHeaders \
-       -XX:ShenandoahGCMode=generational \
-       -XX:-OmitStackTraceInFastThrow  \
-       --sun-misc-unsafe-memory-access=allow \
-       --enable-preview \
-       --enable-native-access=ALL-UNNAMED";
-
-function setup_minio() {
-    # Initialize MINIO config
-    mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
-    mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-    mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-    if [ "$?" = "1" ]; then
-        mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-    fi
-    mc mb penpot-s3/penpot -p -q
-}
-
-

backend/scripts/manage.py

@@ -71,27 +71,19 @@ def run_cmd(params):
         print("EXC:", str(cause))
         sys.exit(-2)
 
-def create_profile(fullname, email, password, skip_tutorial=False, skip_walkthrough=False):
-    props = {}
-    if skip_tutorial:
-        props["viewed-tutorial?"] = True
-    if skip_walkthrough:
-        props["viewed-walkthrough?"] = True
-
+def create_profile(fullname, email, password):
     params = {
         "cmd": "create-profile",
         "params": {
             "fullname": fullname,
             "email": email,
-            "password": password,
-            **props
+            "password": password
         }
     }
 
     res = run_cmd(params)
     print(f"Created: {res['email']} / {res['id']}")
 
-
 def update_profile(email, fullname, password, is_active):
     params = {
         "cmd": "update-profile",
@@ -178,8 +170,6 @@ parser.add_argument("-n", "--fullname", help="fullname", action="store")
 parser.add_argument("-e", "--email", help="email", action="store")
 parser.add_argument("-p", "--password", help="password", action="store")
 parser.add_argument("-c", "--connect", help="connect to PREPL", action="store", default="tcp://localhost:6063")
-parser.add_argument("--skip-tutorial", help="mark tutorial as viewed", action="store_true")
-parser.add_argument("--skip-walkthrough", help="mark walkthrough as viewed", action="store_true")
 
 args = parser.parse_args()
 

backend/scripts/repl

@@ -1,13 +1,114 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
-source $SCRIPT_DIR/_env;
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-login-with-ldap \
+       enable-login-with-password
+       enable-login-with-oidc \
+       enable-login-with-google \
+       enable-login-with-github \
+       enable-login-with-gitlab \
+       enable-backend-worker \
+       enable-backend-asserts \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-audit-log \
+       enable-transit-readable-response \
+       enable-demo-users \
+       disable-secure-session-cookies \
+       enable-smtp \
+       enable-prepl-server \
+       enable-urepl-server \
+       enable-rpc-climit \
+       enable-rpc-rlimit \
+       enable-quotes \
+       enable-soft-rpc-rlimit \
+       enable-auto-file-snapshot \
+       enable-webhooks \
+       enable-access-tokens \
+       enable-tiered-file-data-storage \
+       enable-file-validation \
+       enable-file-schema-validation \
+       enable-subscriptons \
+       enable-subscriptons-old";
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
+# export PENPOT_DATABASE_USERNAME="penpot"
+# export PENPOT_DATABASE_PASSWORD="penpot"
+# export PENPOT_DATABASE_READONLY=true
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
+# export PENPOT_DATABASE_USERNAME="penpot_pre"
+# export PENPOT_DATABASE_PASSWORD="penpot_pre"
+
+# export PENPOT_LOGGERS_LOKI_URI="http://172.17.0.1:3100/loki/api/v1/push"
+# export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
 
 # Initialize MINIO config
-setup_minio;
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
+
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
+       -Djdk.tracePinnedThreads=full \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow                    \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
 
-export JAVA_OPTS="$JAVA_OPTS -Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
 export OPTIONS="-A:jmx-remote -A:dev"
+
+# Setup HEAP
+# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
+# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
+
+# Increase virtual thread pool size
+# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
+
+# Disable C2 Compiler
+# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
+
+# Disable all compilers
+# export OPTIONS="$OPTIONS -J-Xint"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
+
+# Setup GC
+# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
+
+# Enable ImageMagick v7.x support
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
+
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 

backend/scripts/repl-test

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+source /home/penpot/environ
+export PENPOT_FLAGS="$PENPOT_FLAGS disable-backend-worker"
+
+export OPTIONS="
+       -A:jmx-remote -A:dev \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-Dlog4j2.configurationFile=log4j2-experiments.xml \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints \
+       -J-Djdk.tracePinnedThreads=full \
+       -J-XX:+UseTransparentHugePages \
+       -J-XX:ReservedCodeCacheSize=1g \
+       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
+       -J--enable-preview";
+
+# Setup HEAP
+export OPTIONS="$OPTIONS -J-Xms320g -J-Xmx320g -J-XX:+AlwaysPreTouch"
+
+export PENPOT_HTTP_SERVER_IO_THREADS=2
+export PENPOT_HTTP_SERVER_WORKER_THREADS=2
+
+# Increase virtual thread pool size
+# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
+
+# Disable C2 Compiler
+# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
+
+# Disable all compilers
+# export OPTIONS="$OPTIONS -J-Xint"
+
+# Setup GC
+export OPTIONS="$OPTIONS -J-XX:+UseG1GC -J-Xlog:gc:logs/gc.log"
+
+# Setup GC
+#export OPTIONS="$OPTIONS -J-XX:+UseZGC -J-XX:+ZGenerational -J-Xlog:gc:logs/gc.log"
+
+# Enable ImageMagick v7.x support
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
+
+export OPTIONS_EVAL="nil"
+# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
+
+set -ex
+exec clojure $OPTIONS -M -e "$OPTIONS_EVAL" -m rebel-readline.main

backend/scripts/run

@@ -1,13 +1,44 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-backend-asserts \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-file-snapshot \
+       enable-tiered-file-data-storage";
 
-source $SCRIPT_DIR/_env;
-export OPTIONS="-A:dev"
+export JAVA_OPTS="
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints";
+
+export CLOJURE_OPTIONS="-A:dev"
+
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 
 entrypoint=${1:-app.main};
-
 shift 1;
 set -ex
 
-exec clojure $OPTIONS -A:dev -M -m $entrypoint "$@";
+clojure $CLOJURE_OPTIONS -A:dev -M -m $entrypoint "$@";

backend/scripts/run.template.sh

@@ -18,9 +18,9 @@ if [ -f ./environ ]; then
     source ./environ
 fi
 
-export JAVA_OPTS="-Dim4java.useV7=true -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
+export JVM_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS"
 
 ENTRYPOINT=${1:-app.main};
 
 set -ex
-exec $JAVA_CMD $JAVA_OPTS -jar penpot.jar -m $ENTRYPOINT
+exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m $ENTRYPOINT

backend/scripts/start-dev

@@ -1,11 +1,75 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$(dirname $0);
-source $SCRIPT_DIR/_env;
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-prepl-server \
+       enable-urepl-server \
+       enable-nrepl-server \
+       enable-webhooks \
+       enable-backend-asserts \
+       enable-audit-log \
+       enable-login-with-ldap \
+       enable-transit-readable-response \
+       enable-demo-users \
+       enable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       disable-secure-session-cookies \
+       enable-rpc-climit \
+       enable-smtp \
+       enable-quotes \
+       enable-file-snapshot \
+       enable-access-tokens \
+       enable-tiered-file-data-storage \
+       enable-file-validation \
+       enable-file-schema-validation \
+       enable-subscriptons \
+       enable-subscriptons-old ";
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+# Enable ImageMagick v7.x support
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
 
 # Initialize MINIO config
-setup_minio;
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+if [ "$?" = "1" ]; then
+    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+fi
+mc mb penpot-s3/penpot -p -q
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+
+entrypoint=${1:-app.main};
+
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -Djdk.tracePinnedThreads=full \
+       -XX:+EnableDynamicAgentLoading \
+       -XX:-OmitStackTraceInFastThrow                    \
+       -XX:+UnlockDiagnosticVMOptions \
+       -XX:+DebugNonSafepoints \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
+
+export OPTIONS="-A:jmx-remote -A:dev"
 
-shift 1;
 set -ex
-exec clojure -A:jmx-remote -A:dev -M -m app.main "$@";
+clojure $OPTIONS -M -m $entrypoint;

backend/src/app/auth/oidc.clj

@@ -13,7 +13,6 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
@@ -29,6 +28,7 @@
    [app.tokens :as tokens]
    [app.util.inet :as inet]
    [app.util.json :as json]
+   [app.util.time :as dt]
    [buddy.sign.jwk :as jwk]
    [buddy.sign.jwt :as jwt]
    [clojure.set :as set]
@@ -434,10 +434,10 @@
   (sm/validator schema:info))
 
 (defn- get-info
-  [{:keys [::provider] :as cfg} {:keys [params] :as request}]
+  [{:keys [::provider ::setup/props] :as cfg} {:keys [params] :as request}]
   (let [state  (get params :state)
         code   (get params :code)
-        state  (tokens/verify cfg {:token state :iss :oauth})
+        state  (tokens/verify props {:token state :iss :oauth})
         tdata  (fetch-access-token cfg code)
         info   (case (cf/get :oidc-user-info-source)
                  :token (get-user-info cfg tdata)
@@ -514,9 +514,9 @@
   [cfg info request]
   (let [info   (assoc info
                       :iss :prepared-register
-                      :exp (ct/in-future {:hours 48}))
+                      :exp (dt/in-future {:hours 48}))
 
-        params {:token (tokens/generate cfg info)
+        params {:token (tokens/generate (::setup/props cfg) info)
                 :provider (:provider (:path-params request))
                 :fullname (:fullname info)}
         params (d/without-nils params)]
@@ -569,9 +569,9 @@
       :else
       (let [sxf     (session/create-fn cfg (:id profile))
             token   (or (:invitation-token info)
-                        (tokens/generate cfg
+                        (tokens/generate (::setup/props cfg)
                                          {:iss :auth
-                                          :exp (ct/in-future "15m")
+                                          :exp (dt/in-future "15m")
                                           :profile-id (:id profile)}))
             props   (audit/profile->props profile)
             context (d/without-nils {:external-session-id (:external-session-id info)})]
@@ -619,8 +619,9 @@
                 :invitation-token (:invitation-token params)
                 :external-session-id esid
                 :props props
-                :exp (ct/in-future "4h")}
-        state  (tokens/generate cfg (d/without-nils params))
+                :exp (dt/in-future "4h")}
+        state  (tokens/generate (::setup/props cfg)
+                                (d/without-nils params))
         uri    (build-auth-uri cfg state)]
     {::yres/status 200
      ::yres/body {:redirect-uri uri}}))

backend/src/app/binfile/cleaner.clj

@@ -16,40 +16,16 @@
 ;; PRE DECODE
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- pre-clean-bool-content
-  [shape]
-  (if-let [content (get shape :bool-content)]
-    (-> shape
-        (assoc :content content)
-        (dissoc :bool-content))
-    shape))
-
-(defn- pre-clean-shadow-color
-  [shape]
-  (d/update-when shape :shadow
-                 (fn [shadows]
-                   (mapv (fn [shadow]
-                           (update shadow :color
-                                   (fn [color]
-                                     (let [ref-id   (get color :id)
-                                           ref-file (get color :file-id)]
-                                       (-> (d/without-qualified color)
-                                           (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
-                                           (cond-> ref-id
-                                             (assoc :ref-id ref-id))
-                                           (cond-> ref-file
-                                             (assoc :ref-file ref-file)))))))
-                         shadows))))
-
 (defn clean-shape-pre-decode
   "Applies a pre-decode phase migration to the shape"
   [shape]
-  (cond-> shape
-    (= "bool" (:type shape))
-    (pre-clean-bool-content)
-
-    (contains? shape :shadow)
-    (pre-clean-shadow-color)))
+  (if (= "bool" (:type shape))
+    (if-let [content (get shape :bool-content)]
+      (-> shape
+          (assoc :content content)
+          (dissoc :bool-content))
+      shape)
+    shape))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; POST DECODE

backend/src/app/binfile/common.clj

@@ -15,21 +15,19 @@
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
-   [app.common.weak :as weak]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.fdata :as fdata]
-   [app.features.file-migrations :as fmigr]
+   [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
+   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
    [cuerdas.core :as str]
@@ -40,7 +38,6 @@
 
 (def ^:dynamic *state* nil)
 (def ^:dynamic *options* nil)
-(def ^:dynamic *reference-file* nil)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
@@ -57,11 +54,15 @@
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare get-resolved-file-libraries)
-(declare update-file!)
-
 (def file-attrs
-  (sm/keys ctf/schema:file))
+  #{:id
+    :name
+    :migrations
+    :features
+    :project-id
+    :is-shared
+    :version
+    :data})
 
 (defn parse-file-format
   [template]
@@ -141,176 +142,29 @@
   ([index coll attr]
    (reduce #(index-object %1 %2 attr) index coll)))
 
-(defn- decode-row-features
-  [{:keys [features] :as row}]
-  (when row
-    (cond-> row
-      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
+(defn decode-row
+  "A generic decode row helper"
+  [{:keys [data features] :as row}]
+  (cond-> row
+    features (assoc :features (db/decode-pgarray features #{}))
+    data     (assoc :data (blob/decode data))))
 
-(def sql:get-minimal-file
-  "SELECT f.id,
-          f.revn,
-          f.modified_at,
-          f.deleted_at
-     FROM file AS f
-    WHERE f.id = ?")
+(defn decode-file
+  "A general purpose file decoding function that resolves all external
+  pointers, run migrations and return plain vanilla file map"
+  [cfg {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+    (let [file (->> file
+                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (feat.fdata/resolve-file-data cfg))]
 
-(defn get-minimal-file
-  [cfg id & {:as opts}]
-  (db/get-with-sql cfg [sql:get-minimal-file id] opts))
-
-(def sql:files-with-data
-  "SELECT f.id,
-          f.project_id,
-          f.created_at,
-          f.modified_at,
-          f.deleted_at,
-          f.name,
-          f.is_shared,
-          f.has_media_trimmed,
-          f.revn,
-          f.data AS legacy_data,
-          f.ignore_sync_until,
-          f.comment_thread_seqn,
-          f.features,
-          f.version,
-          f.vern,
-          p.team_id,
-          coalesce(fd.backend, 'legacy-db') AS backend,
-          fd.metadata AS metadata,
-          fd.data AS data
-     FROM file AS f
-     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
-    INNER JOIN project AS p ON (p.id = f.project_id)")
-
-(def sql:get-file
-  (str sql:files-with-data " WHERE f.id = ?"))
-
-(def sql:get-file-without-data
-  (str "WITH files AS (" sql:files-with-data ")"
-       "SELECT f.id,
-               f.project_id,
-               f.created_at,
-               f.modified_at,
-               f.deleted_at,
-               f.name,
-               f.is_shared,
-               f.has_media_trimmed,
-               f.revn,
-               f.ignore_sync_until,
-               f.comment_thread_seqn,
-               f.features,
-               f.version,
-               f.vern,
-               f.team_id
-          FROM files AS f
-         WHERE f.id = ?"))
-
-(defn- migrate-file
-  [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)
-            pmap/*tracked* (pmap/create-tracked)]
-    (let [libs (delay (get-resolved-file-libraries cfg file))
-          ;; For avoid unnecesary overhead of creating multiple
-          ;; pointers and handly internally with objects map in their
-          ;; worst case (when probably all shapes and all pointers
-          ;; will be readed in any case), we just realize/resolve them
-          ;; before applying the migration to the file.
-          file (-> (fdata/realize cfg file)
-                   (fmg/migrate-file libs))]
-
-      (if (or read-only? (db/read-only? conn))
-        file
-        (do ;; When file is migrated, we break the rule of no
-          ;; perform mutations on get operations and update the
-          ;; file with all migrations applied
-          (update-file! cfg file)
-          (fmigr/resolve-applied-migrations cfg file))))))
-
-(defn- get-file*
-  [{:keys [::db/conn] :as cfg} id
-   {:keys [migrate?
-           realize?
-           decode?
-           skip-locked?
-           include-deleted?
-           load-data?
-           throw-if-not-exists?
-           lock-for-update?
-           lock-for-share?]
-    :or {lock-for-update? false
-         lock-for-share? false
-         load-data? true
-         migrate? true
-         decode? true
-         include-deleted? false
-         throw-if-not-exists? true
-         realize? false}
-    :as options}]
-
-  (assert (db/connection? conn) "expected cfg with valid connection")
-  (when (and (not load-data?)
-             (or lock-for-share? lock-for-share? skip-locked?))
-    (throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false")))
-
-  (let [sql
-        (if load-data?
-          sql:get-file
-          sql:get-file-without-data)
-
-        sql
-        (cond
-          lock-for-update?
-          (str sql " FOR UPDATE of f")
-
-          lock-for-share?
-          (str sql " FOR SHARE of f")
-
-          :else
-          sql)
-
-        sql
-        (if skip-locked?
-          (str sql " SKIP LOCKED")
-          sql)
-
-        file
-        (db/get-with-sql conn [sql id]
-                         {::db/throw-if-not-exists false
-                          ::db/remove-deleted (not include-deleted?)})
-
-        file
-        (-> file
-            (d/update-when :features db/decode-pgarray #{})
-            (d/update-when :metadata fdata/decode-metadata))]
-
-    (if file
-      (if load-data?
-        (let [file
-              (->> file
-                   (fmigr/resolve-applied-migrations cfg)
-                   (fdata/resolve-file-data cfg))
-
-              will-migrate?
-              (and migrate? (fmg/need-migration? file))]
-
-          (if decode?
-            (cond->> (fdata/decode-file-data cfg file)
-              (and realize? (not will-migrate?))
-              (fdata/realize cfg)
-
-              will-migrate?
-              (migrate-file cfg options))
-
-            file))
-        file)
-
-      (when-not (or skip-locked? (not throw-if-not-exists?))
-        (ex/raise :type :not-found
-                  :code :object-not-found
-                  :hint "database object not found"
-                  :table :file
-                  :file-id id)))))
+      (-> file
+          (update :features db/decode-pgarray #{})
+          (update :data blob/decode)
+          (update :data feat.fdata/process-pointers deref)
+          (update :data feat.fdata/process-objects (partial into {}))
+          (update :data assoc :id id)
+          (fmg/migrate-file)))))
 
 (defn get-file
   "Get file, resolve all features and apply migrations.
@@ -319,7 +173,10 @@
   operations on file, because it removes the ovehead of lazy fetching
   and decoding."
   [cfg file-id & {:as opts}]
-  (db/run! cfg get-file* file-id opts))
+  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                 (some->> (db/get* conn :file {:id file-id}
+                                   (assoc opts ::db/remove-deleted false))
+                          (decode-file cfg)))))
 
 (defn clean-file-features
   [file]
@@ -343,12 +200,12 @@
   (let [conn (db/get-connection cfg)
         ids  (db/create-array conn "uuid" ids)]
     (->> (db/exec! conn [sql:get-teams ids])
-         (map decode-row-features))))
+         (map decode-row))))
 
 (defn get-team
   [cfg team-id]
   (-> (db/get cfg :team {:id team-id})
-      (decode-row-features)))
+      (decode-row)))
 
 (defn get-fonts
   [cfg team-id]
@@ -440,6 +297,7 @@
             (do
               (l/trc :hint "lookup index"
                      :file-id (str file-id)
+                     :snap-id (str (:snapshot-id file))
                      :id (str id)
                      :result (str (get mobj :id)))
               (get mobj :id))
@@ -456,6 +314,7 @@
     (doseq [[old-id item] missing-index]
       (l/dbg :hint "create missing references"
              :file-id (str file-id)
+             :snap-id (str (:snapshot-id file))
              :old-id (str old-id)
              :id (str (:id item)))
       (db/insert! conn :file-media-object item
@@ -466,16 +325,12 @@
 (def sql:get-file-media
   "SELECT * FROM file_media_object WHERE id = ANY(?)")
 
-(defn get-file-media*
-  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
-  (let [used (cfh/collect-used-media data)
-        used (db/create-array conn "uuid" used)]
-    (->> (db/exec! conn [sql:get-file-media used])
-         (mapv (fn [row] (assoc row :file-id id))))))
-
 (defn get-file-media
-  [cfg file]
-  (db/run! cfg get-file-media* file))
+  [cfg {:keys [data] :as file}]
+  (db/run! cfg (fn [{:keys [::db/conn]}]
+                 (let [used (cfh/collect-used-media data)
+                       used (db/create-array conn "uuid" used)]
+                   (db/exec! conn [sql:get-file-media used])))))
 
 (def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
@@ -550,7 +405,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id :include-deleted? true)]
+              (if-let [library (get-file cfg library-id)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data
@@ -562,146 +417,100 @@
     (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
-(defn invalidate-thumbnails
-  [cfg file-id]
-  (let [storage (sto/resolve cfg)
-
-        sql-1
-        (str "update file_tagged_object_thumbnail "
-             "   set deleted_at = now() "
-             " where file_id=? returning media_id")
-
-        sql-2
-        (str "update file_thumbnail "
-             "   set deleted_at = now() "
-             " where file_id=? returning media_id")]
-
-    (run! #(sto/touch-object! storage %)
-          (sequence
-           (keep :media-id)
-           (concat
-            (db/exec! cfg [sql-1 file-id])
-            (db/exec! cfg [sql-2 file-id]))))))
-
 (defn process-file
-  [cfg {:keys [id] :as file}]
-  (let [libs (delay (get-resolved-file-libraries cfg file))]
-    (-> file
-        (update :data (fn [fdata]
-                        (-> fdata
-                            (assoc :id id)
-                            (dissoc :recent-colors))))
-        (update :data (fn [fdata]
-                        (-> fdata
-                            (update :pages-index relink-shapes)
-                            (update :components relink-shapes)
-                            (update :media relink-media)
-                            (update :colors relink-colors)
-                            (d/without-nils))))
-        (fmg/migrate-file libs)
+  [{:keys [id] :as file}]
+  (-> file
+      (update :data (fn [fdata]
+                      (-> fdata
+                          (assoc :id id)
+                          (dissoc :recent-colors))))
+      (fmg/migrate-file)
+      (update :data (fn [fdata]
+                      (-> fdata
+                          (update :pages-index relink-shapes)
+                          (update :components relink-shapes)
+                          (update :media relink-media)
+                          (update :colors relink-colors)
+                          (d/without-nils))))
 
-        ;; NOTE: this is necessary because when we just creating a new
-        ;; file from imported artifact or cloned file there are no
-        ;; migrations registered on the database, so we need to persist
-        ;; all of them, not only the applied
-        (vary-meta dissoc ::fmg/migrated))))
+      ;; NOTE: this is necessary because when we just creating a new
+      ;; file from imported artifact or cloned file there are no
+      ;; migrations registered on the database, so we need to persist
+      ;; all of them, not only the applied
+      (vary-meta dissoc ::fmg/migrated)))
 
-(defn- encode-file
-  [cfg {:keys [id features] :as file}]
-  (let [file (if (and (contains? features "fdata/objects-map")
-                      (:data file))
-               (fdata/enable-objects-map file)
+(defn encode-file
+  [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
+  (let [file (if (contains? features "fdata/objects-map")
+               (feat.fdata/enable-objects-map file)
                file)
 
-        file (if (and (contains? features "fdata/pointer-map")
-                      (:data file))
-
-               (binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
-                 (let [file (fdata/enable-pointer-map file)]
-                   (fdata/persist-pointers! cfg id)
+        file (if (contains? features "fdata/pointer-map")
+               (binding [pmap/*tracked* (pmap/create-tracked)]
+                 (let [file (feat.fdata/enable-pointer-map file)]
+                   (feat.fdata/persist-pointers! cfg id)
                    file))
                file)]
 
     (-> file
-        (d/update-when :features into-array)
-        (d/update-when :data blob/encode))))
+        (update :features db/encode-pgarray conn "text")
+        (update :data blob/encode))))
 
-(defn- file->params
+(defn get-params-from-file
   [file]
-  (-> (select-keys file file-attrs)
-      (assoc :data nil)
-      (dissoc :team-id)
-      (dissoc :migrations)))
+  (let [params {:has-media-trimmed (:has-media-trimmed file)
+                :ignore-sync-until (:ignore-sync-until file)
+                :project-id (:project-id file)
+                :features (:features file)
+                :name (:name file)
+                :is-shared (:is-shared file)
+                :version (:version file)
+                :data (:data file)
+                :id (:id file)
+                :deleted-at (:deleted-at file)
+                :created-at (:created-at file)
+                :modified-at (:modified-at file)
+                :revn (:revn file)
+                :vern (:vern file)}]
 
-(defn- file->file-data-params
-  [{:keys [id] :as file} & {:as opts}]
-  (let [created-at  (or (:created-at file) (ct/now))
-        modified-at (or (:modified-at file) created-at)]
-    (d/without-nils
-     {:id id
-      :type "main"
-      :file-id id
-      :data (:data file)
-      :metadata (:metadata file)
-      :created-at created-at
-      :modified-at modified-at})))
+    (-> (d/without-nils params)
+        (assoc :data-backend nil)
+        (assoc :data-ref-id nil))))
 
 (defn insert-file!
-  "Insert a new file into the database table. Expectes a not-encoded file.
-  Returns nil."
+  "Insert a new file into the database table"
   [{:keys [::db/conn] :as cfg} file & {:as opts}]
-  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-
-  (when (:migrations file)
-    (fmigr/upsert-migrations! conn file))
-
-  (let [file (encode-file cfg file)]
-    (db/insert! conn :file
-                (file->params file)
-                (assoc opts ::db/return-keys false))
-
-    (->> (file->file-data-params file)
-         (fdata/upsert! cfg))
-
-    nil))
+  (feat.fmigr/upsert-migrations! conn file)
+  (let [params (-> (encode-file cfg file)
+                   (get-params-from-file))]
+    (db/insert! conn :file params opts)))
 
 (defn update-file!
-  "Update an existing file on the database. Expects not encoded file."
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
+  "Update an existing file on the database."
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
+  (let [file   (encode-file cfg file)
+        params (-> (get-params-from-file file)
+                   (dissoc :id))]
 
-  (if (::reset-migrations? opts false)
-    (fmigr/reset-migrations! conn file)
-    (fmigr/upsert-migrations! conn file))
+    ;; If file was already offloaded, we touch the underlying storage
+    ;; object for properly trigger storage-gc-touched task
+    (when (feat.fdata/offloaded? file)
+      (some->> (:data-ref-id file) (sto/touch-object! storage)))
 
-  (let [file
-        (encode-file cfg file)
-
-        file-params
-        (file->params (dissoc file :id))
-
-        file-data-params
-        (file->file-data-params file)]
-
-    (db/update! conn :file file-params
-                {:id id}
-                {::db/return-keys false})
-
-    (fdata/upsert! cfg file-data-params)
-    nil))
+    (feat.fmigr/upsert-migrations! conn file)
+    (db/update! conn :file params {:id id} opts)))
 
 (defn save-file!
   "Applies all the final validations and perist the file, binfile
-  specific, should not be used outside of binfile domain.
-  Returns nil"
+  specific, should not be used outside of binfile domain"
   [{:keys [::timestamp] :as cfg} file & {:as opts}]
 
-  (assert (ct/inst? timestamp) "expected valid timestamp")
+  (assert (dt/instant? timestamp) "expected valid timestamp")
 
   (let [file (-> file
                  (assoc :created-at timestamp)
                  (assoc :modified-at timestamp)
-                 (cond-> (not (::overwrite cfg))
-                   (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))))
+                 (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
                  (update :features
                          (fn [features]
                            (-> (::features cfg #{})
@@ -718,62 +527,4 @@
         (when (ex/exception? result)
           (l/error :hint "file schema validation error" :cause result))))
 
-    (if (::overwrite cfg)
-      (update-file! cfg file (assoc opts ::reset-migrations? true))
-      (insert-file! cfg file opts))))
-
-(def ^:private sql:get-file-libraries
-  "WITH RECURSIVE libs AS (
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-      WHERE flr.file_id = ?::uuid
-    UNION
-     SELECT fl.*, flr.synced_at
-       FROM file AS fl
-       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
-       JOIN libs AS l ON (flr.file_id = l.id)
-   )
-   SELECT l.id,
-          l.features,
-          l.project_id,
-          p.team_id,
-          l.created_at,
-          l.modified_at,
-          l.deleted_at,
-          l.name,
-          l.revn,
-          l.vern,
-          l.synced_at,
-          l.is_shared,
-          l.version
-     FROM libs AS l
-    INNER JOIN project AS p ON (p.id = l.project_id)
-    WHERE l.deleted_at IS NULL;")
-
-(defn get-file-libraries
-  [conn file-id]
-  (into []
-        (comp
-         ;; FIXME: :is-indirect set to false to all rows looks
-         ;; completly useless
-         (map #(assoc % :is-indirect false))
-         (map decode-row-features))
-        (db/exec! conn [sql:get-file-libraries file-id])))
-
-(defn get-resolved-file-libraries
-  "Get all file libraries including itself. Returns an instance of
-  LoadableWeakValueMap that allows do not have strong references to
-  the loaded libraries and reduce possible memory pressure on having
-  all this libraries loaded at same time on processing file validation
-  or file migration.
-
-  This still requires at least one library at time to be loaded while
-  access to it is performed, but it improves considerable not having
-  the need of loading all the libraries at the same time."
-  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
-  (let [library-ids (->> (get-file-libraries conn (:id file))
-                         (map :id)
-                         (cons (:id file)))
-        load-fn     #(get-file cfg % :migrate? false)]
-    (weak/loadable-weak-value-map library-ids load-fn {id file})))
+    (insert-file! cfg file opts)))

backend/src/app/binfile/migrations.clj

@@ -10,6 +10,7 @@
    [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
+   [app.features.components-v2 :as feat.compv2]
    [clojure.set :as set]
    [cuerdas.core :as str]))
 
@@ -27,15 +28,22 @@
 
 (defn apply-pending-migrations!
   "Apply alredy registered pending migrations to files"
-  [_cfg]
-  (doseq [[feature _file-id] (-> bfc/*state* deref :pending-to-migrate)]
+  [cfg]
+  (doseq [[feature file-id] (-> bfc/*state* deref :pending-to-migrate)]
     (case feature
       "components/v2"
-      nil
+      (feat.compv2/migrate-file! cfg file-id
+                                 :validate? (::validate cfg true)
+                                 :skip-on-graphic-error? true)
 
       "fdata/shape-data-type"
       nil
 
+      ;; There is no migration needed, but we don't want to allow
+      ;; copy paste nor import of variant files into no-variant teams
+      "variants/v1"
+      nil
+
       (ex/raise :type :internal
                 :code :no-migration-defined
                 :hint (str/ffmt "no migation for feature '%' on file importation" feature)

backend/src/app/binfile/v1.clj

@@ -17,7 +17,6 @@
    [app.common.fressian :as fres]
    [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -31,6 +30,7 @@
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.events :as events]
+   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.java.io :as jio]
    [clojure.set :as set]
@@ -346,7 +346,7 @@
           thumbnails (->> (bfc/get-file-object-thumbnails cfg file-id)
                           (mapv #(dissoc % :file-id)))
 
-          file       (cond-> (bfc/get-file cfg file-id :realize? true)
+          file       (cond-> (bfc/get-file cfg file-id)
                        detach?
                        (-> (ctf/detach-external-references file-id)
                            (dissoc :libraries))
@@ -434,7 +434,7 @@
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
   format."
-  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (ct/now)} :as options}]
+  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
 
   (dm/assert!
    "expected input stream"
@@ -442,7 +442,7 @@
 
   (dm/assert!
    "expected valid instant"
-   (ct/inst? timestamp))
+   (dt/instant? timestamp))
 
   (let [version (read-header! input)]
     (read-import (assoc options ::version version ::bfc/timestamp timestamp))))
@@ -551,8 +551,8 @@
                      (cond-> (and (= idx 0) (some? name))
                        (assoc :name name))
                      (assoc :project-id project-id)
-                     (dissoc :thumbnails))
-            file  (bfc/process-file system file)]
+                     (dissoc :thumbnails)
+                     (bfc/process-file))]
 
         ;; All features that are enabled and requires explicit migration are
         ;; added to the state for a posterior migration step.
@@ -682,7 +682,7 @@
    (io/coercible? output))
 
   (let [id (uuid/next)
-        tp (ct/tpoint)
+        tp (dt/tpoint)
         ab (volatile! false)
         cs (volatile! nil)]
     (try
@@ -720,7 +720,7 @@
    (satisfies? jio/IOFactory input))
 
   (let [id (uuid/next)
-        tp (ct/tpoint)
+        tp (dt/tpoint)
         cs (volatile! nil)]
 
     (l/info :hint "import: started" :id (str id))
@@ -742,6 +742,6 @@
       (finally
         (l/info :hint "import: terminated"
                 :id (str id)
-                :elapsed (ct/format-duration (tp))
+                :elapsed (dt/format-duration (tp))
                 :error? (some? @cs))))))
 

backend/src/app/binfile/v2.clj

@@ -13,7 +13,6 @@
    [app.common.data :as d]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
-   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -24,6 +23,7 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.events :as events]
+   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
    [cuerdas.core :as str]
@@ -153,7 +153,7 @@
 
 (defn- write-file!
   [cfg file-id]
-  (let [file   (bfc/get-file cfg file-id :realize? true)
+  (let [file   (bfc/get-file cfg file-id)
         thumbs (bfc/get-file-object-thumbnails cfg file-id)
         media  (bfc/get-file-media cfg file)
         rels   (bfc/get-files-rels cfg #{file-id})]
@@ -281,8 +281,8 @@
 
   (let [file (-> (read-obj cfg :file file-id)
                  (update :id bfc/lookup-index)
-                 (update :project-id bfc/lookup-index))
-        file (bfc/process-file cfg file)]
+                 (update :project-id bfc/lookup-index)
+                 (bfc/process-file))]
 
     (events/tap :progress
                 {:op :import
@@ -344,7 +344,7 @@
 (defn export-team!
   [cfg team-id]
   (let [id  (uuid/next)
-        tp  (ct/tpoint)
+        tp  (dt/tpoint)
         cfg (create-database cfg)]
 
     (l/inf :hint "start"
@@ -378,15 +378,15 @@
           (l/inf :hint "end"
                  :operation "export"
                  :id (str id)
-                 :elapsed (ct/format-duration elapsed)))))))
+                 :elapsed (dt/format-duration elapsed)))))))
 
 (defn import-team!
   [cfg path]
   (let [id  (uuid/next)
-        tp  (ct/tpoint)
+        tp  (dt/tpoint)
 
         cfg (-> (create-database cfg path)
-                (assoc ::bfc/timestamp (ct/now)))]
+                (assoc ::bfc/timestamp (dt/now)))]
 
     (l/inf :hint "start"
            :operation "import"
@@ -434,4 +434,4 @@
           (l/inf :hint "end"
                  :operation "import"
                  :id (str id)
-                 :elapsed (ct/format-duration elapsed)))))))
+                 :elapsed (dt/format-duration elapsed)))))))

backend/src/app/binfile/v3.clj

@@ -12,22 +12,21 @@
    [app.binfile.common :as bfc]
    [app.binfile.migrations :as bfm]
    [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.files.migrations :as-alias fmg]
    [app.common.json :as json]
    [app.common.logging :as l]
-   [app.common.media :as cmedia]
    [app.common.schema :as sm]
    [app.common.thumbnails :as cth]
-   [app.common.time :as ct]
    [app.common.types.color :as ctcl]
    [app.common.types.component :as ctc]
    [app.common.types.file :as ctf]
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
-   [app.common.types.tokens-lib :as ctob]
+   [app.common.types.tokens-lib :as cto]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -36,15 +35,14 @@
    [app.storage :as sto]
    [app.storage.impl :as sto.impl]
    [app.util.events :as events]
+   [app.util.time :as dt]
    [clojure.java.io :as jio]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
-   java.io.File
    java.io.InputStream
    java.io.OutputStreamWriter
-   java.lang.AutoCloseable
    java.util.zip.ZipEntry
    java.util.zip.ZipFile
    java.util.zip.ZipOutputStream))
@@ -55,7 +53,7 @@
   [:map {:title "Manifest"}
    [:version ::sm/int]
    [:type :string]
-   [:referer {:optional true} :string]
+
    [:generated-by {:optional true} :string]
 
    [:files
@@ -75,7 +73,7 @@
    [:size ::sm/int]
    [:content-type :string]
    [:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
-   [:hash {:optional true} :string]])
+   [:hash :string]])
 
 (def ^:private schema:file-thumbnail
   [:map {:title "FileThumbnail"}
@@ -90,40 +88,34 @@
    ctf/schema:file
    [:map [:options {:optional true} ctf/schema:options]]])
 
-;; --- HELPERS
-
-(defn- default-now
-  [o]
-  (or o (ct/now)))
-
 ;; --- ENCODERS
 
 (def encode-file
   (sm/encoder schema:file sm/json-transformer))
 
 (def encode-page
-  (sm/encoder ctp/schema:page sm/json-transformer))
+  (sm/encoder ::ctp/page sm/json-transformer))
 
 (def encode-shape
-  (sm/encoder cts/schema:shape sm/json-transformer))
+  (sm/encoder ::cts/shape sm/json-transformer))
 
 (def encode-media
-  (sm/encoder ctf/schema:media sm/json-transformer))
+  (sm/encoder ::ctf/media sm/json-transformer))
 
 (def encode-component
-  (sm/encoder ctc/schema:component sm/json-transformer))
+  (sm/encoder ::ctc/component sm/json-transformer))
 
 (def encode-color
-  (sm/encoder ctcl/schema:library-color sm/json-transformer))
+  (sm/encoder ::ctcl/color sm/json-transformer))
 
 (def encode-typography
-  (sm/encoder cty/schema:typography sm/json-transformer))
+  (sm/encoder ::cty/typography sm/json-transformer))
 
 (def encode-tokens-lib
-  (sm/encoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/encoder ::cto/tokens-lib sm/json-transformer))
 
 (def encode-plugin-data
-  (sm/encoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
 
 (def encode-storage-object
   (sm/encoder schema:storage-object sm/json-transformer))
@@ -137,31 +129,31 @@
   (sm/decoder schema:manifest sm/json-transformer))
 
 (def decode-media
-  (sm/decoder ctf/schema:media sm/json-transformer))
+  (sm/decoder ::ctf/media sm/json-transformer))
 
 (def decode-component
-  (sm/decoder ctc/schema:component sm/json-transformer))
+  (sm/decoder ::ctc/component sm/json-transformer))
 
 (def decode-color
-  (sm/decoder ctcl/schema:library-color sm/json-transformer))
+  (sm/decoder ::ctcl/color sm/json-transformer))
 
 (def decode-file
   (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
-  (sm/decoder ctp/schema:page sm/json-transformer))
+  (sm/decoder ::ctp/page sm/json-transformer))
 
 (def decode-shape
-  (sm/decoder cts/schema:shape sm/json-transformer))
+  (sm/decoder ::cts/shape sm/json-transformer))
 
 (def decode-typography
-  (sm/decoder cty/schema:typography sm/json-transformer))
+  (sm/decoder ::cty/typography sm/json-transformer))
 
 (def decode-tokens-lib
-  (sm/decoder ctob/schema:tokens-lib sm/json-transformer))
+  (sm/decoder ::cto/tokens-lib sm/json-transformer))
 
 (def decode-plugin-data
-  (sm/decoder ctpg/schema:plugin-data sm/json-transformer))
+  (sm/decoder ::ctpg/plugin-data sm/json-transformer))
 
 (def decode-storage-object
   (sm/decoder schema:storage-object sm/json-transformer))
@@ -175,31 +167,31 @@
   (sm/check-fn schema:manifest))
 
 (def validate-file
-  (sm/check-fn ctf/schema:file))
+  (sm/check-fn ::ctf/file))
 
 (def validate-page
-  (sm/check-fn ctp/schema:page))
+  (sm/check-fn ::ctp/page))
 
 (def validate-shape
-  (sm/check-fn cts/schema:shape))
+  (sm/check-fn ::cts/shape))
 
 (def validate-media
-  (sm/check-fn ctf/schema:media))
+  (sm/check-fn ::ctf/media))
 
 (def validate-color
-  (sm/check-fn ctcl/schema:library-color))
+  (sm/check-fn ::ctcl/color))
 
 (def validate-component
-  (sm/check-fn ctc/schema:component))
+  (sm/check-fn ::ctc/component))
 
 (def validate-typography
-  (sm/check-fn cty/schema:typography))
+  (sm/check-fn ::cty/typography))
 
 (def validate-tokens-lib
-  (sm/check-fn ctob/schema:tokens-lib))
+  (sm/check-fn ::cto/tokens-lib))
 
 (def validate-plugin-data
-  (sm/check-fn ctpg/schema:plugin-data))
+  (sm/check-fn ::ctpg/plugin-data))
 
 (def validate-storage-object
   (sm/check-fn schema:storage-object))
@@ -224,12 +216,9 @@
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
-  (let [detach? (and (not embed-assets) (not include-libraries))]
+  (let [detach?  (and (not embed-assets) (not include-libraries))]
     (db/tx-run! cfg (fn [cfg]
-                      (cond-> (bfc/get-file cfg file-id
-                                            {:realize? true
-                                             :include-deleted? true
-                                             :lock-for-update? true})
+                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
                             (dissoc :libraries))
@@ -240,13 +229,27 @@
                         :always
                         (bfc/clean-file-features))))))
 
+(defn- resolve-extension
+  [mtype]
+  (case mtype
+    "image/png"     ".png"
+    "image/jpeg"    ".jpg"
+    "image/gif"     ".gif"
+    "image/svg+xml" ".svg"
+    "image/webp"    ".webp"
+    "font/woff"     ".woff"
+    "font/woff2"    ".woff2"
+    "font/ttf"      ".ttf"
+    "font/otf"      ".otf"
+    "application/octet-stream" ".bin"))
+
 (defn- export-storage-objects
   [{:keys [::output] :as cfg}]
   (let [storage (sto/resolve cfg)]
     (doseq [id (-> bfc/*state* deref :storage-objects not-empty)]
       (let [sobject (sto/get-object storage id)
             smeta   (meta sobject)
-            ext     (cmedia/mtype->extension (:content-type smeta))
+            ext     (resolve-extension (:content-type smeta))
             path    (str "objects/" id ".json")
             params  (-> (meta sobject)
                         (assoc :id (:id sobject))
@@ -256,9 +259,9 @@
         (write-entry! output path params)
 
         (with-open [input (sto/get-object-data storage sobject)]
-          (.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
+          (.putNextEntry output (ZipEntry. (str "objects/" id ext)))
           (io/copy input output :size (:size sobject))
-          (.closeEntry ^ZipOutputStream output))))))
+          (.closeEntry output))))))
 
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
@@ -286,13 +289,13 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data)))
+                 (assoc :options (:options data))
 
-          file (-> file
-                   (dissoc :data)
-                   (dissoc :deleted-at)
-                   (encode-file))
+                 :always
+                 (dissoc :data)
 
+                 :always
+                 (encode-file))
           path (str "files/" file-id ".json")]
       (write-entry! output path file))
 
@@ -350,8 +353,7 @@
             typography (encode-typography object)]
         (write-entry! output path typography)))
 
-    (when (and tokens-lib
-               (not (ctob/empty-lib? tokens-lib)))
+    (when tokens-lib
       (let [path           (str "files/" file-id "/tokens.json")
             encoded-tokens (encode-tokens-lib tokens-lib)]
         (write-entry! output path encoded-tokens)))))
@@ -378,7 +380,6 @@
           params {:type "penpot/export-files"
                   :version 1
                   :generated-by (str "penpot/" (:full cf/version))
-                  :refer "penpot"
                   :files (vec (vals files))
                   :relations rels}]
       (write-entry! output "manifest.json" params))))
@@ -451,7 +452,7 @@
 (defn- read-manifest
   [^ZipFile input]
   (let [entry (get-zip-entry input "manifest.json")]
-    (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+    (with-open [reader (zip-entry-reader input entry)]
       (let [manifest (json/read reader :key-fn json/read-kebab-key)]
         (decode-manifest manifest)))))
 
@@ -541,27 +542,24 @@
 
 (defn- read-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
 (defn- read-plain-entry
   [^ZipFile input entry]
-  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
+  (with-open [reader (zip-entry-reader input entry)]
     (json/read reader)))
 
 (defn- read-file
-  [{:keys [::bfc/input ::bfc/timestamp]} file-id]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
         (decode-file)
-        (update :revn d/nilv 1)
-        (update :created-at d/nilv timestamp)
-        (update :modified-at d/nilv timestamp)
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::bfc/input]} file-id]
+  [{:keys [::bfc/input ::file-id]}]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -570,19 +568,13 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::bfc/input ::entries]} file-id]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
                                    (decode-media)
                                    (validate-media))
-                       object (-> object
-                                  (assoc :file-id file-id)
-                                  (update :created-at default-now)
-                                  ;; FIXME: this is set default to true for
-                                  ;; setting a value, this prop is no longer
-                                  ;; relevant;
-                                  (assoc :is-local true))]
+                       object (assoc object :file-id file-id)]
                    (if (= id (:id object))
                      (conj result object)
                      result)))
@@ -590,7 +582,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::bfc/input ::entries]} file-id]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -603,7 +595,7 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::bfc/input ::entries]} file-id]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (let [clean-component-post-decode
         (fn [component]
           (d/update-when component :objects
@@ -626,7 +618,8 @@
                    (let [object (->> (read-entry input entry)
                                      (clean-component-pre-decode)
                                      (decode-component)
-                                     (clean-component-post-decode))]
+                                     (clean-component-post-decode)
+                                     (validate-component))]
                      (if (= id (:id object))
                        (assoc result id object)
                        result)))
@@ -634,7 +627,7 @@
          (not-empty))))
 
 (defn- read-file-typographies
-  [{:keys [::bfc/input ::entries]} file-id]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -647,20 +640,21 @@
        (not-empty)))
 
 (defn- read-file-tokens-lib
-  [{:keys [::bfc/input ::entries]} file-id]
+  [{:keys [::bfc/input ::file-id ::entries]}]
   (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
     (->> (read-plain-entry input entry)
          (decode-tokens-lib)
          (validate-tokens-lib))))
 
 (defn- read-file-shapes
-  [{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
+  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
                                    (bfl/clean-shape-pre-decode)
                                    (decode-shape)
-                                   (bfl/clean-shape-post-decode))]
+                                   (bfl/clean-shape-post-decode)
+                                   (validate-shape))]
                    (if (= id (:id object))
                      (assoc result id object)
                      result)))
@@ -668,14 +662,15 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::bfc/input ::entries] :as cfg} file-id]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
                                (decode-page))
                      page (dissoc page :options)]
                  (when (= id (:id page))
-                   (let [objects (read-file-shapes cfg file-id id)]
+                   (let [objects (-> (assoc cfg ::page-id id)
+                                     (read-file-shapes))]
                      (assoc page :objects objects))))))
        (sort-by :index)
        (reduce (fn [result {:keys [id] :as page}]
@@ -683,7 +678,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::bfc/input ::entries] :as cfg} file-id]
+  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -698,13 +693,14 @@
        (not-empty)))
 
 (defn- read-file-data
-  [cfg file-id]
-  (let [colors       (read-file-colors cfg file-id)
-        typographies (read-file-typographies cfg file-id)
-        tokens-lib   (read-file-tokens-lib cfg file-id)
-        components   (read-file-components cfg file-id)
-        plugin-data  (read-file-plugin-data cfg file-id)
-        pages        (read-file-pages cfg file-id)]
+  [cfg]
+  (let [colors       (read-file-colors cfg)
+        typographies (read-file-typographies cfg)
+        tokens-lib   (read-file-tokens-lib cfg)
+        components   (read-file-components cfg)
+        plugin-data  (read-file-plugin-data cfg)
+        pages        (read-file-pages cfg)]
+
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
@@ -714,11 +710,11 @@
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
+  [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
   (let [file-id'   (bfc/lookup-index file-id)
-        file       (read-file cfg file-id)
-        media      (read-file-media cfg file-id)
-        thumbnails (read-file-thumbnails cfg file-id)]
+        file       (read-file cfg)
+        media      (read-file-media cfg)
+        thumbnails (read-file-thumbnails cfg)]
 
     (l/dbg :hint "processing file"
            :id (str file-id')
@@ -727,50 +723,28 @@
            :version (:version file)
            ::l/sync? true)
 
-    (vswap! bfc/*state* update :index bfc/update-index media :id)
+    (events/tap :progress {:section :file :name file-name})
 
-    (events/tap :progress {:section :media :file-id file-id})
+    (when media
+      ;; Update index with media
+      (l/dbg :hint "update media index"
+             :file-id (str file-id')
+             :total (count media)
+             ::l/sync? true)
 
-    (doseq [item media]
-      (let [params (-> item
-                       (update :id bfc/lookup-index)
-                       (assoc :file-id file-id')
-                       (d/update-when :media-id bfc/lookup-index)
-                       (d/update-when :thumbnail-id bfc/lookup-index))]
+      (vswap! bfc/*state* update :index bfc/update-index (map :id media))
+      (vswap! bfc/*state* update :media into media))
 
-        (l/dbg :hint "inserting media object"
-               :file-id (str file-id')
-               :id (str (:id params))
-               :media-id (str (:media-id params))
-               :thumbnail-id (str (:thumbnail-id params))
-               :old-id (str (:id item))
-               ::l/sync? true)
+    (when thumbnails
+      (l/dbg :hint "update thumbnails index"
+             :file-id (str file-id')
+             :total (count thumbnails)
+             ::l/sync? true)
 
-        (db/insert! conn :file-media-object params
-                    ::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
+      (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
+      (vswap! bfc/*state* update :thumbnails into thumbnails))
 
-    (events/tap :progress {:section :thumbnails :file-id file-id})
-
-    (doseq [item thumbnails]
-      (let [media-id  (bfc/lookup-index (:media-id item))
-            object-id (-> (assoc item :file-id file-id')
-                          (cth/fmt-object-id))
-            params    {:file-id file-id'
-                       :object-id object-id
-                       :tag (:tag item)
-                       :media-id media-id}]
-
-        (l/dbg :hint "inserting object thumbnail"
-               :file-id (str file-id')
-               :media-id (str media-id)
-               ::l/sync? true)
-
-        (db/insert! conn :file-tagged-object-thumbnail params
-                    ::db/on-conflict-do-nothing? true)))
-
-    (events/tap :progress {:section :file :file-id file-id})
-
-    (let [data (-> (read-file-data cfg file-id)
+    (let [data (-> (read-file-data cfg)
                    (d/without-nils)
                    (assoc :id file-id')
                    (cond-> (:options file)
@@ -781,13 +755,11 @@
                    (assoc :data data)
                    (assoc :name file-name)
                    (assoc :project-id project-id)
-                   (dissoc :options))
-
-          file  (bfc/process-file cfg file)
-          file  (ctf/check-file file)]
+                   (dissoc :options)
+                   (bfc/process-file))]
 
       (bfm/register-pending-migrations! cfg file)
-      (bfc/save-file! cfg file)
+      (bfc/save-file! cfg file ::db/return-keys false)
 
       file-id')))
 
@@ -817,113 +789,107 @@
         entries (keep (match-storage-entry-fn) entries)]
 
     (doseq [{:keys [id entry]} entries]
-      (let [object  (->> (read-entry input entry)
-                         (decode-storage-object)
-                         (validate-storage-object))
+      (let [object (->> (read-entry input entry)
+                        (decode-storage-object)
+                        (validate-storage-object))]
 
-            ext     (cmedia/mtype->extension (:content-type object))
-            path    (str "objects/" id ext)
-            content (->> path
-                         (get-zip-entry input)
-                         (zip-entry-storage-content input))]
-
-        (when (not= (:size object) (sto/get-size content))
+        (when (not= id (:id object))
           (ex/raise :type :validation
                     :code :inconsistent-penpot-file
-                    :hint "found corrupted storage object: size does not match"
-                    :path path
-                    :expected-size (:size object)
-                    :found-size (sto/get-size content)))
+                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"
+                    :expected-id (str id)
+                    :found-id (str (:id object))))
 
-        (when-let [hash (get object :hash)]
-          (when (not= hash (sto/get-hash content))
+        (let [ext     (resolve-extension (:content-type object))
+              path    (str "objects/" id ext)
+              content (->> path
+                           (get-zip-entry input)
+                           (zip-entry-storage-content input))]
+
+          (when (not= (:size object) (sto/get-size content))
+            (ex/raise :type :validation
+                      :code :inconsistent-penpot-file
+                      :hint "found corrupted storage object: size does not match"
+                      :path path
+                      :expected-size (:size object)
+                      :found-size (sto/get-size content)))
+
+          (when (not= (:hash object) (sto/get-hash content))
             (ex/raise :type :validation
                       :code :inconsistent-penpot-file
                       :hint "found corrupted storage object: hash does not match"
                       :path path
                       :expected-hash (:hash object)
-                      :found-hash (sto/get-hash content))))
+                      :found-hash (sto/get-hash content)))
 
-        (let [params  (-> object
-                          (dissoc :id :size)
-                          (assoc ::sto/content content)
-                          (assoc ::sto/deduplicate? true)
-                          (assoc ::sto/touched-at timestamp))
-              sobject (sto/put-object! storage params)]
+          (let [params  (-> object
+                            (dissoc :id :size)
+                            (assoc ::sto/content content)
+                            (assoc ::sto/deduplicate? true)
+                            (assoc ::sto/touched-at timestamp))
+                sobject (sto/put-object! storage params)]
 
-          (l/dbg :hint "persisted storage object"
-                 :id (str (:id sobject))
-                 :prev-id (str id)
-                 :bucket (:bucket params)
-                 ::l/sync? true)
+            (l/dbg :hint "persisted storage object"
+                   :id (str (:id sobject))
+                   :prev-id (str id)
+                   :bucket (:bucket params)
+                   ::l/sync? true)
 
-          (vswap! bfc/*state* update :index assoc id (:id sobject)))))))
+            (vswap! bfc/*state* update :index assoc id (:id sobject))))))))
 
-(defn- import-files*
-  [{:keys [::manifest] :as cfg}]
-  (bfc/disable-database-timeouts! cfg)
+(defn- import-file-media
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:section :media})
 
-  (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
+  (doseq [item (:media @bfc/*state*)]
+    (let [params (-> item
+                     (update :id bfc/lookup-index)
+                     (update :file-id bfc/lookup-index)
+                     (d/update-when :media-id bfc/lookup-index)
+                     (d/update-when :thumbnail-id bfc/lookup-index))]
 
-  (import-storage-objects cfg)
+      (l/dbg :hint "inserting file media object"
+             :old-id (str (:id item))
+             :id (str (:id params))
+             :file-id (str (:file-id params))
+             ::l/sync? true)
 
-  (let [files  (get manifest :files)
-        result (reduce (fn [result {:keys [id] :as file}]
-                         (let [name' (get file :name)
-                               name' (if (map? name)
-                                       (get name id)
-                                       name')
-                               file (assoc file :name name')]
-                           (conj result (import-file cfg file))))
-                       []
-                       files)]
+      (db/insert! conn :file-media-object params))))
 
-    (import-file-relations cfg)
-    (bfm/apply-pending-migrations! cfg)
+(defn- import-file-thumbnails
+  [{:keys [::db/conn] :as cfg}]
+  (events/tap :progress {:section :thumbnails})
+  (doseq [item (:thumbnails @bfc/*state*)]
+    (let [file-id   (bfc/lookup-index (:file-id item))
+          media-id  (bfc/lookup-index (:media-id item))
+          object-id (-> (assoc item :file-id file-id)
+                        (cth/fmt-object-id))
+          params    {:file-id file-id
+                     :object-id object-id
+                     :tag (:tag item)
+                     :media-id media-id}]
 
-    result))
+      (l/dbg :hint "inserting file object thumbnail"
+             :file-id (str file-id)
+             :media-id (str media-id)
+             ::l/sync? true)
 
-(defn- import-file-and-overwrite*
-  [{:keys [::manifest ::bfc/file-id] :as cfg}]
-
-  (when (not= 1 (count (:files manifest)))
-    (ex/raise :type :validation
-              :code :invalid-condition
-              :hint "unable to perform in-place update with binfile containing more than 1 file"
-              :manifest manifest))
-
-  (bfc/disable-database-timeouts! cfg)
-
-  (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
-        file     (first (get manifest :files))
-        cfg      (assoc cfg ::bfc/overwrite true)]
-
-    (vswap! bfc/*state* update :index assoc (:id file) file-id)
-
-    (binding [bfc/*options* cfg
-              bfc/*reference-file* ref-file]
-
-      (import-storage-objects cfg)
-      (import-file cfg file)
-
-      (bfc/invalidate-thumbnails cfg file-id)
-      (bfm/apply-pending-migrations! cfg)
-
-      [file-id])))
+      (db/insert! conn :file-tagged-object-thumbnail params))))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
 
-  (assert (instance? ZipFile input) "expected zip file")
-  (assert (ct/inst? timestamp) "expected valid instant")
+  (dm/assert!
+   "expected zip file"
+   (instance? ZipFile input))
+
+  (dm/assert!
+   "expected valid instant"
+   (dt/instant? timestamp))
 
   (let [manifest (-> (read-manifest input)
                      (validate-manifest))
-        entries  (read-zip-entries input)
-        cfg      (-> cfg
-                     (assoc ::entries entries)
-                     (assoc ::manifest manifest)
-                     (assoc ::bfc/timestamp timestamp))]
+        entries  (read-zip-entries input)]
 
     (when-not (= "penpot/export-files" (:type manifest))
       (ex/raise :type :validation
@@ -946,10 +912,35 @@
 
     (events/tap :progress {:section :manifest})
 
-    (binding [bfc/*state* (volatile! {:media [] :index {}})]
-      (if (::bfc/file-id cfg)
-        (db/tx-run! cfg import-file-and-overwrite*)
-        (db/tx-run! cfg import-files*)))))
+    (let [index (bfc/update-index (map :id (:files manifest)))
+          state {:media [] :index index}
+          cfg   (-> cfg
+                    (assoc ::entries entries)
+                    (assoc ::manifest manifest)
+                    (assoc ::bfc/timestamp timestamp))]
+
+      (binding [bfc/*state* (volatile! state)]
+        (db/tx-run! cfg (fn [cfg]
+                          (bfc/disable-database-timeouts! cfg)
+                          (let [ids (->> (:files manifest)
+                                         (reduce (fn [result {:keys [id] :as file}]
+                                                   (let [name' (get file :name)
+                                                         name' (if (map? name)
+                                                                 (get name id)
+                                                                 name')]
+                                                     (conj result (-> cfg
+                                                                      (assoc ::file-id id)
+                                                                      (assoc ::file-name name')
+                                                                      (import-file)))))
+                                                 []))]
+                            (import-file-relations cfg)
+                            (import-storage-objects cfg)
+                            (import-file-media cfg)
+                            (import-file-thumbnails cfg)
+
+                            (bfm/apply-pending-migrations! cfg)
+
+                            ids)))))))
 
 ;; --- PUBLIC API
 
@@ -966,23 +957,24 @@
 
   [{:keys [::bfc/ids] :as cfg} output]
 
-  (assert
-   (and (set? ids) (every? uuid? ids))
-   "expected a set of uuid's for `::bfc/ids` parameter")
+  (dm/assert!
+   "expected a set of uuid's for `::bfc/ids` parameter"
+   (and (set? ids)
+        (every? uuid? ids)))
 
-  (assert
-   (satisfies? jio/IOFactory output)
-   "expected instance of jio/IOFactory for `input`")
+  (dm/assert!
+   "expected instance of jio/IOFactory for `input`"
+   (satisfies? jio/IOFactory output))
 
   (let [id (uuid/next)
-        tp (ct/tpoint)
+        tp (dt/tpoint)
         ab (volatile! false)
         cs (volatile! nil)]
     (try
       (l/info :hint "start exportation" :export-id (str id))
       (binding [bfc/*state* (volatile! (bfc/initial-state))]
-        (with-open [^AutoCloseable output (io/output-stream output)]
-          (with-open [^AutoCloseable output (ZipOutputStream. output)]
+        (with-open [output (io/output-stream output)]
+          (with-open [output (ZipOutputStream. output)]
             (let [cfg (assoc cfg ::output output)]
               (export-files cfg)
               (export-storage-objects cfg)))))
@@ -1011,22 +1003,22 @@
 (defn import-files!
   [{:keys [::bfc/input] :as cfg}]
 
-  (assert
+  (dm/assert!
+   "expected valid profile-id and project-id on `cfg`"
    (and (uuid? (::bfc/profile-id cfg))
-        (uuid? (::bfc/project-id cfg)))
-   "expected valid profile-id and project-id on `cfg`")
+        (uuid? (::bfc/project-id cfg))))
 
-  (assert
-   (io/coercible? input)
-   "expected instance of jio/IOFactory for `input`")
+  (dm/assert!
+   "expected instance of jio/IOFactory for `input`"
+   (io/coercible? input))
 
   (let [id (uuid/next)
-        tp (ct/tpoint)
+        tp (dt/tpoint)
         cs (volatile! nil)]
 
     (l/info :hint "import: started" :id (str id))
     (try
-      (with-open [input (ZipFile. ^File (fs/file input))]
+      (with-open [input (ZipFile. (fs/file input))]
         (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
@@ -1036,11 +1028,5 @@
       (finally
         (l/info :hint "import: terminated"
                 :id (str id)
-                :elapsed (ct/format-duration (tp))
+                :elapsed (dt/format-duration (tp))
                 :error? (some? @cs))))))
-
-(defn get-manifest
-  [path]
-  (with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
-    (-> (read-manifest input)
-        (validate-manifest))))

backend/src/app/config.clj

@@ -12,10 +12,10 @@
    [app.common.exceptions :as ex]
    [app.common.flags :as flags]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.common.version :as v]
    [app.util.overrides]
+   [app.util.time :as dt]
    [clojure.core :as c]
    [clojure.java.io :as io]
    [cuerdas.core :as str]
@@ -52,8 +52,6 @@
 
    :redis-uri "redis://redis/0"
 
-   :file-data-backend "legacy-db"
-
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
@@ -61,10 +59,10 @@
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
-   :profile-complaint-max-age (ct/duration {:days 7})
+   :profile-complaint-max-age (dt/duration {:days 7})
    :profile-complaint-threshold 2
 
-   :profile-bounce-max-age (ct/duration {:days 7})
+   :profile-bounce-max-age (dt/duration {:days 7})
    :profile-bounce-threshold 10
 
    :telemetry-uri "https://telemetry.penpot.app/"
@@ -98,19 +96,16 @@
     [:http-server-max-body-size {:optional true} ::sm/int]
     [:http-server-max-multipart-body-size {:optional true} ::sm/int]
     [:http-server-io-threads {:optional true} ::sm/int]
-    [:http-server-max-worker-threads {:optional true} ::sm/int]
-
-    [:management-api-shared-key {:optional true} :string]
+    [:http-server-worker-threads {:optional true} ::sm/int]
 
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
 
     [:auto-file-snapshot-every {:optional true} ::sm/int]
-    [:auto-file-snapshot-timeout {:optional true} ::ct/duration]
+    [:auto-file-snapshot-timeout {:optional true} ::dt/duration]
 
     [:media-max-file-size {:optional true} ::sm/int]
-    [:deletion-delay {:optional true} ::ct/duration]
-    [:file-clean-delay {:optional true} ::ct/duration]
+    [:deletion-delay {:optional true} ::dt/duration] ;; REVIEW
     [:telemetry-enabled {:optional true} ::sm/boolean]
     [:default-blob-version {:optional true} ::sm/int]
     [:allow-demo-users {:optional true} ::sm/boolean]
@@ -151,11 +146,12 @@
     [:quotes-team-access-requests-per-team {:optional true} ::sm/int]
     [:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
 
+    [:auth-data-cookie-domain {:optional true} :string]
     [:auth-token-cookie-name {:optional true} :string]
-    [:auth-token-cookie-max-age {:optional true} ::ct/duration]
+    [:auth-token-cookie-max-age {:optional true} ::dt/duration]
 
     [:registration-domain-whitelist {:optional true} [::sm/set :string]]
-    [:email-verify-threshold {:optional true} ::ct/duration]
+    [:email-verify-threshold {:optional true} ::dt/duration]
 
     [:github-client-id {:optional true} :string]
     [:github-client-secret {:optional true} :string]
@@ -190,9 +186,9 @@
     [:ldap-starttls {:optional true} ::sm/boolean]
     [:ldap-user-query {:optional true} :string]
 
-    [:profile-bounce-max-age {:optional true} ::ct/duration]
+    [:profile-bounce-max-age {:optional true} ::dt/duration]
     [:profile-bounce-threshold {:optional true} ::sm/int]
-    [:profile-complaint-max-age {:optional true} ::ct/duration]
+    [:profile-complaint-max-age {:optional true} ::dt/duration]
     [:profile-complaint-threshold {:optional true} ::sm/int]
 
     [:redis-uri {:optional true} ::sm/uri]
@@ -214,27 +210,24 @@
     [:prepl-host {:optional true} :string]
     [:prepl-port {:optional true} ::sm/int]
 
-    [:file-data-backend {:optional true} [:enum "db" "legacy-db" "storage"]]
-
     [:media-directory {:optional true} :string] ;; REVIEW
     [:media-uri {:optional true} :string]
     [:assets-path {:optional true} :string]
 
-    [:netty-io-threads {:optional true} ::sm/int]
-    [:executor-threads {:optional true} ::sm/int]
-
-    ;; DEPRECATED
+    ;; Legacy, will be removed in 2.5
     [:assets-storage-backend {:optional true} :keyword]
     [:storage-assets-fs-directory {:optional true} :string]
     [:storage-assets-s3-bucket {:optional true} :string]
     [:storage-assets-s3-region {:optional true} :keyword]
     [:storage-assets-s3-endpoint {:optional true} ::sm/uri]
+    [:storage-assets-s3-io-threads {:optional true} ::sm/int]
 
     [:objects-storage-backend {:optional true} :keyword]
     [:objects-storage-fs-directory {:optional true} :string]
     [:objects-storage-s3-bucket {:optional true} :string]
     [:objects-storage-s3-region {:optional true} :keyword]
-    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]]))
+    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
+    [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
 (defn- parse-flags
   [config]
@@ -305,12 +298,7 @@
 (defn get-deletion-delay
   []
   (or (c/get config :deletion-delay)
-      (ct/duration {:days 7})))
-
-(defn get-file-clean-delay
-  []
-  (or (c/get config :file-clean-delay)
-      (ct/duration {:days 2})))
+      (dt/duration {:days 7})))
 
 (defn get
   "A configuration getter. Helps code be more testable."
@@ -319,9 +307,5 @@
   ([key default]
    (c/get config key default)))
 
-(defn logging-context
-  []
-  {:version/backend (:full version)})
-
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -10,20 +10,19 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
-   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.db.sql :as sql]
    [app.metrics :as mtx]
+   [app.util.json :as json]
+   [app.util.time :as dt]
    [clojure.java.io :as io]
    [clojure.set :as set]
    [integrant.core :as ig]
    [next.jdbc :as jdbc]
    [next.jdbc.date-time :as jdbc-dt]
-   [next.jdbc.prepare :as jdbc.prepare]
    [next.jdbc.transaction])
   (:import
    com.zaxxer.hikari.HikariConfig
@@ -34,7 +33,6 @@
    java.io.InputStream
    java.io.OutputStream
    java.sql.Connection
-   java.sql.PreparedStatement
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
@@ -298,7 +296,7 @@
 (defn insert!
   "A helper that builds an insert sql statement and executes it. By
   default returns the inserted row with all the field; you can delimit
-  the returned columns with the `::sql/columns` option."
+  the returned columns with the `::columns` option."
   [ds table params & {:as opts}]
   (let [conn (get-connectable ds)
         sql  (sql/insert table params opts)
@@ -379,7 +377,9 @@
 
 (defn is-row-deleted?
   [{:keys [deleted-at]}]
-  (some? deleted-at))
+  (and (dt/instant? deleted-at)
+       (< (inst-ms deleted-at)
+          (inst-ms (dt/now)))))
 
 (defn get*
   "Retrieve a single row from database that matches a simple filters. Do
@@ -404,23 +404,6 @@
                 :hint "database object not found"))
     row))
 
-(defn get-with-sql
-  [ds sql & {:as opts}]
-  (let [rows
-        (cond->> (exec! ds sql opts)
-          (::remove-deleted opts true)
-          (remove is-row-deleted?)
-
-          :always
-          (not-empty))]
-
-    (when (and (not rows) (::throw-if-not-exists opts true))
-      (ex/raise :type :not-found
-                :code :object-not-found
-                :hint "database object not found"))
-
-    (first rows)))
-
 (def ^:private default-plan-opts
   (-> default-opts
       (assoc :fetch-size 1000)
@@ -575,10 +558,10 @@
   [system f & params]
   (cond
     (connection? system)
-    (apply run! {::conn system} f params)
+    (run! {::conn system} f)
 
     (pool? system)
-    (apply run! {::pool system} f params)
+    (run! {::pool system} f)
 
     (::conn system)
     (apply f system params)
@@ -602,7 +585,7 @@
     (string? o)
     (pginterval o)
 
-    (ct/duration? o)
+    (dt/duration? o)
     (interval (inst-ms o))
 
     :else
@@ -616,7 +599,7 @@
           val (.getValue o)]
       (if (or (= typ "json")
               (= typ "jsonb"))
-        (json/decode val :key-fn keyword)
+        (json/decode val)
         val))))
 
 (defn decode-transit-pgobject
@@ -657,7 +640,7 @@
   (when data
     (doto (org.postgresql.util.PGobject.)
       (.setType "jsonb")
-      (.setValue (json/encode data)))))
+      (.setValue (json/encode-str data)))))
 
 ;; --- Locks
 
@@ -703,8 +686,3 @@
   [cause]
   (and (sql-exception? cause)
        (= "40001" (.getSQLState ^java.sql.SQLException cause))))
-
-(extend-protocol jdbc.prepare/SettableParameter
-  clojure.lang.Keyword
-  (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]
-    (.setObject s i ^String (d/name v))))

backend/src/app/db/sql.clj

@@ -53,15 +53,8 @@
          opts (cond-> opts
                 (::order-by opts)   (assoc :order-by (::order-by opts))
                 (::columns opts)    (assoc :columns (::columns opts))
-
-                (or (::db/for-update opts)
-                    (::for-update opts))
-                (assoc :suffix "FOR UPDATE")
-
-                (or (::db/for-share opts)
-                    (::for-share opts))
-                (assoc :suffix "FOR SHARE"))]
-
+                (::for-update opts) (assoc :suffix "FOR UPDATE")
+                (::for-share opts)  (assoc :suffix "FOR SHARE"))]
      (sql/for-query table where-params opts))))
 
 (defn update

backend/src/app/features/fdata.clj

@@ -9,22 +9,46 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.files.helpers :as cfh]
+   [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.types.objects-map :as omap]
-   [app.config :as cf]
+   [app.common.types.path :as path]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap.legacy]
+   [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OFFLOAD
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn offloaded?
+  [file]
+  (= "objects-storage" (:data-backend file)))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OBJECTS-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn enable-objects-map
+  [file & _opts]
+  (let [update-page
+        (fn [page]
+          (if (and (pmap/pointer-map? page)
+                   (not (pmap/loaded? page)))
+            page
+            (update page :objects omap/wrap)))
+
+        update-data
+        (fn [fdata]
+          (update fdata :pages-index d/update-vals update-page))]
+
+    (-> file
+        (update :data update-data)
+        (update :features conj "fdata/objects-map"))))
+
 (defn process-objects
   "Apply a function to all objects-map on the file. Usualy used for convert
   the objects-map instances to plain maps"
@@ -34,237 +58,35 @@
             (fn [page]
               (update page :objects
                       (fn [objects]
-                        (if (or (omap/objects-map? objects)
-                                (omap.legacy/objects-map? objects))
+                        (if (omap/objects-map? objects)
                           (update-fn objects)
                           objects)))))
     fdata))
 
-
-(defn realize-objects
-  "Process a file and remove all instances of objects map realizing them
-  to a plain data. Used in operation where is more efficient have the
-  whole file loaded in memory or we going to persist it in an
-  alterantive storage."
-  [_cfg file]
-  (update file :data process-objects (partial into {})))
-
-(defn enable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (update page :objects omap/wrap))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features conj "fdata/objects-map"))))
-
-(defn disable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (update page :objects #(into {} %)))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features disj "fdata/objects-map"))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; STORAGE
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmulti resolve-file-data
-  (fn [_cfg file] (get file :backend "legacy-db")))
-
-(defmethod resolve-file-data "legacy-db"
-  [_cfg {:keys [legacy-data] :as file}]
-  (-> file
-      (assoc :data legacy-data)
-      (dissoc :legacy-data)))
-
-(defmethod resolve-file-data "db"
-  [_cfg file]
-  (dissoc file :legacy-data))
-
-(defmethod resolve-file-data "storage"
-  [cfg {:keys [metadata] :as file}]
-  (let [storage (sto/resolve cfg ::db/reuse-conn true)
-        ref-id  (:storage-ref-id metadata)
-        data    (->> (sto/get-object storage ref-id)
-                     (sto/get-object-bytes storage))]
-    (-> file
-        (assoc :data data)
-        (dissoc :legacy-data))))
-
-(defn decode-file-data
-  [_cfg {:keys [data] :as file}]
-  (cond-> file
-    (bytes? data)
-    (assoc :data (blob/decode data))))
-
-(def ^:private sql:insert-file-data
-  "INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at,
-                          type, backend, metadata, data)
-   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
-
-(def ^:private sql:upsert-file-data
-  (str sql:insert-file-data
-       " ON CONFLICT (file_id, id)
-         DO UPDATE SET modified_at=?,
-                       deleted_at=?,
-                       backend=?,
-                       metadata=?,
-                       data=?"))
-
-(defn- upsert-in-database
-  [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}]
-  (let [created-at  (or created-at (ct/now))
-        metadata    (some-> metadata db/json)
-        modified-at (or modified-at created-at)]
-
-    (db/exec-one! cfg [sql:upsert-file-data
-                       file-id id
-                       created-at
-                       modified-at
-                       deleted-at
-                       type
-                       backend
-                       metadata
-                       data
-                       modified-at
-                       deleted-at
-                       backend
-                       metadata
-                       data])))
-
-(defn- handle-persistence
-  [cfg {:keys [type backend id file-id data] :as params}]
-
-  (cond
-    (= backend "storage")
-    (let [storage  (sto/resolve cfg)
-          content  (sto/content data)
-          sobject  (sto/put-object! storage
-                                    {::sto/content content
-                                     ::sto/touch true
-                                     :bucket "file-data"
-                                     :content-type "application/octet-stream"
-                                     :file-id file-id
-                                     :id id})
-          metadata {:storage-ref-id (:id sobject)}
-          params   (-> params
-                       (assoc :metadata metadata)
-                       (assoc :data nil))]
-      (upsert-in-database cfg params))
-
-    (= backend "db")
-    (->> (dissoc params :metadata)
-         (upsert-in-database cfg))
-
-    (= backend "legacy-db")
-    (cond
-      (= type "main")
-      (do
-        (db/delete! cfg :file-data
-                    {:id id :file-id file-id :type "main"}
-                    {::db/return-keys false})
-        (db/update! cfg :file
-                    {:data data}
-                    {:id file-id}
-                    {::db/return-keys false}))
-
-      (= type "snapshot")
-      (do
-        (db/delete! cfg :file-data
-                    {:id id :file-id file-id :type "snapshot"}
-                    {::db/return-keys false})
-        (db/update! cfg :file-change
-                    {:data data}
-                    {:file-id file-id :id id}
-                    {::db/return-keys false}))
-
-      (= type "fragment")
-      (upsert-in-database cfg
-                          (-> (dissoc params :metadata)
-                              (assoc :backend "db")))
-
-      :else
-      (throw (RuntimeException. "not implemented")))
-
-    :else
-    (throw (IllegalArgumentException.
-            (str "backend '" backend "' not supported")))))
-
-(defn process-metadata
-  [cfg metadata]
-  (when-let [storage-id (:storage-ref-id metadata)]
-    (let [storage (sto/resolve cfg ::db/reuse-conn true)]
-      (sto/touch-object! storage storage-id))))
-
-(defn- default-backend
-  [backend]
-  (or backend (cf/get :file-data-backend)))
-
-(def ^:private schema:metadata
-  [:map {:title "Metadata"}
-   [:storage-ref-id {:optional true} ::sm/uuid]])
-
-(def decode-metadata-with-schema
-  (sm/decoder schema:metadata sm/json-transformer))
-
-(defn decode-metadata
-  [metadata]
-  (some-> metadata
-          (db/decode-json-pgobject)
-          (decode-metadata-with-schema)))
-
-(def ^:private schema:update-params
-  [:map {:closed true}
-   [:id ::sm/uuid]
-   [:type [:enum "main" "snapshot" "fragment"]]
-   [:file-id ::sm/uuid]
-   [:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
-   [:metadata {:optional true} [:maybe schema:metadata]]
-   [:data {:optional true} bytes?]
-   [:created-at {:optional true} ::ct/inst]
-   [:modified-at {:optional true} [:maybe ::ct/inst]]
-   [:deleted-at {:optional true} [:maybe ::ct/inst]]])
-
-(def ^:private check-update-params
-  (sm/check-fn schema:update-params :hint "invalid params received for update"))
-
-(defn upsert!
-  "Create or update file data"
-  [cfg params & {:as opts}]
-  (let [params (-> (check-update-params params)
-                   (update :backend default-backend))]
-
-    (some->> (:metadata params)
-             (process-metadata cfg))
-
-    (-> (handle-persistence cfg params)
-        (db/get-update-count)
-        (pos?))))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; POINTER-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn get-file-data
+  "Get file data given a file instance."
+  [system file]
+  (if (offloaded? file)
+    (let [storage (sto/resolve system ::db/reuse-conn true)]
+      (->> (sto/get-object storage (:data-ref-id file))
+           (sto/get-object-bytes storage)))
+    (:data file)))
+
+(defn resolve-file-data
+  [system file]
+  (let [data (get-file-data system file)]
+    (assoc file :data data)))
+
 (defn load-pointer
   "A database loader pointer helper"
-  [cfg file-id id]
-  (let [fragment (some-> (db/get* cfg :file-data
-                                  {:id id :file-id file-id :type "fragment"}
-                                  {::sql/columns [:data :backend :id :metadata]})
-                         (update :metadata decode-metadata))]
+  [system file-id id]
+  (let [fragment (db/get* system :file-data-fragment
+                          {:id id :file-id file-id}
+                          {::sql/columns [:data :data-backend :data-ref-id :id]})]
 
     (l/trc :hint "load pointer"
            :file-id (str file-id)
@@ -278,21 +100,22 @@
                 :file-id file-id
                 :fragment-id id))
 
-    (-> (resolve-file-data cfg fragment)
-        (get :data)
-        (blob/decode))))
+    (let [data (get-file-data system fragment)]
+      ;; FIXME: conditional thread scheduling for decoding big objects
+      (blob/decode data))))
 
 (defn persist-pointers!
   "Persist all currently tracked pointer objects"
-  [cfg file-id]
-  (doseq [[id item] @pmap/*tracked*]
-    (when (pmap/modified? item)
-      (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
-      (let [content (-> item deref blob/encode)]
-        (upsert! cfg {:id id
-                      :file-id file-id
-                      :type "fragment"
-                      :data content})))))
+  [system file-id]
+  (let [conn (db/get-connection system)]
+    (doseq [[id item] @pmap/*tracked*]
+      (when (pmap/modified? item)
+        (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
+        (let [content (-> item deref blob/encode)]
+          (db/insert! conn :file-data-fragment
+                      {:id id
+                       :file-id file-id
+                       :data content}))))))
 
 (defn process-pointers
   "Apply a function to all pointers on the file. Usuly used for
@@ -306,14 +129,6 @@
         (d/update-vals update-fn')
         (update :pages-index d/update-vals update-fn'))))
 
-(defn realize-pointers
-  "Process a file and remove all instances of pointers realizing them to
-  a plain data. Used in operation where is more efficient have the
-  whole file loaded in memory."
-  [cfg {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial load-pointer cfg id)]
-    (update file :data process-pointers deref)))
-
 (defn get-used-pointer-ids
   "Given a file, return all pointer ids used in the data."
   [fdata]
@@ -333,12 +148,47 @@
       (update :features conj "fdata/pointer-map")))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; GENERAL PURPOSE HELPERS
+;; PATH-DATA
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn realize
-  "A helper that combines realize-pointers and realize-objects"
-  [cfg file]
-  (->> file
-       (realize-pointers cfg)
-       (realize-objects cfg)))
+(defn enable-path-data
+  "Enable the fdata/path-data feature on the file."
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content path/content)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features conj "fdata/path-data"))))
+
+(defn disable-path-data
+  [file & _opts]
+  (letfn [(update-object [object]
+            (if (or (cfh/path-shape? object)
+                    (cfh/bool-shape? object))
+              (update object :content vec)
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (when-let [conn db/*conn*]
+      (db/delete! conn :file-migration {:file-id (:id file)
+                                        :name "0003-convert-path-content"}))
+    (-> file
+        (update :data (fn [data]
+                        (-> data
+                            (update :pages-index d/update-vals update-container)
+                            (d/update-when :components d/update-vals update-container))))
+        (update :features disj "fdata/path-data")
+        (update :migrations disj "0003-convert-path-content")
+        (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))

backend/src/app/features/file_migrations.clj

@@ -8,7 +8,6 @@
   "Backend specific code for file migrations. Implemented as permanent feature of files."
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
    [app.common.files.migrations :as fmg :refer [xf:map-name]]
    [app.db :as db]
    [app.db.sql :as-alias sql]))
@@ -27,27 +26,14 @@
 (defn upsert-migrations!
   "Persist or update file migrations. Return the updated/inserted number
   of rows"
-  [cfg {:keys [id] :as file}]
-  (let [conn       (db/get-connection cfg)
-        migrations (or (-> file meta ::fmg/migrated)
-                       (-> file :migrations))
+  [conn {:keys [id] :as file}]
+  (let [migrations (or (-> file meta ::fmg/migrated)
+                       (-> file :migrations not-empty)
+                       fmg/available-migrations)
         columns    [:file-id :name]
-        rows       (->> migrations
-                        (mapv (fn [name] [id name]))
-                        (not-empty))]
-
-    (when-not rows
-      (ex/raise :type :internal
-                :code :missing-migrations
-                :hint "no migrations available on file"))
+        rows       (mapv (fn [name] [id name]) migrations)]
 
     (-> (db/insert-many! conn :file-migration columns rows
                          {::db/return-keys false
                           ::sql/on-conflict-do-nothing true})
         (db/get-update-count))))
-
-(defn reset-migrations!
-  "Replace file migrations"
-  [cfg {:keys [id] :as file}]
-  (db/delete! cfg :file-migration {:file-id id})
-  (upsert-migrations! cfg file))

backend/src/app/features/file_snapshots.clj

@@ -1,446 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.features.file-snapshots
-  (:require
-   [app.binfile.common :as bfc]
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.features :as-alias cfeat]
-   [app.common.files.migrations :as fmg]
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.uuid :as uuid]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.db.sql :as-alias sql]
-   [app.features.fdata :as fdata]
-   [app.storage :as sto]
-   [app.util.blob :as blob]
-   [app.worker :as wrk]
-   [cuerdas.core :as str]))
-
-(def sql:snapshots
-  "SELECT c.id,
-          c.label,
-          c.created_at,
-          c.updated_at AS modified_at,
-          c.deleted_at,
-          c.profile_id,
-          c.created_by,
-          c.locked_by,
-          c.revn,
-          c.features,
-          c.migrations,
-          c.version,
-          c.file_id,
-          c.data AS legacy_data,
-          fd.data AS data,
-          coalesce(fd.backend, 'legacy-db') AS backend,
-          fd.metadata AS metadata
-     FROM file_change AS c
-     LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id
-                                   AND fd.id = c.id
-                                   AND fd.type = 'snapshot')
-    WHERE c.label IS NOT NULL")
-
-(defn- decode-snapshot
-  [snapshot]
-  (some-> snapshot
-          (-> (d/update-when :metadata fdata/decode-metadata)
-              (d/update-when :migrations db/decode-pgarray [])
-              (d/update-when :features db/decode-pgarray #{}))))
-
-(def ^:private sql:get-minimal-file
-  "SELECT f.id,
-          f.revn,
-          f.modified_at,
-          f.deleted_at,
-          fd.backend AS backend,
-          fd.metadata AS metadata
-     FROM file AS f
-     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
-    WHERE f.id = ?")
-
-(defn- get-minimal-file
-  [cfg id & {:as opts}]
-  (-> (db/get-with-sql cfg [sql:get-minimal-file id] opts)
-      (d/update-when :metadata fdata/decode-metadata)))
-
-(def ^:private sql:get-snapshot-without-data
-  (str "WITH snapshots AS (" sql:snapshots ")"
-       "SELECT c.id,
-               c.label,
-               c.revn,
-               c.created_at,
-               c.modified_at,
-               c.deleted_at,
-               c.profile_id,
-               c.created_by,
-               c.locked_by,
-               c.features,
-               c.metadata,
-               c.migrations,
-               c.version,
-               c.file_id
-         FROM snapshots AS c
-        WHERE c.id = ?
-          AND CASE WHEN c.created_by = 'user'
-                   THEN c.deleted_at IS NULL
-                   WHEN c.created_by = 'system'
-                   THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz
-               END"))
-
-(defn get-minimal-snapshot
-  [cfg snapshot-id]
-  (let [now (ct/now)]
-    (-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id now]
-                         {::db/remove-deleted false})
-        (decode-snapshot))))
-
-(def ^:private sql:get-snapshot
-  (str sql:snapshots
-       " AND c.file_id = ?
-         AND c.id = ?
-         AND CASE WHEN c.created_by = 'user'
-                  THEN (c.deleted_at IS NULL)
-                  WHEN c.created_by = 'system'
-                  THEN (c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz)
-              END"))
-
-(defn- get-snapshot
-  "Get snapshot with decoded data"
-  [cfg file-id snapshot-id]
-  (let [now (ct/now)]
-    (->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now]
-                          {::db/remove-deleted false})
-         (decode-snapshot)
-         (fdata/resolve-file-data cfg)
-         (fdata/decode-file-data cfg))))
-
-(def ^:private sql:get-visible-snapshots
-  (str "WITH "
-       "snapshots1 AS ( " sql:snapshots "),"
-       "snapshots2 AS (
-          SELECT c.id,
-                 c.label,
-                 c.revn,
-                 c.version,
-                 c.created_at,
-                 c.modified_at,
-                 c.created_by,
-                 c.locked_by,
-                 c.profile_id,
-                 c.deleted_at
-            FROM snapshots1 AS c
-           WHERE c.file_id = ?
-       ), snapshots3 AS (
-          (SELECT * FROM snapshots2
-            WHERE created_by = 'system'
-              AND (deleted_at IS NULL OR
-                   deleted_at >= ?::timestamptz)
-            LIMIT 500)
-          UNION ALL
-          (SELECT * FROM snapshots2
-            WHERE created_by = 'user'
-              AND deleted_at IS NULL
-            LIMIT 500)
-       )
-       SELECT * FROM snapshots3
-        ORDER BY created_at DESC"))
-
-(defn get-visible-snapshots
-  "Return a list of snapshots fecheable from the API, it has a limited
-  set of fields and applies big but safe limits over all available
-  snapshots. It return a ordered vector by the snapshot date of
-  creation."
-  [cfg file-id]
-  (let [now (ct/now)]
-    (->> (db/exec! cfg [sql:get-visible-snapshots file-id now])
-         (mapv decode-snapshot))))
-
-(def ^:private schema:decoded-file
-  [:map {:title "DecodedFile"}
-   [:id ::sm/uuid]
-   [:revn :int]
-   [:vern :int]
-   [:data :map]
-   [:version :int]
-   [:features ::cfeat/features]
-   [:migrations [::sm/set :string]]])
-
-(def ^:private schema:snapshot
-  [:map {:title "Snapshot"}
-   [:id ::sm/uuid]
-   [:revn [::sm/int {:min 0}]]
-   [:version [::sm/int {:min 0}]]
-   [:features ::cfeat/features]
-   [:migrations [::sm/set ::sm/text]]
-   [:profile-id {:optional true} ::sm/uuid]
-   [:label ::sm/text]
-   [:file-id ::sm/uuid]
-   [:created-by [:enum "system" "user" "admin"]]
-   [:deleted-at {:optional true} ::ct/inst]
-   [:modified-at ::ct/inst]
-   [:created-at ::ct/inst]])
-
-(def ^:private check-snapshot
-  (sm/check-fn schema:snapshot))
-
-(def ^:private check-decoded-file
-  (sm/check-fn schema:decoded-file))
-
-(defn- generate-snapshot-label
-  []
-  (let [ts (-> (ct/now)
-               (ct/format-inst)
-               (str/replace #"[T:\.]" "-")
-               (str/rtrim "Z"))]
-    (str "snapshot-" ts)))
-
-(def ^:private schema:create-params
-  [:map {:title "SnapshotCreateParams"}
-   [:profile-id ::sm/uuid]
-   [:created-by {:optional true} [:enum "user" "system"]]
-   [:label {:optional true} ::sm/text]
-   [:session-id {:optional true} ::sm/uuid]
-   [:modified-at {:optional true} ::ct/inst]
-   [:deleted-at {:optional true} ::ct/inst]])
-
-(def ^:private check-create-params
-  (sm/check-fn schema:create-params))
-
-(defn create!
-  "Create a file snapshot; expects a non-encoded file"
-  [cfg file & {:as params}]
-  (let [{:keys [label created-by deleted-at profile-id session-id]}
-        (check-create-params params)
-
-        file
-        (check-decoded-file file)
-
-        created-by
-        (or created-by "system")
-
-        snapshot-id
-        (uuid/next)
-
-        created-at
-        (ct/now)
-
-        deleted-at
-        (or deleted-at
-            (if (= created-by "system")
-              (ct/in-future (cf/get-deletion-delay))
-              nil))
-
-        label
-        (or label (generate-snapshot-label))
-
-        snapshot
-        (cond-> {:id snapshot-id
-                 :revn (:revn file)
-                 :version (:version file)
-                 :file-id (:id file)
-                 :features (:features file)
-                 :migrations (:migrations file)
-                 :label label
-                 :created-at created-at
-                 :modified-at created-at
-                 :created-by created-by}
-
-          deleted-at
-          (assoc :deleted-at deleted-at)
-
-          :always
-          (check-snapshot))]
-
-    (db/insert! cfg :file-change
-                (-> snapshot
-                    (update :features into-array)
-                    (update :migrations into-array)
-                    (assoc :updated-at created-at)
-                    (assoc :profile-id profile-id)
-                    (assoc :session-id session-id)
-                    (dissoc :modified-at))
-                {::db/return-keys false})
-
-    (fdata/upsert! cfg
-                   {:id snapshot-id
-                    :file-id (:id file)
-                    :type "snapshot"
-                    :data (blob/encode (:data file))
-                    :created-at created-at
-                    :deleted-at deleted-at})
-
-    snapshot))
-
-(def ^:private schema:update-params
-  [:map {:title "SnapshotUpdateParams"}
-   [:id ::sm/uuid]
-   [:file-id ::sm/uuid]
-   [:label ::sm/text]
-   [:modified-at {:optional true} ::ct/inst]])
-
-(def ^:private check-update-params
-  (sm/check-fn schema:update-params))
-
-(defn update!
-  [cfg params]
-
-  (let [{:keys [id file-id label modified-at]}
-        (check-update-params params)
-
-        modified-at
-        (or modified-at (ct/now))]
-
-    (db/update! cfg :file-data
-                {:deleted-at nil
-                 :modified-at modified-at}
-                {:file-id file-id
-                 :id id
-                 :type "snapshot"}
-                {::db/return-keys false})
-
-    (-> (db/update! cfg :file-change
-                    {:label label
-                     :created-by "user"
-                     :updated-at modified-at
-                     :deleted-at nil}
-                    {:file-id file-id
-                     :id id}
-                    {::db/return-keys false})
-        (db/get-update-count)
-        (pos?))))
-
-(defn restore!
-  [{:keys [::db/conn] :as cfg} file-id snapshot-id]
-  (let [file (get-minimal-file conn file-id {::db/for-update true})
-        vern (rand-int Integer/MAX_VALUE)
-
-        storage
-        (sto/resolve cfg {::db/reuse-conn true})
-
-        snapshot
-        (get-snapshot cfg file-id snapshot-id)]
-
-    (when-not snapshot
-      (ex/raise :type :not-found
-                :code :snapshot-not-found
-                :hint "unable to find snapshot with the provided label"
-                :snapshot-id snapshot-id
-                :file-id file-id))
-
-    (when-not (:data snapshot)
-      (ex/raise :type :internal
-                :code :snapshot-without-data
-                :hint "snapshot has no data"
-                :label (:label snapshot)
-                :file-id file-id))
-
-    (let [;; If the snapshot has applied migrations stored, we reuse
-          ;; them, if not, we take a safest set of migrations as
-          ;; starting point. This is because, at the time of
-          ;; implementing snapshots, migrations were not taken into
-          ;; account so we need to make this backward compatible in
-          ;; some way.
-          migrations
-          (or (:migrations snapshot)
-              (fmg/generate-migrations-from-version 67))
-
-          file
-          (-> file
-              (update :revn inc)
-              (assoc :migrations migrations)
-              (assoc :data (:data snapshot))
-              (assoc :vern vern)
-              (assoc :version (:version snapshot))
-              (assoc :has-media-trimmed false)
-              (assoc :modified-at (:modified-at snapshot))
-              (assoc :features (:features snapshot)))]
-
-      (l/dbg :hint "restoring snapshot"
-             :file-id (str file-id)
-             :label (:label snapshot)
-             :snapshot-id (str (:id snapshot)))
-
-      ;; In the same way, on reseting the file data, we need to restore
-      ;; the applied migrations on the moment of taking the snapshot
-      (bfc/update-file! cfg file ::bfc/reset-migrations? true)
-
-      ;; FIXME: this should be separated functions, we should not have
-      ;; inline sql here.
-
-      ;; clean object thumbnails
-      (let [sql (str "update file_tagged_object_thumbnail "
-                     "   set deleted_at = now() "
-                     " where file_id=? returning media_id")
-            res (db/exec! conn [sql file-id])]
-        (doseq [media-id (into #{} (keep :media-id) res)]
-          (sto/touch-object! storage media-id)))
-
-      ;; clean file thumbnails
-      (let [sql (str "update file_thumbnail "
-                     "   set deleted_at = now() "
-                     " where file_id=? returning media_id")
-            res (db/exec! conn [sql file-id])]
-        (doseq [media-id (into #{} (keep :media-id) res)]
-          (sto/touch-object! storage media-id)))
-
-      vern)))
-
-(defn delete!
-  [cfg & {:keys [id file-id deleted-at]}]
-  (assert (uuid? id) "missing id")
-  (assert (uuid? file-id) "missing file-id")
-  (assert (ct/inst? deleted-at) "missing deleted-at")
-
-  (wrk/submit! {::db/conn (db/get-connection cfg)
-                ::wrk/task :delete-object
-                ::wrk/params {:object :snapshot
-                              :deleted-at deleted-at
-                              :file-id file-id
-                              :id id}})
-  (db/update! cfg :file-change
-              {:deleted-at deleted-at}
-              {:id id :file-id file-id}
-              {::db/return-keys false})
-  true)
-
-(def ^:private sql:get-snapshots
-  (str sql:snapshots " AND c.file_id = ?"))
-
-(defn lock-by!
-  [conn id profile-id]
-  (-> (db/update! conn :file-change
-                  {:locked-by profile-id}
-                  {:id id}
-                  {::db/return-keys false})
-      (db/get-update-count)
-      (pos?)))
-
-(defn unlock!
-  [conn id]
-  (-> (db/update! conn :file-change
-                  {:locked-by nil}
-                  {:id id}
-                  {::db/return-keys false})
-      (db/get-update-count)
-      (pos?)))
-
-(defn reduce-snapshots
-  "Process the file snapshots using efficient reduction; the file
-  reduction comes with all snapshots, including maked as deleted"
-  [cfg file-id xform f init]
-  (let [conn  (db/get-connection cfg)
-        xform (comp
-               (map (partial fdata/resolve-file-data cfg))
-               (map (partial fdata/decode-file-data cfg))
-               xform)]
-
-    (->> (db/plan conn [sql:get-snapshots file-id] {:fetch-size 1})
-         (transduce xform f init))))

backend/src/app/features/logical_deletion.clj

@@ -1,32 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.features.logical-deletion
-  "A code related to handle logical deletion mechanism"
-  (:require
-   [app.common.time :as ct]
-   [app.config :as cf]))
-
-(def ^:private canceled-status
-  #{"canceled" "unpaid"})
-
-(defn get-deletion-delay
-  "Calculate the next deleted-at for a resource (file, team, etc) in function
-  of team settings"
-  [team]
-  (if-let [{:keys [type status]} (get team :subscription)]
-    (cond
-      (and (= "unlimited" type) (not (contains? canceled-status status)))
-      (ct/duration {:days 30})
-
-      (and (= "enterprise" type) (not (contains? canceled-status status)))
-      (ct/duration {:days 90})
-
-      :else
-      (cf/get-deletion-delay))
-
-    (cf/get-deletion-delay)))
-

backend/src/app/http.clj

@@ -17,9 +17,7 @@
    [app.http.awsns :as-alias awsns]
    [app.http.debug :as-alias debug]
    [app.http.errors :as errors]
-   [app.http.management :as mgmt]
    [app.http.middleware :as mw]
-   [app.http.security :as sec]
    [app.http.session :as session]
    [app.http.websocket :as-alias ws]
    [app.main :as-alias main]
@@ -28,6 +26,7 @@
    [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
    [yetti.adapter :as yt]
@@ -54,8 +53,6 @@
   [:map
    [::port ::sm/int]
    [::host ::sm/text]
-   [::io-threads {:optional true} ::sm/int]
-   [::max-worker-threads {:optional true} ::sm/int]
    [::max-body-size {:optional true} ::sm/int]
    [::max-multipart-body-size {:optional true} ::sm/int]
    [::router {:optional true} [:fn r/router?]]
@@ -66,41 +63,30 @@
   (assert (sm/check schema:server-params params)))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [::handler ::router ::host ::port ::mtx/metrics] :as cfg}]
+  [_ {:keys [::handler ::router ::host ::port] :as cfg}]
   (l/info :hint "starting http server" :port port :host host)
-  (let [on-dispatch
-        (fn [_ start-at-ns]
-          (let [timing (- (System/nanoTime) start-at-ns)
-                timing (int (/ timing 1000000))]
-            (mtx/run! metrics
-                      :id :http-server-dispatch-timing
-                      :val timing)))
+  (let [options {:http/port port
+                 :http/host host
+                 :http/max-body-size (::max-body-size cfg)
+                 :http/max-multipart-body-size (::max-multipart-body-size cfg)
+                 :xnio/io-threads (or (::io-threads cfg)
+                                      (max 3 (px/get-available-processors)))
+                 :xnio/dispatch :virtual
+                 :ring/compat :ring2
+                 :socket/backlog 4069}
 
-        options
-        {:http/port port
-         :http/host host
-         :http/max-body-size (::max-body-size cfg)
-         :http/max-multipart-body-size (::max-multipart-body-size cfg)
-         :xnio/direct-buffers false
-         :xnio/io-threads (::io-threads cfg)
-         :xnio/max-worker-threads (::max-worker-threads cfg)
-         :ring/compat :ring2
-         :events/on-dispatch on-dispatch
-         :socket/backlog 4069}
+        handler (cond
+                  (some? router)
+                  (router-handler router)
 
-        handler
-        (cond
-          (some? router)
-          (router-handler router)
+                  (some? handler)
+                  handler
 
-          (some? handler)
-          handler
+                  :else
+                  (throw (UnsupportedOperationException. "handler or router are required")))
 
-          :else
-          (throw (UnsupportedOperationException. "handler or router are required")))
-
-        server
-        (yt/server handler (d/without-nils options))]
+        options (d/without-nils options)
+        server  (yt/server handler options)]
 
     (assoc cfg ::server (yt/start! server))))
 
@@ -155,7 +141,6 @@
    [::debug/routes schema:routes]
    [::mtx/routes schema:routes]
    [::awsns/routes schema:routes]
-   [::mgmt/routes schema:routes]
    ::session/manager
    ::setup/props
    ::db/pool])
@@ -168,7 +153,6 @@
   [_ cfg]
   (rr/router
    [["" {:middleware [[mw/server-timing]
-                      [sec/sec-fetch-metadata]
                       [mw/params]
                       [mw/format-response]
                       [session/soft-auth cfg]
@@ -184,13 +168,9 @@
      ["/webhooks"
       (::awsns/routes cfg)]
 
-     ["/management"
-      (::mgmt/routes cfg)]
-
      (::ws/routes cfg)
 
-     ["/api" {:middleware [[mw/cors]
-                           [sec/client-header-check]]}
+     ["/api" {:middleware [[mw/cors]]}
       (::oidc/routes cfg)
       (::rpc.doc/routes cfg)
       (::rpc/routes cfg)]]]))

backend/src/app/http/access_token.clj

@@ -14,18 +14,18 @@
    [app.tokens :as tokens]
    [yetti.request :as yreq]))
 
-(def header-re #"(?i)^Token\s+(.*)")
+(def header-re #"^Token\s+(.*)")
 
-(defn get-token
+(defn- get-token
   [request]
   (some->> (yreq/get-header request "authorization")
            (re-matches header-re)
            (second)))
 
 (defn- decode-token
-  [cfg token]
+  [props token]
   (when token
-    (tokens/verify cfg {:token token :iss "access-token"})))
+    (tokens/verify props {:token token :iss "access-token"})))
 
 (def sql:get-token-data
   "SELECT perms, profile_id, expires_at
@@ -43,11 +43,11 @@
 (defn- wrap-soft-auth
   "Soft Authentication, will be executed synchronously on the undertow
   worker thread."
-  [handler cfg]
+  [handler {:keys [::setup/props]}]
   (letfn [(handle-request [request]
             (try
               (let [token  (get-token request)
-                    claims (decode-token cfg token)]
+                    claims (decode-token props token)]
                 (cond-> request
                   (map? claims)
                   (assoc ::id (:tid claims))))

backend/src/app/http/assets.clj

@@ -9,18 +9,18 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.db :as db]
    [app.storage :as sto]
+   [app.util.time :as dt]
    [integrant.core :as ig]
    [yetti.response :as-alias yres]))
 
 (def ^:private cache-max-age
-  (ct/duration {:hours 24}))
+  (dt/duration {:hours 24}))
 
 (def ^:private signature-max-age
-  (ct/duration {:hours 24 :minutes 15}))
+  (dt/duration {:hours 24 :minutes 15}))
 
 (defn get-id
   [{:keys [path-params]}]

backend/src/app/http/awsns.clj

@@ -17,9 +17,11 @@
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.worker :as-alias wrk]
    [clojure.data.json :as j]
    [cuerdas.core :as str]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [yetti.request :as yreq]
    [yetti.response :as-alias yres]))
 
@@ -38,8 +40,8 @@
   [_ cfg]
   (letfn [(handler [request]
             (let [data (-> request yreq/body slurp)]
-              (handle-request cfg data)
-              {::yres/status 200}))]
+              (px/run! :vthread (partial handle-request cfg data)))
+            {::yres/status 200})]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 
@@ -107,7 +109,7 @@
   [cfg headers]
   (let [tdata (get headers "x-penpot-data")]
     (when-not (str/empty? tdata)
-      (let [result (tokens/verify cfg {:token tdata :iss :profile-identity})]
+      (let [result (tokens/verify (::setup/props cfg) {:token tdata :iss :profile-identity})]
         (:profile-id result)))))
 
 (defn- parse-notification

backend/src/app/http/debug.clj

@@ -15,24 +15,21 @@
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
-   [app.common.time :as ct]
-   [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.file-migrations :as feat.fmig]
    [app.http.session :as session]
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
-   [app.setup.clock :as clock]
    [app.srepl.main :as srepl]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [app.util.blob :as blob]
    [app.util.template :as tmpl]
+   [app.util.time :as dt]
    [cuerdas.core :as str]
    [datoteka.io :as io]
    [emoji.core :as emj]
@@ -50,35 +47,29 @@
 
 (defn index-handler
   [_cfg _request]
-  (let [{:keys [clock offset]} @clock/current]
-    {::yres/status  200
-     ::yres/headers {"content-type" "text/html"}
-     ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
-                        (tmpl/render {:version (:full cf/version)
-                                      :current-clock  (str clock)
-                                      :current-offset (if offset
-                                                        (ct/format-duration offset)
-                                                        "NO OFFSET")
-                                      :current-time  (ct/format-inst (ct/now) :http)
-                                      :supported-features cfeat/supported-features}))}))
+  {::yres/status  200
+   ::yres/headers {"content-type" "text/html"}
+   ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
+                      (tmpl/render {:version (:full cf/version)}))})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE CHANGES
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- get-resolved-file
-  [cfg file-id]
-  (some-> (bfc/get-file cfg file-id :migrate? false)
-          (update :data blob/encode)))
+(defn prepare-response
+  [body]
+  (let [headers {"content-type" "application/transit+json"}]
+    {::yres/status 200
+     ::yres/body body
+     ::yres/headers headers}))
 
-(defn prepare-download
-  [file filename]
-  {::yres/status 200
-   ::yres/headers
-   {"content-disposition" (str "attachment; filename=" filename ".json")
-    "content-type" "application/octet-stream"}
-   ::yres/body
-   (t/encode file {:type :json-verbose})})
+(defn prepare-download-response
+  [body filename]
+  (let [headers {"content-disposition" (str "attachment; filename=" filename)
+                 "content-type" "application/octet-stream"}]
+    {::yres/status 200
+     ::yres/body body
+     ::yres/headers headers}))
 
 (def sql:retrieve-range-of-changes
   "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
@@ -86,51 +77,45 @@
 (def sql:retrieve-single-change
   "select revn, changes, data from file_change where file_id=? and revn = ?")
 
-(defn- download-file-data
-  [cfg {:keys [params ::session/profile-id] :as request}]
+(defn- retrieve-file-data
+  [{:keys [::db/pool]} {:keys [params ::session/profile-id] :as request}]
   (let [file-id  (some-> params :file-id parse-uuid)
+        revn     (some-> params :revn parse-long)
         filename (str file-id)]
 
     (when-not file-id
       (ex/raise :type :validation
                 :code :missing-arguments))
 
-    (if-let [file (get-resolved-file cfg file-id)]
+    (let [data (if (integer? revn)
+                 (some-> (db/exec-one! pool [sql:retrieve-single-change file-id revn]) :data)
+                 (some-> (db/get-by-id pool :file file-id) :data))]
+
+      (when-not data
+        (ex/raise :type :not-found
+                  :code :enpty-data
+                  :hint "empty response"))
       (cond
         (contains? params :download)
-        (prepare-download file filename)
+        (prepare-download-response data filename)
 
         (contains? params :clone)
-        (db/tx-run! cfg
-                    (fn [{:keys [::db/conn] :as cfg}]
-                      (let [profile    (profile/get-profile conn profile-id)
-                            project-id (:default-project-id profile)
-                            file       (-> (create-file cfg {:id (uuid/next)
-                                                             :name (str "Cloned: " (:name file))
-                                                             :features (:features file)
-                                                             :project-id project-id
-                                                             :profile-id profile-id})
-                                           (assoc :data (:data file))
-                                           (assoc :migrations (:migrations file)))]
+        (let [profile    (profile/get-profile pool profile-id)
+              project-id (:default-project-id profile)]
 
-                        (feat.fmig/reset-migrations! conn file)
-                        (db/update! conn :file
-                                    {:data (:data file)}
-                                    {:id (:id file)}
-                                    {::db/return-keys false})
-
-
-                        {::yres/status 201
-                         ::yres/body "OK CLONED"})))
+          (db/run! pool (fn [{:keys [::db/conn] :as cfg}]
+                          (create-file cfg {:id file-id
+                                            :name (str "Cloned file: " filename)
+                                            :project-id project-id
+                                            :profile-id profile-id})
+                          (db/update! conn :file
+                                      {:data data}
+                                      {:id file-id})
+                          {::yres/status 201
+                           ::yres/body "OK CREATED"})))
 
         :else
-        (ex/raise :type :validation
-                  :code :invalid-params
-                  :hint "invalid button"))
-
-      (ex/raise :type :not-found
-                :code :enpty-data
-                :hint "empty response"))))
+        (prepare-response (blob/decode data))))))
 
 (defn- is-file-exists?
   [pool id]
@@ -138,61 +123,81 @@
     (-> (db/exec-one! pool [sql id]) :exists)))
 
 (defn- upload-file-data
-  [{:keys [::db/pool] :as cfg} {:keys [::session/profile-id params] :as request}]
+  [{:keys [::db/pool]} {:keys [::session/profile-id params] :as request}]
   (let [profile    (profile/get-profile pool profile-id)
         project-id (:default-project-id profile)
-        file       (some-> params :file :path io/read* t/decode)]
+        data       (some-> params :file :path io/read*)]
 
-    (if (and file project-id)
-      (let [fname     (str "Imported: " (:name file) "(" (ct/now) ")")
+    (if (and data project-id)
+      (let [fname     (str "Imported file *: " (dt/now))
             reuse-id? (contains? params :reuseid)
             file-id   (or (and reuse-id? (ex/ignoring (-> params :file :filename parse-uuid)))
                           (uuid/next))]
 
         (if (and reuse-id? file-id
                  (is-file-exists? pool file-id))
-          (db/tx-run! cfg
-                      (fn [{:keys [::db/conn] :as cfg}]
-                        (db/update! conn :file
-                                    {:data (:data file)
-                                     :features (into-array (:features file))
-                                     :deleted-at nil}
-                                    {:id file-id}
-                                    {::db/return-keys false})
-                        (feat.fmig/reset-migrations! conn file)
-                        {::yres/status 200
-                         ::yres/body "OK UPDATED"}))
-
-          (db/tx-run! cfg
-                      (fn [{:keys [::db/conn] :as cfg}]
-                        (let [file (-> (create-file cfg {:id file-id
-                                                         :name fname
-                                                         :features (:features file)
-                                                         :project-id project-id
-                                                         :profile-id profile-id})
-                                       (assoc :data (:data file))
-                                       (assoc :migrations (:migrations file)))]
+          (do
+            (db/update! pool :file
+                        {:data data
+                         :deleted-at nil}
+                        {:id file-id})
+            {::yres/status 200
+             ::yres/body "OK UPDATED"})
 
+          (db/run! pool (fn [{:keys [::db/conn] :as cfg}]
+                          (create-file cfg {:id file-id
+                                            :name fname
+                                            :project-id project-id
+                                            :profile-id profile-id})
                           (db/update! conn :file
-                                      {:data (:data file)}
-                                      {:id file-id}
-                                      {::db/return-keys false})
-                          (feat.fmig/reset-migrations! conn file)
+                                      {:data data}
+                                      {:id file-id})
                           {::yres/status 201
-                           ::yres/body "OK CREATED"})))))
+                           ::yres/body "OK CREATED"}))))
 
-      (ex/raise :type :validation
-                :code :invalid-params
-                :hint "invalid file uploaded"))))
+      {::yres/status 500
+       ::yres/body "ERROR"})))
 
-(defn raw-export-import-handler
+(defn file-data-handler
   [cfg request]
   (case (yreq/method request)
-    :get (download-file-data cfg request)
+    :get (retrieve-file-data cfg request)
     :post (upload-file-data cfg request)
     (ex/raise :type :http
               :code :method-not-found)))
 
+(defn file-changes-handler
+  [{:keys [::db/pool]} {:keys [params] :as request}]
+  (letfn [(retrieve-changes [file-id revn]
+            (if (str/includes? revn ":")
+              (let [[start end] (->> (str/split revn #":")
+                                     (map str/trim)
+                                     (map parse-long))]
+                (some->> (db/exec! pool [sql:retrieve-range-of-changes file-id start end])
+                         (map :changes)
+                         (map blob/decode)
+                         (mapcat identity)
+                         (vec)))
+
+              (if-let [revn (parse-long revn)]
+                (let [item (db/exec-one! pool [sql:retrieve-single-change file-id revn])]
+                  (some-> item :changes blob/decode vec))
+                (ex/raise :type :validation :code :invalid-arguments))))]
+
+    (let [file-id  (some-> params :id parse-uuid)
+          revn     (or (some-> params :revn parse-long) "latest")
+          filename (str file-id)]
+
+      (when (or (not file-id) (not revn))
+        (ex/raise :type :validation
+                  :code :invalid-arguments
+                  :hint "missing arguments"))
+
+      (let [data (retrieve-changes file-id revn)]
+        (if (contains? params :download)
+          (prepare-download-response data filename)
+          (prepare-response data))))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; ERROR BROWSER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -229,7 +234,7 @@
             (-> (io/resource "app/templates/error-report.v3.tmpl")
                 (tmpl/render (-> content
                                  (assoc :id id)
-                                 (assoc :created-at (ct/format-inst created-at :rfc1123))))))]
+                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))]
 
     (if-let [report (get-report request)]
       (let [result (case (:version report)
@@ -253,7 +258,7 @@
 (defn error-list-handler
   [{:keys [::db/pool]} _request]
   (let [items (->> (db/exec! pool [sql:error-reports])
-                   (map #(update % :created-at ct/format-inst :rfc1123)))]
+                   (map #(update % :created-at dt/format-instant :rfc1123)))]
     {::yres/status 200
      ::yres/body (-> (io/resource "app/templates/error-list.tmpl")
                      (tmpl/render {:items items}))
@@ -397,67 +402,77 @@
                            ::yres/headers {"content-type" "text/plain"}
                            ::yres/body    (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))))
 
-(defn- handle-team-features
+
+(defn- reset-file-version
   [cfg {:keys [params] :as request}]
-  (let [team-id    (some-> params :team-id d/parse-uuid)
-        feature    (some-> params :feature str)
-        action     (some-> params :action)
+  (let [file-id (some-> params :file-id d/parse-uuid)
+        version (some-> params :version d/parse-integer)]
+
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
+    (when (nil? file-id)
+      (ex/raise :type :validation
+                :code :invalid-file-id
+                :hint "provided invalid file id"))
+
+    (when (nil? version)
+      (ex/raise :type :validation
+                :code :invalid-version
+                :hint "provided invalid version"))
+
+    (db/tx-run! cfg srepl/process-file! file-id #(assoc % :version version))
+
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
+
+
+(defn- add-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id (some-> params :team-id d/parse-uuid)
+        feature (some-> params :feature str)
         skip-check (contains? params :skip-check)]
 
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
+
     (when (nil? team-id)
       (ex/raise :type :validation
                 :code :invalid-team-id
                 :hint "provided invalid team id"))
 
-    (if (= action "show")
-      (let [team (db/run! cfg teams/get-team-info {:id team-id})]
-        {::yres/status 200
-         ::yres/headers {"content-type" "text/plain"}
-         ::yres/body (apply str "Team features:\n"
-                            (->> (:features team)
-                                 (map (fn [feature]
-                                        (str "- " feature "\n")))))})
+    (srepl/enable-team-feature! team-id feature :skip-check skip-check)
 
-      (do
-        (when-not (contains? params :force)
-          (ex/raise :type :validation
-                    :code :missing-force
-                    :hint "missing force checkbox"))
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
 
-        (cond
-          (= action "enable")
-          (srepl/enable-team-feature! team-id feature :skip-check skip-check)
+(defn- remove-team-feature
+  [{:keys [params] :as request}]
+  (let [team-id   (some-> params :team-id d/parse-uuid)
+        feature   (some-> params :feature str)
+        skip-check (contains? params :skip-check)]
 
-          (= action "disable")
-          (srepl/disable-team-feature! team-id feature :skip-check skip-check)
+    (when-not (contains? params :force)
+      (ex/raise :type :validation
+                :code :missing-force
+                :hint "missing force checkbox"))
 
-          :else
-          (ex/raise :type :validation
-                    :code :invalid-action
-                    :hint (str "invalid action: " action)))
+    (when (nil? team-id)
+      (ex/raise :type :validation
+                :code :invalid-team-id
+                :hint "provided invalid team id"))
 
+    (srepl/disable-team-feature! team-id feature :skip-check skip-check)
 
-        {::yres/status  200
-         ::yres/headers {"content-type" "text/plain"}
-         ::yres/body    "OK"}))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; VIRTUAL CLOCK
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- set-virtual-clock
-  [_ {:keys [params] :as request}]
-  (let [offset (some-> params :offset str/trim not-empty ct/duration)
-        reset? (contains? params :reset)]
-    (if (= "production" (cf/get :tenant))
-      {::yres/status 501
-       ::yres/body "OPERATION NOT ALLOWED"}
-      (do
-        (if (or reset? (zero? (inst-ms offset)))
-          (clock/set-offset! nil)
-          (clock/set-offset! offset))
-        {::yres/status 302
-         ::yres/headers {"location" "/dbg"}}))))
+    {::yres/status  200
+     ::yres/headers {"content-type" "text/plain"}
+     ::yres/body    "OK"}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
@@ -510,25 +525,6 @@
            (ex/raise :type :authentication
                      :code :only-admins-allowed)))))})
 
-(def errors
-  (letfn [(handle-error [cause]
-            (when-let [data (ex-data cause)]
-              (when (= :validation (:type data))
-                (str "Error: " (or (:hint data) (ex-message cause)) "\n"))))]
-    {:name ::errors
-     :compile
-     (fn [& _params]
-       (fn [handler]
-         (fn [request]
-           (try
-             (handler request)
-             (catch Throwable cause
-               (let [body (or (handle-error cause)
-                              (ex/format-throwable cause))]
-                 {::yres/status 400
-                  ::yres/headers {"content-type" "text/plain"}
-                  ::yres/body body}))))))}))
-
 (defmethod ig/assert-key ::routes
   [_ params]
   (assert (db/pool? (::db/pool params)) "expected a valid database pool")
@@ -544,14 +540,15 @@
     ["/changelog" {:handler (partial changelog-handler cfg)}]
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
-    ["/actions" {:middleware [[errors]]}
-     ["/set-virtual-clock"
-      {:handler (partial set-virtual-clock cfg)}]
-     ["/resend-email-verification"
-      {:handler (partial resend-email-notification cfg)}]
-     ["/handle-team-features"
-      {:handler (partial handle-team-features cfg)}]
-     ["/file-export" {:handler (partial export-handler cfg)}]
-     ["/file-import" {:handler (partial import-handler cfg)}]
-     ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])
-
+    ["/actions/resend-email-verification"
+     {:handler (partial resend-email-notification cfg)}]
+    ["/actions/reset-file-version"
+     {:handler (partial reset-file-version cfg)}]
+    ["/actions/add-team-feature"
+     {:handler (partial add-team-feature)}]
+    ["/actions/remove-team-feature"
+     {:handler (partial remove-team-feature)}]
+    ["/file/export" {:handler (partial export-handler cfg)}]
+    ["/file/import" {:handler (partial import-handler cfg)}]
+    ["/file/data" {:handler (partial file-data-handler cfg)}]
+    ["/file/changes" {:handler (partial file-changes-handler cfg)}]]])

backend/src/app/http/errors.clj

@@ -25,14 +25,15 @@
   (let [claims (-> {}
                    (into (::session/token-claims request))
                    (into (::actoken/token-claims request)))]
-    (-> (cf/logging-context)
-        (assoc :request/path (:path request))
-        (assoc :request/method (:method request))
-        (assoc :request/params (:params request))
-        (assoc :request/user-agent (yreq/get-header request "user-agent"))
-        (assoc :request/ip-addr (inet/parse-request request))
-        (assoc :request/profile-id (:uid claims))
-        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
+    {:request/path       (:path request)
+     :request/method     (:method request)
+     :request/params     (:params request)
+     :request/user-agent (yreq/get-header request "user-agent")
+     :request/ip-addr    (inet/parse-request request)
+     :request/profile-id (:uid claims)
+     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
+     :version/backend    (:full cf/version)}))
+
 
 (defmulti handle-error
   (fn [cause _ _]
@@ -60,7 +61,8 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
-        (l/wrn :hint "restriction error" :cause err)
+        (l/err :hint "restriction error"
+               :cause err)
         {::yres/status 400
          ::yres/body data}))))
 

backend/src/app/http/management.clj

@@ -1,250 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.management
-  "Internal mangement HTTP API"
-  (:require
-   [app.common.logging :as l]
-   [app.common.schema :as sm]
-   [app.common.schema.generators :as sg]
-   [app.common.time :as ct]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.http.access-token :refer [get-token]]
-   [app.main :as-alias main]
-   [app.rpc.commands.profile :as cmd.profile]
-   [app.setup :as-alias setup]
-   [app.tokens :as tokens]
-   [app.worker :as-alias wrk]
-   [integrant.core :as ig]
-   [yetti.response :as-alias yres]))
-
-;; ---- ROUTES
-
-(declare ^:private authenticate)
-(declare ^:private get-customer)
-(declare ^:private update-customer)
-
-(defmethod ig/assert-key ::routes
-  [_ params]
-  (assert (db/pool? (::db/pool params)) "expect valid database pool"))
-
-(def ^:private auth
-  {:name ::auth
-   :compile
-   (fn [_ _]
-     (fn [handler shared-key]
-       (if shared-key
-         (fn [request]
-           (let [token (get-token request)]
-             (if (= token shared-key)
-               (handler request)
-               {::yres/status 403})))
-         (fn [_ _]
-           {::yres/status 403}))))})
-
-(def ^:private default-system
-  {:name ::default-system
-   :compile
-   (fn [_ _]
-     (fn [handler cfg]
-       (fn [request]
-         (handler cfg request))))})
-
-(def ^:private transaction
-  {:name ::transaction
-   :compile
-   (fn [data _]
-     (when (:transaction data)
-       (fn [handler]
-         (fn [cfg request]
-           (db/tx-run! cfg handler request)))))})
-
-(defmethod ig/init-key ::routes
-  [_ cfg]
-  ["" {:middleware [[auth (cf/get :management-api-shared-key)]
-                    [default-system cfg]
-                    [transaction]]}
-   ["/authenticate"
-    {:handler authenticate
-     :allowed-methods #{:post}}]
-
-   ["/get-customer"
-    {:handler get-customer
-     :transaction true
-     :allowed-methods #{:post}}]
-
-   ["/update-customer"
-    {:handler update-customer
-     :allowed-methods #{:post}
-     :transaction true}]])
-
-;; ---- HELPERS
-
-(defn- coercer
-  [schema & {:as opts}]
-  (let [decode-fn (sm/decoder schema sm/json-transformer)
-        check-fn  (sm/check-fn schema opts)]
-    (fn [data]
-      (-> data decode-fn check-fn))))
-
-;; ---- API: AUTHENTICATE
-
-(defn- authenticate
-  [cfg request]
-  (let [token  (-> request :params :token)
-        result (tokens/verify cfg {:token token :iss "authentication"})]
-    {::yres/status 200
-     ::yres/body result}))
-
-;; ---- API: GET-CUSTOMER
-
-(def ^:private schema:get-customer
-  [:map [:id ::sm/uuid]])
-
-(def ^:private coerce-get-customer-params
-  (coercer schema:get-customer
-           :type :validation
-           :hint "invalid data provided for `get-customer` rpc call"))
-
-(def ^:private sql:get-customer-slots
-  "WITH teams AS (
-    SELECT tpr.team_id AS id,
-           tpr.profile_id AS profile_id
-      FROM team_profile_rel AS tpr
-     WHERE tpr.is_owner IS true
-       AND tpr.profile_id = ?
-  ), teams_with_slots AS (
-    SELECT tpr.team_id AS id,
-           count(*) AS total
-      FROM team_profile_rel AS tpr
-     WHERE tpr.team_id IN (SELECT id FROM teams)
-       AND tpr.can_edit IS true
-     GROUP BY 1
-     ORDER BY 2
-  )
-  SELECT max(total) AS total FROM teams_with_slots;")
-
-(defn- get-customer-slots
-  [cfg profile-id]
-  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
-    (:total result)))
-
-(defn- get-customer
-  [cfg request]
-  (let [profile-id (-> request :params coerce-get-customer-params :id)
-        profile    (cmd.profile/get-profile cfg profile-id)
-        result     {:id (get profile :id)
-                    :name (get profile :fullname)
-                    :email (get profile :email)
-                    :num-editors (get-customer-slots cfg profile-id)
-                    :subscription (-> profile :props :subscription)}]
-    {::yres/status 200
-     ::yres/body result}))
-
-
-;; ---- API: UPDATE-CUSTOMER
-
-(def ^:private schema:timestamp
-  (sm/type-schema
-   {:type ::timestamp
-    :pred ct/inst?
-    :type-properties
-    {:title "inst"
-     :description "The same as :app.common.time/inst but encodes to epoch"
-     :error/message "should be an instant"
-     :gen/gen (->> (sg/small-int)
-                   (sg/fmap (fn [v] (ct/inst v))))
-     :decode/string ct/inst
-     :encode/string inst-ms
-     :decode/json ct/inst
-     :encode/json inst-ms}}))
-
-(def ^:private schema:subscription
-  [:map {:title "Subscription"}
-   [:id ::sm/text]
-   [:customer-id ::sm/text]
-   [:type [:enum
-           "unlimited"
-           "professional"
-           "enterprise"]]
-   [:status [:enum
-             "active"
-             "canceled"
-             "incomplete"
-             "incomplete_expired"
-             "past_due"
-             "paused"
-             "trialing"
-             "unpaid"]]
-
-   [:billing-period [:enum
-                     "month"
-                     "day"
-                     "week"
-                     "year"]]
-   [:quantity :int]
-   [:description [:maybe ::sm/text]]
-   [:created-at schema:timestamp]
-   [:start-date [:maybe schema:timestamp]]
-   [:ended-at [:maybe schema:timestamp]]
-   [:trial-end [:maybe schema:timestamp]]
-   [:trial-start [:maybe schema:timestamp]]
-   [:cancel-at [:maybe schema:timestamp]]
-   [:canceled-at [:maybe schema:timestamp]]
-   [:current-period-end [:maybe schema:timestamp]]
-   [:current-period-start [:maybe schema:timestamp]]
-   [:cancel-at-period-end :boolean]
-
-   [:cancellation-details
-    [:map {:title "CancellationDetails"}
-     [:comment [:maybe ::sm/text]]
-     [:reason [:maybe ::sm/text]]
-     [:feedback [:maybe
-                 [:enum
-                  "customer_service"
-                  "low_quality"
-                  "missing_feature"
-                  "other"
-                  "switched_service"
-                  "too_complex"
-                  "too_expensive"
-                  "unused"]]]]]])
-
-(def ^:private schema:update-customer
-  [:map
-   [:id ::sm/uuid]
-   [:subscription [:maybe schema:subscription]]])
-
-(def ^:private coerce-update-customer-params
-  (coercer schema:update-customer
-           :type :validation
-           :hint "invalid data provided for `update-customer` rpc call"))
-
-(defn- update-customer
-  [cfg request]
-  (let [{:keys [id subscription]}
-        (-> request :params coerce-update-customer-params)
-
-        {:keys [props] :as profile}
-        (cmd.profile/get-profile cfg id ::db/for-update true)
-
-        props
-        (assoc props :subscription subscription)]
-
-    (l/dbg :hint "update customer"
-           :profile-id (str id)
-           :subscription-type (get subscription :type)
-           :subscription-status (get subscription :status)
-           :subscription-quantity (get subscription :quantity))
-
-    (db/update! cfg :profile
-                {:props (db/tjson props)}
-                {:id id}
-                {::db/return-keys false})
-
-    {::yres/status 201
-     ::yres/body nil}))

backend/src/app/http/security.clj

@@ -1,55 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.http.security
-  "Additional security layer middlewares"
-  (:require
-   [app.config :as cf]
-   [yetti.request :as yreq]
-   [yetti.response :as yres]))
-
-(def ^:private safe-methods
-  #{:get :head :options})
-
-(defn- wrap-sec-fetch-metadata
-  "Sec-Fetch metadata security layer middleware"
-  [handler]
-  (fn [request]
-    (let [site (yreq/get-header request "sec-fetch-site")]
-      (cond
-        (= site "same-origin")
-        (handler request)
-
-        (or (= site "same-site")
-            (= site "cross-site"))
-        (if (contains? safe-methods (yreq/method request))
-          (handler request)
-          {::yres/status 403})
-
-        :else
-        (handler request)))))
-
-(def sec-fetch-metadata
-  {:name ::sec-fetch-metadata
-   :compile (fn [_ _]
-              (when (contains? cf/flags :sec-fetch-metadata-middleware)
-                wrap-sec-fetch-metadata))})
-
-(defn- wrap-client-header-check
-  "Check for a penpot custom header to be present as additional CSRF
-  protection"
-  [handler]
-  (fn [request]
-    (let [client (yreq/get-header request "x-client")]
-      (if (some? client)
-        (handler request)
-        {::yres/status 403}))))
-
-(def client-header-check
-  {:name ::client-header-check
-   :compile (fn [_ _]
-              (when (contains? cf/flags :client-header-check-middleware)
-                wrap-client-header-check))})

backend/src/app/http/session.clj

@@ -10,7 +10,7 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
@@ -18,6 +18,7 @@
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.util.time :as dt]
    [cuerdas.core :as str]
    [integrant.core :as ig]
    [yetti.request :as yreq]))
@@ -34,10 +35,10 @@
 (def default-auth-data-cookie-name "auth-data")
 
 ;; Default value for cookie max-age
-(def default-cookie-max-age (ct/duration {:days 7}))
+(def default-cookie-max-age (dt/duration {:days 7}))
 
 ;; Default age for automatic session renewal
-(def default-renewal-max-age (ct/duration {:hours 6}))
+(def default-renewal-max-age (dt/duration {:hours 6}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PROTOCOLS
@@ -65,13 +66,13 @@
   [:map {:title "session-params"}
    [:user-agent ::sm/text]
    [:profile-id ::sm/uuid]
-   [:created-at ::ct/inst]])
+   [:created-at ::sm/inst]])
 
 (def ^:private valid-params?
   (sm/validator schema:params))
 
 (defn- prepare-session-params
-  [params key]
+  [key params]
   (assert (string? key) "expected key to be a string")
   (assert (not (str/blank? key)) "expected key to be not empty")
   (assert (valid-params? params) "expected valid params")
@@ -89,14 +90,12 @@
       (db/exec-one! pool (sql/select :http-session {:id token})))
 
     (write! [_ key params]
-      (let [params (-> params
-                       (assoc :created-at (ct/now))
-                       (prepare-session-params key))]
+      (let [params (prepare-session-params key params)]
         (db/insert! pool :http-session params)
         params))
 
     (update! [_ params]
-      (let [updated-at (ct/now)]
+      (let [updated-at (dt/now)]
         (db/update! pool :http-session
                     {:updated-at updated-at}
                     {:id (:id params)})
@@ -114,14 +113,12 @@
         (get @cache token))
 
       (write! [_ key params]
-        (let [params (-> params
-                         (assoc :created-at (ct/now))
-                         (prepare-session-params key))]
+        (let [params (prepare-session-params key params)]
           (swap! cache assoc key params)
           params))
 
       (update! [_ params]
-        (let [updated-at (ct/now)]
+        (let [updated-at (dt/now)]
           (swap! cache update (:id params) assoc :updated-at updated-at)
           (assoc params :updated-at updated-at)))
 
@@ -147,23 +144,27 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (declare ^:private assign-auth-token-cookie)
+(declare ^:private assign-auth-data-cookie)
 (declare ^:private clear-auth-token-cookie)
+(declare ^:private clear-auth-data-cookie)
 (declare ^:private gen-token)
 
 (defn create-fn
-  [{:keys [::manager] :as cfg} profile-id]
+  [{:keys [::manager ::setup/props]} profile-id]
   (assert (manager? manager) "expected valid session manager")
   (assert (uuid? profile-id) "expected valid uuid for profile-id")
 
   (fn [request response]
     (let [uagent  (yreq/get-header request "user-agent")
           params  {:profile-id profile-id
-                   :user-agent uagent}
-          token   (gen-token cfg params)
+                   :user-agent uagent
+                   :created-at (dt/now)}
+          token   (gen-token props params)
           session (write! manager token params)]
-      (l/trc :hint "create" :profile-id (str profile-id))
+      (l/trace :hint "create" :profile-id (str profile-id))
       (-> response
-          (assign-auth-token-cookie session)))))
+          (assign-auth-token-cookie session)
+          (assign-auth-data-cookie session)))))
 
 (defn delete-fn
   [{:keys [::manager]}]
@@ -171,22 +172,23 @@
   (fn [request response]
     (let [cname   (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
           cookie  (yreq/get-cookie request cname)]
-      (l/trc :hint "delete" :profile-id (:profile-id request))
+      (l/trace :hint "delete" :profile-id (:profile-id request))
       (some->> (:value cookie) (delete! manager))
       (-> response
           (assoc :status 204)
           (assoc :body nil)
-          (clear-auth-token-cookie)))))
+          (clear-auth-token-cookie)
+          (clear-auth-data-cookie)))))
 
 (defn- gen-token
-  [cfg {:keys [profile-id created-at]}]
-  (tokens/generate cfg {:iss "authentication"
-                        :iat created-at
-                        :uid profile-id}))
+  [props {:keys [profile-id created-at]}]
+  (tokens/generate props {:iss "authentication"
+                          :iat created-at
+                          :uid profile-id}))
 (defn- decode-token
-  [cfg token]
+  [props token]
   (when token
-    (tokens/verify cfg {:token token :iss "authentication"})))
+    (tokens/verify props {:token token :iss "authentication"})))
 
 (defn- get-token
   [request]
@@ -201,23 +203,23 @@
 
 (defn- renew-session?
   [{:keys [updated-at] :as session}]
-  (and (ct/inst? updated-at)
-       (let [elapsed (ct/diff updated-at (ct/now))]
+  (and (dt/instant? updated-at)
+       (let [elapsed (dt/diff updated-at (dt/now))]
          (neg? (compare default-renewal-max-age elapsed)))))
 
 (defn- wrap-soft-auth
-  [handler {:keys [::manager] :as cfg}]
+  [handler {:keys [::manager ::setup/props]}]
   (assert (manager? manager) "expected valid session manager")
   (letfn [(handle-request [request]
             (try
               (let [token  (get-token request)
-                    claims (decode-token cfg token)]
+                    claims (decode-token props token)]
                 (cond-> request
                   (map? claims)
                   (-> (assoc ::token-claims claims)
                       (assoc ::token token))))
               (catch Throwable cause
-                (l/trc :hint "exception on decoding malformed token" :cause cause)
+                (l/trace :hint "exception on decoding malformed token" :cause cause)
                 request)))]
 
     (fn [request]
@@ -237,7 +239,8 @@
       (if (renew-session? session)
         (let [session (update! manager session)]
           (-> response
-              (assign-auth-token-cookie session)))
+              (assign-auth-token-cookie session)
+              (assign-auth-data-cookie session)))
         response))))
 
 (def soft-auth
@@ -253,14 +256,14 @@
 (defn- assign-auth-token-cookie
   [response {token :id updated-at :updated-at}]
   (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        created-at updated-at
-        renewal    (ct/plus created-at default-renewal-max-age)
-        expires    (ct/plus created-at max-age)
+        created-at (or updated-at (dt/now))
+        renewal    (dt/plus created-at default-renewal-max-age)
+        expires    (dt/plus created-at max-age)
         secure?    (contains? cf/flags :secure-session-cookies)
         strict?    (contains? cf/flags :strict-session-cookies)
         cors?      (contains? cf/flags :cors)
         name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
-        comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
+        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
         cookie     {:path "/"
                     :http-only true
                     :expires expires
@@ -270,19 +273,57 @@
                     :secure secure?}]
     (update response :cookies assoc name cookie)))
 
+(defn- assign-auth-data-cookie
+  [response {profile-id :profile-id updated-at :updated-at}]
+  (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
+        domain     (cf/get :auth-data-cookie-domain)
+        cname      default-auth-data-cookie-name
+
+        created-at (or updated-at (dt/now))
+        renewal    (dt/plus created-at default-renewal-max-age)
+        expires    (dt/plus created-at max-age)
+
+        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
+        secure?    (contains? cf/flags :secure-session-cookies)
+        strict?    (contains? cf/flags :strict-session-cookies)
+        cors?      (contains? cf/flags :cors)
+
+        cookie     {:domain domain
+                    :expires expires
+                    :path "/"
+                    :comment comment
+                    :value (u/map->query-string {:profile-id profile-id})
+                    :same-site (if cors? :none (if strict? :strict :lax))
+                    :secure secure?}]
+
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc cname cookie))))
+
 (defn- clear-auth-token-cookie
   [response]
   (let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
     (update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
 
+(defn- clear-auth-data-cookie
+  [response]
+  (let [cname  default-auth-data-cookie-name
+        domain (cf/get :auth-data-cookie-domain)]
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc cname {:domain domain :path "/" :value "" :max-age 0}))))
+
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASK: SESSION GC
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; FIXME: MOVE
+
 (defmethod ig/assert-key ::tasks/gc
   [_ params]
   (assert (db/pool? (::db/pool params)) "expected valid database pool")
-  (assert (ct/duration? (::tasks/max-age params))))
+  (assert (dt/duration? (::tasks/max-age params))))
 
 (defmethod ig/expand-key ::tasks/gc
   [k v]
@@ -291,23 +332,22 @@
 
 (def ^:private
   sql:delete-expired
-  "DELETE FROM http_session
-    WHERE updated_at < ?::timestamptz
+  "delete from http_session
+    where updated_at < now() - ?::interval
        or (updated_at is null and
-           created_at < ?::timestamptz)")
+           created_at < now() - ?::interval)")
 
 (defn- collect-expired-tasks
   [{:keys [::db/conn ::tasks/max-age]}]
-  (let [threshold (ct/minus (ct/now) max-age)
-        result    (-> (db/exec-one! conn [sql:delete-expired threshold threshold])
-                      (db/get-update-count))]
-    (l/dbg :task "gc"
-           :hint "clean http sessions"
-           :deleted result)
+  (let [interval (db/interval max-age)
+        result   (db/exec-one! conn [sql:delete-expired interval interval])
+        result   (:next.jdbc/update-count result)]
+    (l/debug :task "gc"
+             :hint "clean http sessions"
+             :deleted result)
     result))
 
 (defmethod ig/init-key ::tasks/gc
   [_ {:keys [::tasks/max-age] :as cfg}]
-  (l/dbg :hint "initializing session gc task" :max-age max-age)
-  (fn [_]
-    (db/tx-run! cfg collect-expired-tasks)))
+  (l/debug :hint "initializing session gc task" :max-age max-age)
+  (fn [_] (db/tx-run! cfg collect-expired-tasks)))

backend/src/app/http/sse.clj

@@ -9,6 +9,7 @@
   (:refer-clojure :exclude [tap])
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.transit :as t]
    [app.http.errors :as errors]
@@ -33,7 +34,7 @@
                  (println "event:" (d/name name))
                  (println "data:" (t/encode-str data {:type :json-verbose}))
                  (println))]
-      (.getBytes ^String data "UTF-8"))
+      (.getBytes data "UTF-8"))
     (catch Throwable cause
       (l/err :hint "unexpected error on encoding value on sse stream"
              :cause cause)
@@ -44,8 +45,7 @@
 (def default-headers
   {"Content-Type" "text/event-stream;charset=UTF-8"
    "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
-   "Pragma" "no-cache"
-   "X-Accel-Buffering" "no"})
+   "Pragma" "no-cache"})
 
 (defn response
   [handler & {:keys [buf] :or {buf 32} :as opts}]
@@ -54,20 +54,18 @@
      ::yres/status 200
      ::yres/body (yres/stream-body
                   (fn [_ output]
-                    (let [channel  (sp/chan :buf buf :xf (keep encode))
-                          listener (events/spawn-listener
-                                    channel
-                                    (partial write! output)
-                                    (partial pu/close! output))]
-                      (try
-                        (binding [events/*channel* channel]
+                    (binding [events/*channel* (sp/chan :buf buf :xf (keep encode))]
+                      (let [listener (events/start-listener
+                                      (partial write! output)
+                                      (partial pu/close! output))]
+                        (try
                           (let [result (handler)]
-                            (events/tap :end result)))
-
-                        (catch Throwable cause
-                          (let [result (errors/handle' cause request)]
-                            (events/tap channel :error result)))
-
-                        (finally
-                          (sp/close! channel)
-                          (px/await! listener))))))}))
+                            (events/tap :end result))
+                          (catch Throwable cause
+                            (events/tap :error (errors/handle' cause request))
+                            (when-not (ex/instance? java.io.EOFException cause)
+                              (binding [l/*context* (errors/request->context request)]
+                                (l/err :hint "unexpected error on processing sse response" :cause cause))))
+                          (finally
+                            (sp/close! events/*channel*)
+                            (px/await! listener)))))))}))

backend/src/app/http/websocket.clj

@@ -11,12 +11,12 @@
    [app.common.logging :as l]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.http.session :as session]
    [app.metrics :as mtx]
    [app.msgbus :as mbus]
+   [app.util.time :as dt]
    [app.util.websocket :as ws]
    [integrant.core :as ig]
    [promesa.exec.csp :as sp]
@@ -239,7 +239,7 @@
 
 (defn- on-connect
   [{:keys [::mtx/metrics]} {:keys [::ws/id] :as wsp}]
-  (let [created-at (ct/now)]
+  (let [created-at (dt/now)]
     (l/trace :fn "on-connect" :conn-id id)
     (swap! state assoc id wsp)
     (mtx/run! metrics
@@ -253,7 +253,7 @@
              (mtx/run! metrics :id :websocket-active-connections :dec 1)
              (mtx/run! metrics
                        :id :websocket-session-timing
-                       :val (/ (inst-ms (ct/diff created-at (ct/now))) 1000.0))))))
+                       :val (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0))))))
 
 (defn- on-rcv-message
   [{:keys [::mtx/metrics ::profile-id ::session-id]} message]

backend/src/app/loggers/audit.clj

@@ -11,7 +11,6 @@
    [app.common.data.macros :as dm]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -24,6 +23,7 @@
    [app.setup :as-alias setup]
    [app.util.inet :as inet]
    [app.util.services :as-alias sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]
    [cuerdas.core :as str]))
 
@@ -108,9 +108,9 @@
    [::ip-addr {:optional true} ::sm/text]
    [::props {:optional true} [:map-of :keyword :any]]
    [::context {:optional true} [:map-of :keyword :any]]
-   [::tracked-at {:optional true} ::ct/inst]
+   [::tracked-at {:optional true} ::sm/inst]
    [::webhooks/event? {:optional true} ::sm/boolean]
-   [::webhooks/batch-timeout {:optional true} ::ct/duration]
+   [::webhooks/batch-timeout {:optional true} ::dt/duration]
    [::webhooks/batch-key {:optional true}
     [:or ::sm/fn ::sm/text :keyword]]])
 
@@ -199,7 +199,7 @@
 (defn- handle-event!
   [cfg event]
   (let [params (event->params event)
-        tnow   (ct/now)]
+        tnow   (dt/now)]
 
     (when (contains? cf/flags :audit-log)
       ;; NOTE: this operation may cause primary key conflicts on inserts
@@ -273,7 +273,7 @@
     (let [event (-> (d/without-nils event)
                     (check-event))]
       (db/run! cfg (fn [cfg]
-                     (let [tnow   (ct/now)
+                     (let [tnow   (dt/now)
                            params (-> (event->params event)
                                       (assoc :created-at tnow)
                                       (update :tracked-at #(or % tnow)))]

backend/src/app/loggers/audit/archive_task.clj

@@ -16,6 +16,7 @@
    [app.http.client :as http]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.util.time :as dt]
    [integrant.core :as ig]
    [lambdaisland.uri :as u]
    [promesa.exec :as px]))
@@ -52,8 +53,9 @@
 
 (defn- send!
   [{:keys [::uri] :as cfg} events]
-  (let [token   (tokens/generate cfg
+  (let [token   (tokens/generate (::setup/props cfg)
                                  {:iss "authentication"
+                                  :iat (dt/now)
                                   :uid uuid/zero})
         body    (t/encode {:events events})
         headers {"content-type" "application/transit+json"

backend/src/app/loggers/database.clj

@@ -41,7 +41,7 @@
   (if (or (instance? java.util.concurrent.CompletionException cause)
           (instance? java.util.concurrent.ExecutionException cause))
     (-> record
-        (assoc ::trace (ex/format-throwable cause :data? true :explain? false :header? false :summary? false))
+        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
         (assoc ::l/cause (ex-cause cause))
         (record->report))
 
@@ -64,18 +64,18 @@
                          message))
                      @message)
         :trace   (or (::trace record)
-                     (some-> cause (ex/format-throwable :data? true :explain? false :header? false :summary? false)))}
+                     (some-> cause (ex/format-throwable :data? false :explain? false :header? false :summary? false)))}
 
        (when-let [params (or (:request/params context) (:params context))]
-         {:params (pp/pprint-str params :length 20 :level 20)})
+         {:params (pp/pprint-str params :length 30 :level 13)})
 
        (when-let [value (:value context)]
-         {:value (pp/pprint-str value :length 30 :level 13)})
+         {:value (pp/pprint-str value :length 30 :level 12)})
 
        (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-         {:data (pp/pprint-str data :length 30 :level 13)})
+         {:data (pp/pprint-str data :length 30 :level 12)})
 
-       (when-let [explain (ex/explain data :length 30 :level 13)]
+       (when-let [explain (ex/explain data :length 30 :level 12)]
          {:explain explain})))))
 
 (defn error-record?

backend/src/app/loggers/webhooks.clj

@@ -10,13 +10,13 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.logging :as l]
-   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uri :as uri]
    [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
    [app.loggers.audit :as audit]
+   [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.data.json :as json]
    [cuerdas.core :as str]
@@ -124,7 +124,7 @@
                               {:id (:id whook)})))
 
               (db/update! pool :webhook
-                          {:updated-at (ct/now)
+                          {:updated-at (dt/now)
                            :error-code nil
                            :error-count 0}
                           {:id (:id whook)})))
@@ -132,7 +132,7 @@
           (report-delivery! [whook req rsp err]
             (db/insert! pool :webhook-delivery
                         {:webhook-id (:id whook)
-                         :created-at (ct/now)
+                         :created-at (dt/now)
                          :error-code err
                          :req-data (db/tjson req)
                          :rsp-data (db/tjson rsp)}))]
@@ -155,7 +155,7 @@
         (let [req {:uri (:uri whook)
                    :headers {"content-type" (:mtype whook)
                              "user-agent" (str/ffmt "penpot/%" (:main cf/version))}
-                   :timeout (ct/duration "4s")
+                   :timeout (dt/duration "4s")
                    :method :post
                    :body body}]
           (try

backend/src/app/main.clj

@@ -11,7 +11,6 @@
    [app.auth.oidc.providers :as-alias oidc.providers]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
-   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.email :as-alias email]
@@ -20,7 +19,6 @@
    [app.http.awsns :as http.awsns]
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
-   [app.http.management :as mgmt]
    [app.http.session :as-alias session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
@@ -40,9 +38,9 @@
    [app.storage.gc-touched :as-alias sto.gc-touched]
    [app.storage.s3 :as-alias sto.s3]
    [app.svgo :as-alias svgo]
-   [app.util.cron]
+   [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [app.worker.executor]
+   [cider.nrepl :refer [cider-nrepl-handler]]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -149,11 +147,23 @@
     ::mdef/labels []
     ::mdef/type :histogram}
 
-   :http-server-dispatch-timing
-   {::mdef/name "penpot_http_server_dispatch_timing"
-    ::mdef/help "Histogram of dispatch handler"
-    ::mdef/labels []
-    ::mdef/type :histogram}})
+   :executors-active-threads
+   {::mdef/name "penpot_executors_active_threads"
+    ::mdef/help "Current number of threads available in the executor service."
+    ::mdef/labels ["name"]
+    ::mdef/type :gauge}
+
+   :executors-completed-tasks
+   {::mdef/name "penpot_executors_completed_tasks_total"
+    ::mdef/help "Approximate number of completed tasks by the executor."
+    ::mdef/labels ["name"]
+    ::mdef/type :counter}
+
+   :executors-running-threads
+   {::mdef/name "penpot_executors_running_threads"
+    ::mdef/help "Current number of threads with state RUNNING."
+    ::mdef/labels ["name"]
+    ::mdef/type :gauge}})
 
 (def system-config
   {::db/pool
@@ -165,12 +175,14 @@
     ::db/max-size   (cf/get :database-max-pool-size 60)
     ::mtx/metrics   (ig/ref ::mtx/metrics)}
 
-   ;; Default netty IO pool (shared between several services)
-   ::wrk/netty-io-executor
-   {:threads (cf/get :netty-io-threads)}
+   ;; Default thread pool for IO operations
+   ::wrk/executor
+   {}
 
-   ::wrk/netty-executor
-   {:threads (cf/get :executor-threads)}
+   ::wrk/monitor
+   {::mtx/metrics  (ig/ref ::mtx/metrics)
+    ::wrk/executor (ig/ref ::wrk/executor)
+    ::wrk/name     "default"}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -181,27 +193,17 @@
    ::mtx/routes
    {::mtx/metrics (ig/ref ::mtx/metrics)}
 
-   ::rds/client
-   {::rds/uri
-    (cf/get :redis-uri)
-
-    ::wrk/netty-executor
-    (ig/ref ::wrk/netty-executor)
-
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
-
-   ::rds/pool
-   {::rds/client  (ig/ref ::rds/client)
-    ::mtx/metrics (ig/ref ::mtx/metrics)}
+   ::rds/redis
+   {::rds/uri      (cf/get :redis-uri)
+    ::mtx/metrics  (ig/ref ::mtx/metrics)
+    ::wrk/executor (ig/ref ::wrk/executor)}
 
    ::mbus/msgbus
-   {::wrk/executor (ig/ref ::wrk/netty-executor)
-    ::rds/client   (ig/ref ::rds/client)
-    ::mtx/metrics  (ig/ref ::mtx/metrics)}
+   {::wrk/executor  (ig/ref ::wrk/executor)
+    ::rds/redis     (ig/ref ::rds/redis)}
 
    :app.storage.tmp/cleaner
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    ::sto.gc-deleted/handler
    {::db/pool      (ig/ref ::db/pool)
@@ -229,10 +231,8 @@
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
     ::http/io-threads              (cf/get :http-server-io-threads)
-    ::http/max-worker-threads      (cf/get :http-server-max-worker-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
-    ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)
-    ::mtx/metrics                  (ig/ref ::mtx/metrics)}
+    ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
 
    ::ldap/provider
    {:host           (cf/get :ldap-host)
@@ -272,10 +272,6 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   ::mgmt/routes
-   {::db/pool            (ig/ref ::db/pool)
-    ::setup/props        (ig/ref ::setup/props)}
-
    :app.http/router
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
@@ -284,7 +280,6 @@
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
-    ::mgmt/routes        (ig/ref ::mgmt/routes)
     ::http.debug/routes  (ig/ref ::http.debug/routes)
     ::http.assets/routes (ig/ref ::http.assets/routes)
     ::http.ws/routes     (ig/ref ::http.ws/routes)
@@ -304,30 +299,29 @@
 
    :app.http.assets/routes
    {::http.assets/path  (cf/get :assets-path)
-    ::http.assets/cache-max-age (ct/duration {:hours 24})
-    ::http.assets/cache-max-agesignature-max-age (ct/duration {:hours 24 :minutes 5})
+    ::http.assets/cache-max-age (dt/duration {:hours 24})
+    ::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
     ::sto/storage  (ig/ref ::sto/storage)}
 
    ::rpc/climit
    {::mtx/metrics        (ig/ref ::mtx/metrics)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::wrk/executor       (ig/ref ::wrk/executor)
     ::climit/config      (cf/get :rpc-climit-config)
     ::climit/enabled     (contains? cf/flags :rpc-climit)}
 
    :app.rpc/rlimit
-   {::wrk/executor (ig/ref ::wrk/netty-executor)}
+   {::wrk/executor (ig/ref ::wrk/executor)}
 
    :app.rpc/methods
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
-    ::rds/pool           (ig/ref ::rds/pool)
-    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::wrk/executor       (ig/ref ::wrk/executor)
     ::session/manager    (ig/ref ::session/manager)
     ::ldap/provider      (ig/ref ::ldap/provider)
     ::sto/storage        (ig/ref ::sto/storage)
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
-    ::rds/client         (ig/ref ::rds/client)
+    ::rds/redis          (ig/ref ::rds/redis)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
@@ -340,7 +334,7 @@
    :app.rpc.doc/routes
    {:app.rpc/methods (ig/ref :app.rpc/methods)}
 
-   ::rpc/routes
+   :app.rpc/routes
    {::rpc/methods     (ig/ref :app.rpc/methods)
     ::db/pool         (ig/ref ::db/pool)
     ::session/manager (ig/ref ::session/manager)
@@ -432,9 +426,6 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
-   ::setup/clock
-   {}
-
    :app.loggers.audit.archive-task/handler
    {::setup/props        (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
@@ -478,50 +469,49 @@
                             (cf/get :objects-storage-s3-bucket))
     ::sto.s3/io-threads (or (cf/get :storage-assets-s3-io-threads)
                             (cf/get :objects-storage-s3-io-threads))
-
-    ::wrk/netty-io-executor
-    (ig/ref ::wrk/netty-io-executor)}
+    ::wrk/executor      (ig/ref ::wrk/executor)}
 
    :app.storage.fs/backend
    {::sto.fs/directory (or (cf/get :storage-assets-fs-directory)
                            (cf/get :objects-storage-fs-directory))}})
 
+
 (def worker-config
   {::wrk/cron
    {::wrk/registry            (ig/ref ::wrk/registry)
     ::db/pool                 (ig/ref ::db/pool)
     ::wrk/entries
-    [{:cron #penpot/cron "0 0 0 * * ?" ;; daily
+    [{:cron #app/cron "0 0 0 * * ?" ;; daily
       :task :session-gc}
 
-     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
+     {:cron #app/cron "0 0 0 * * ?" ;; daily
       :task :objects-gc}
 
-     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
+     {:cron #app/cron "0 0 0 * * ?" ;; daily
       :task :storage-gc-deleted}
 
-     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
+     {:cron #app/cron "0 0 0 * * ?" ;; daily
       :task :storage-gc-touched}
 
-     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
+     {:cron #app/cron "0 0 0 * * ?" ;; daily
       :task :tasks-gc}
 
-     {:cron #penpot/cron "0 0 2 * * ?" ;; daily
+     {:cron #app/cron "0 0 2 * * ?" ;; daily
       :task :file-gc-scheduler}
 
-     {:cron #penpot/cron "0 30 */3,23 * * ?"
+     {:cron #app/cron "0 30 */3,23 * * ?"
       :task :telemetry}
 
      (when (contains? cf/flags :audit-log-archive)
-       {:cron #penpot/cron "0 */5 * * * ?" ;; every 5m
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
         :task :audit-log-archive})
 
      (when (contains? cf/flags :audit-log-gc)
-       {:cron #penpot/cron "30 */5 * * * ?" ;; every 5m
+       {:cron #app/cron "30 */5 * * * ?" ;; every 5m
         :task :audit-log-gc})]}
 
    ::wrk/dispatcher
-   {::rds/client  (ig/ref ::rds/client)
+   {::rds/redis   (ig/ref ::rds/redis)
     ::mtx/metrics (ig/ref ::mtx/metrics)
     ::db/pool     (ig/ref ::db/pool)
     ::wrk/tenant  (cf/get :tenant)}
@@ -530,7 +520,7 @@
    {::wrk/parallelism (cf/get ::worker-default-parallelism 1)
     ::wrk/queue       :default
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/client      (ig/ref ::rds/client)
+    ::rds/redis       (ig/ref ::rds/redis)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}
@@ -539,7 +529,7 @@
    {::wrk/parallelism (cf/get ::worker-webhook-parallelism 1)
     ::wrk/queue       :webhooks
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/client      (ig/ref ::rds/client)
+    ::rds/redis       (ig/ref ::rds/redis)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}})
@@ -615,7 +605,7 @@
     (let [p (promise)]
       (when (contains? cf/flags :nrepl-server)
         (l/inf :hint "start nrepl server" :port 6064)
-        (nrepl/start-server :bind "0.0.0.0" :port 6064))
+        (nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
 
       (start)
       (deref p))

backend/src/app/media.clj

@@ -8,54 +8,56 @@
   "Media & Font postprocessing."
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
-   [app.common.logging :as l]
    [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.schema.openapi :as-alias oapi]
-   [app.common.time :as ct]
+   [app.common.spec :as us]
+   [app.common.svg :as csvg]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
+   [app.util.time :as dt]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.shell :as sh]
-   [clojure.string]
-   [clojure.xml :as xml]
+   [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
-   clojure.lang.XMLHandler
-   java.io.InputStream
-   javax.xml.XMLConstants
-   javax.xml.parsers.SAXParserFactory
-   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(def schema:upload
-  [:map {:title "Upload"}
-   [:filename :string]
-   [:size ::sm/int]
-   [:path ::fs/path]
-   [:mtype {:optional true} :string]
-   [:headers {:optional true}
-    [:map-of :string :string]]])
+(s/def ::path fs/path?)
+(s/def ::filename string?)
+(s/def ::size integer?)
+(s/def ::headers (s/map-of string? string?))
+(s/def ::mtype string?)
 
-(def ^:private schema:input
-  [:map {:title "Input"}
-   [:path ::fs/path]
-   [:mtype {:optional true} ::sm/text]])
+(s/def ::upload
+  (s/keys :req-un [::filename ::size ::path]
+          :opt-un [::mtype ::headers]))
 
-(def ^:private check-input
-  (sm/check-fn schema:input))
+;; A subset of fields from the ::upload spec
+(s/def ::input
+  (s/keys :req-un [::path]
+          :opt-un [::mtype]))
+
+(sm/register!
+ ^{::sm/type ::upload}
+ [:map {:title "Upload"}
+  [:filename :string]
+  [:size ::sm/int]
+  [:path ::fs/path]
+  [:mtype {:optional true} :string]
+  [:headers {:optional true}
+   [:map-of :string :string]]])
 
 (defn validate-media-type!
-  ([upload] (validate-media-type! upload cm/image-types))
+  ([upload] (validate-media-type! upload cm/valid-image-types))
   ([upload allowed]
    (when-not (contains? allowed (:mtype upload))
      (ex/raise :type :validation
@@ -95,44 +97,17 @@
     (catch Throwable e
       (process-error e))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; SVG PARSING
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- secure-parser-factory
-  [^InputStream input ^XMLHandler handler]
-  (.. (doto (SAXParserFactory/newInstance)
-        (.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
-        (.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
-      (newSAXParser)
-      (parse input handler)))
-
-(defn- strip-doctype
-  [data]
-  (cond-> data
-    (str/includes? data "<!DOCTYPE")
-    (str/replace #"<\!DOCTYPE[^>]*>" "")))
-
-(defn- parse-svg
-  [text]
-  (let [text (strip-doctype text)]
-    (dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
-      (xml/parse istream secure-parser-factory))))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; IMAGE THUMBNAILS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(def ^:private schema:thumbnail-params
-  [:map {:title "ThumbnailParams"}
-   [:input schema:input]
-   [:format [:enum :jpeg :webp :png]]
-   [:quality [:int {:min 1 :max 100}]]
-   [:width :int]
-   [:height :int]])
+(s/def ::width integer?)
+(s/def ::height integer?)
+(s/def ::format #{:jpeg :webp :png})
+(s/def ::quality #(< 0 % 101))
 
-(def ^:private check-thumbnail-params
-  (sm/check-fn schema:thumbnail-params))
+(s/def ::thumbnail-params
+  (s/keys :req-un [::input ::format ::width ::height]))
 
 ;; Related info on how thumbnails generation
 ;;  http://www.imagemagick.org/Usage/thumbnails/
@@ -154,38 +129,30 @@
            :data   tmp)))
 
 (defmethod process :generic-thumbnail
-  [params]
-  (let [{:keys [quality width height] :as params}
-        (check-thumbnail-params params)
-
-        operation
-        (doto (IMOperation.)
-          (.addImage)
-          (.autoOrient)
-          (.strip)
-          (.thumbnail ^Integer (int width) ^Integer (int height) ">")
-          (.quality (double quality))
-          (.addImage))]
-
-    (generic-process (assoc params :operation operation))))
+  [{:keys [quality width height] :as params}]
+  (us/assert ::thumbnail-params params)
+  (let [op (doto (IMOperation.)
+             (.addImage)
+             (.autoOrient)
+             (.strip)
+             (.thumbnail ^Integer (int width) ^Integer (int height) ">")
+             (.quality (double quality))
+             (.addImage))]
+    (generic-process (assoc params :operation op))))
 
 (defmethod process :profile-thumbnail
-  [params]
-  (let [{:keys [quality width height] :as params}
-        (check-thumbnail-params params)
-
-        operation
-        (doto (IMOperation.)
-          (.addImage)
-          (.autoOrient)
-          (.strip)
-          (.thumbnail ^Integer (int width) ^Integer (int height) "^")
-          (.gravity "center")
-          (.extent (int width) (int height))
-          (.quality (double quality))
-          (.addImage))]
-
-    (generic-process (assoc params :operation operation))))
+  [{:keys [quality width height] :as params}]
+  (us/assert ::thumbnail-params params)
+  (let [op (doto (IMOperation.)
+             (.addImage)
+             (.autoOrient)
+             (.strip)
+             (.thumbnail ^Integer (int width) ^Integer (int height) "^")
+             (.gravity "center")
+             (.extent (int width) (int height))
+             (.quality (double quality))
+             (.addImage))]
+    (generic-process (assoc params :operation op))))
 
 (defn get-basic-info-from-svg
   [{:keys [tag attrs] :as data}]
@@ -215,33 +182,17 @@
                  {:width (int width)
                   :height (int height)})))]))
 
-(defn- get-dimensions-with-orientation [^String path]
-  ;; Image magick doesn't give info about exif rotation so we use the identify command
-  ;; If we are processing an animated gif we use the first frame with -scene 0
-  (let [dim-result (sh/sh "identify" "-format" "%w %h\n" path)
-        orient-result (sh/sh "identify" "-format" "%[EXIF:Orientation]\n" path)]
-    (if (and (= 0 (:exit dim-result))
-             (= 0 (:exit orient-result)))
-      (let [[w h] (-> (:out dim-result)
-                      str/trim
-                      (clojure.string/split #"\s+")
-                      (->> (mapv #(Integer/parseInt %))))
-            orientation (-> orient-result :out str/trim)]
-        (case orientation
-          ("6" "8") {:width h :height w} ; Rotated 90 or 270 degrees
-          {:width w :height h}))         ; Normal or unknown orientation
-      nil)))
-
 (defmethod process :info
   [{:keys [input] :as params}]
-  (let [{:keys [path mtype] :as input} (check-input input)]
+  (us/assert ::input input)
+  (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp parse-svg get-basic-info-from-svg)]
+      (let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (merge input info {:ts (ct/now)}))
+        (merge input info {:ts (dt/now)}))
 
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
@@ -251,17 +202,13 @@
                     :code :media-type-mismatch
                     :hint (str "Seems like you are uploading a file whose content does not match the extension."
                                "Expected: " mtype ". Got: " mtype')))
-        (let [{:keys [width height]}
-              (or (get-dimensions-with-orientation (str path))
-                  (do
-                    (l/warn "Failed to read image dimensions with orientation; falling back to im4java"
-                            {:path path})
-                    {:width  (.getPageWidth instance)
-                     :height (.getPageHeight instance)}))]
-          (assoc input
-                 :width  width
-                 :height height
-                 :ts (ct/now)))))))
+        ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
+        ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
+        ;; any frame.
+        (assoc input
+               :width  (.getPageWidth instance)
+               :height (.getPageHeight instance)
+               :ts (dt/now))))))
 
 (defmethod process-error org.im4java.core.InfoException
   [error]

backend/src/app/migrations.clj

@@ -438,19 +438,7 @@
     :fn (mg/resource "app/migrations/sql/0138-mod-file-data-fragment-table.sql")}
 
    {:name "0139-mod-file-change-table.sql"
-    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}
-
-   {:name "0140-mod-file-change-table.sql"
-    :fn (mg/resource "app/migrations/sql/0140-mod-file-change-table.sql")}
-
-   {:name "0140-add-locked-by-column-to-file-change-table"
-    :fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}
-
-   {:name "0141-add-idx-to-file-library-rel"
-    :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
-
-   {:name "0141-add-file-data-table.sql"
-    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/media_refs.clj

@@ -10,8 +10,8 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint]
+   [app.srepl.fixes.media-refs :refer [process-file]]
    [app.srepl.main :as srepl]
-   [app.srepl.procs.media-refs]
    [clojure.edn :as edn]))
 
 (def ^:private required-services
@@ -20,10 +20,7 @@
    :app.storage/storage
    :app.metrics/metrics
    :app.db/pool
-   :app.worker/netty-io-executor])
-
-(def default-options
-  {:rollback? false})
+   :app.worker/executor])
 
 (defn -main
   [& [options]]
@@ -31,20 +28,22 @@
     (let [config-var (requiring-resolve 'app.main/system-config)
           start-var  (requiring-resolve 'app.main/start-custom)
           stop-var   (requiring-resolve 'app.main/stop)
-
-          config     (select-keys @config-var required-services)
-          options    (if (string? options)
-                       (ex/ignoring (edn/read-string options))
-                       {})
-          options    (-> (merge default-options options)
-                         (assoc :proc-fn #'app.srepl.procs.media-refs/fix-media-refs))]
+          config     (select-keys @config-var required-services)]
 
       (start-var config)
-      (l/inf :hint "executing media-refs migration" :options options)
-      (srepl/process! options)
+
+      (let [options (if (string? options)
+                      (ex/ignoring (edn/read-string options))
+                      {})]
+
+        (l/inf :hint "executing media-refs migration" :options options)
+        (srepl/process-files! process-file options))
+
       (stop-var)
       (System/exit 0))
     (catch Throwable cause
       (ex/print-throwable cause)
       (flush)
       (System/exit -1))))
+
+

backend/src/app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql

@@ -1,11 +0,0 @@
--- Add locked_by column to file_change table for version locking feature
--- This allows users to lock their own saved versions to prevent deletion by others
-
-ALTER TABLE file_change
-  ADD COLUMN locked_by uuid NULL REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;
-
--- Create index for locked versions queries
-CREATE INDEX file_change__locked_by__idx ON file_change (locked_by) WHERE locked_by IS NOT NULL;
-
--- Add comment for documentation
-COMMENT ON COLUMN file_change.locked_by IS 'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.';

backend/src/app/migrations/sql/0140-mod-file-change-table.sql

@@ -1,2 +0,0 @@
-ALTER TABLE file_change
- ADD COLUMN migrations text[];

backend/src/app/migrations/sql/0141-add-file-data-table.sql

@@ -1,38 +0,0 @@
-CREATE TABLE file_data (
-  file_id uuid NOT NULL REFERENCES file(id) DEFERRABLE,
-  id uuid NOT NULL,
-
-  created_at timestamptz NOT NULL DEFAULT now(),
-  modified_at timestamptz NOT NULL DEFAULT now(),
-  deleted_at timestamptz NULL,
-
-  type text NOT NULL,
-  backend text NULL,
-
-  metadata jsonb NULL,
-  data bytea NULL,
-
-  PRIMARY KEY (file_id, id)
-
-) PARTITION BY HASH (file_id);
-
-CREATE TABLE file_data_00 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 0);
-CREATE TABLE file_data_01 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 1);
-CREATE TABLE file_data_02 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 2);
-CREATE TABLE file_data_03 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 3);
-CREATE TABLE file_data_04 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 4);
-CREATE TABLE file_data_05 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 5);
-CREATE TABLE file_data_06 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 6);
-CREATE TABLE file_data_07 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 7);
-CREATE TABLE file_data_08 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 8);
-CREATE TABLE file_data_09 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 9);
-CREATE TABLE file_data_10 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 10);
-CREATE TABLE file_data_11 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 11);
-CREATE TABLE file_data_12 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 12);
-CREATE TABLE file_data_13 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 13);
-CREATE TABLE file_data_14 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 14);
-CREATE TABLE file_data_15 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 15);
-
-CREATE INDEX file_data__deleted_at__idx
-    ON file_data (deleted_at, file_id, id)
- WHERE deleted_at IS NOT NULL;

backend/src/app/migrations/sql/0141-add-idx-to-file-library-rel.sql

@@ -1,2 +0,0 @@
-CREATE INDEX IF NOT EXISTS file_library_rel__library_file_id__idx
-    ON file_library_rel (library_file_id);

backend/src/app/msgbus.clj

@@ -10,12 +10,13 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.config :as cfg]
    [app.redis :as rds]
+   [app.util.time :as dt]
    [app.worker :as wrk]
    [integrant.core :as ig]
+   [promesa.core :as p]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]))
 
@@ -55,19 +56,17 @@
   [k v]
   {k (-> (d/without-nils v)
          (assoc ::buffer-size 128)
-         (assoc ::timeout (ct/duration {:seconds 30})))})
+         (assoc ::timeout (dt/duration {:seconds 30})))})
 
 (def ^:private schema:params
-  [:map
-   ::rds/client
-   ::wrk/executor])
+  [:map ::rds/redis ::wrk/executor])
 
 (defmethod ig/assert-key ::msgbus
   [_ params]
   (assert (sm/check schema:params params)))
 
 (defmethod ig/init-key ::msgbus
-  [_ {:keys [::buffer-size ::wrk/executor ::timeout] :as cfg}]
+  [_ {:keys [::buffer-size ::wrk/executor ::timeout ::rds/redis] :as cfg}]
   (l/info :hint "initialize msgbus" :buffer-size buffer-size)
   (let [cmd-ch (sp/chan :buf buffer-size)
         rcv-ch (sp/chan :buf (sp/dropping-buffer buffer-size))
@@ -75,9 +74,8 @@
                         :xf  xform-prefix-topic)
         state  (agent {})
 
-        ;; Open persistent connections to redis
-        pconn  (rds/connect cfg :timeout timeout)
-        sconn  (rds/connect-pubsub cfg :timeout timeout)
+        pconn  (rds/connect redis :type :default :timeout timeout)
+        sconn  (rds/connect redis :type :pubsub :timeout timeout)
 
         _      (set-error-handler! state #(l/error :cause % :hint "unexpected error on agent" ::l/sync? true))
         _      (set-error-mode! state :continue)
@@ -191,13 +189,14 @@
 
 (defn- create-listener
   [rcv-ch]
-  {:on-message (fn [_ topic message]
+  (rds/pubsub-listener
+   :on-message (fn [_ topic message]
                  ;; There are no back pressure, so we use a slidding
                  ;; buffer for cases when the pubsub broker sends
                  ;; more messages that we can process.
-                 (let [val {:topic topic :message (t/decode-str message)}]
+                 (let [val {:topic topic :message (t/decode message)}]
                    (when-not (sp/offer! rcv-ch val)
-                     (l/warn :msg "dropping message on subscription loop"))))})
+                     (l/warn :msg "dropping message on subscription loop"))))))
 
 (defn- process-input
   [{:keys [::state ::wrk/executor] :as cfg} topic message]
@@ -217,7 +216,8 @@
   (rds/add-listener sconn (create-listener rcv-ch))
 
   (px/thread
-    {:name "penpot/msgbus"}
+    {:name "penpot/msgbus/io-loop"
+     :virtual true}
     (try
       (loop []
         (let [timeout-ch (sp/timeout-chan 1000)
@@ -263,7 +263,7 @@
   intended to be used in core.async go blocks."
   [{:keys [::pconn] :as cfg} {:keys [topic message]}]
   (try
-    (rds/publish pconn topic (t/encode-str message))
+    (p/await! (rds/publish pconn topic (t/encode message)))
     (catch InterruptedException cause
       (throw cause))
     (catch Throwable cause

backend/src/app/redis.clj

@@ -6,22 +6,23 @@
 
 (ns app.redis
   "The msgbus abstraction implemented using redis as underlying backend."
-  (:refer-clojure :exclude [eval get set run!])
+  (:refer-clojure :exclude [eval])
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.generic-pool :as gpool]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.metrics :as mtx]
    [app.redis.script :as-alias rscript]
-   [app.worker :as wrk]
-   [app.worker.executor]
+   [app.util.cache :as cache]
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
    [clojure.core :as c]
    [clojure.java.io :as io]
    [cuerdas.core :as str]
-   [integrant.core :as ig])
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px])
   (:import
    clojure.lang.MapEntry
    io.lettuce.core.KeyValue
@@ -31,10 +32,12 @@
    io.lettuce.core.RedisException
    io.lettuce.core.RedisURI
    io.lettuce.core.ScriptOutputType
-   io.lettuce.core.SetArgs
+   io.lettuce.core.api.StatefulConnection
    io.lettuce.core.api.StatefulRedisConnection
+   io.lettuce.core.api.async.RedisAsyncCommands
+   io.lettuce.core.api.async.RedisScriptingAsyncCommands
    io.lettuce.core.api.sync.RedisCommands
-   io.lettuce.core.api.sync.RedisScriptingCommands
+   io.lettuce.core.codec.ByteArrayCodec
    io.lettuce.core.codec.RedisCodec
    io.lettuce.core.codec.StringCodec
    io.lettuce.core.pubsub.RedisPubSubListener
@@ -42,238 +45,244 @@
    io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
    io.lettuce.core.resource.ClientResources
    io.lettuce.core.resource.DefaultClientResources
-   io.netty.channel.nio.NioEventLoopGroup
    io.netty.util.HashedWheelTimer
    io.netty.util.Timer
-   io.netty.util.concurrent.EventExecutorGroup
    java.lang.AutoCloseable
    java.time.Duration))
 
 (set! *warn-on-reflection* true)
 
-(def ^:const MAX-EVAL-RETRIES 18)
+(declare ^:private initialize-resources)
+(declare ^:private shutdown-resources)
+(declare ^:private impl-eval)
 
-(def default-timeout
-  (ct/duration "10s"))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; IMPL & PRIVATE API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defprotocol IRedis
+  (-connect [_ options])
+  (-get-or-connect [_ key options]))
 
 (defprotocol IConnection
-  (-set-timeout [_ timeout] "set connection timeout")
-  (-get-timeout [_] "get current timeout")
-  (-reset-timeout [_] "reset to default timeout"))
-
-(defprotocol IDefaultConnection
-  "Public API of default redis connection"
-  (-publish [_ topic message])
-  (-rpush [_ key payload])
-  (-blpop [_ timeout keys])
-  (-eval [_ script])
-  (-get [_ key])
-  (-set [_ key val args])
-  (-del [_ key-or-keys])
-  (-ping [_]))
+  (publish [_ topic message])
+  (rpush [_ key payload])
+  (blpop [_ timeout keys])
+  (eval [_ script]))
 
 (defprotocol IPubSubConnection
-  (-add-listener [_ listener])
-  (-subscribe [_ topics])
-  (-unsubscribe [_ topics]))
+  (add-listener [_ listener])
+  (subscribe [_ topics])
+  (unsubscribe [_ topics]))
 
-(def ^:private default-codec
+(def default-codec
+  (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE))
+
+(def string-codec
   (RedisCodec/of StringCodec/UTF8 StringCodec/UTF8))
 
-(defn- impl-eval
-  [cmd cache metrics script]
-  (let [keys    (into-array String (map str (::rscript/keys script)))
-        vals    (into-array String (map str (::rscript/vals script)))
-        sname   (::rscript/name script)
+(sm/register!
+ {:type ::connection
+  :pred #(satisfies? IConnection %)
+  :type-properties
+  {:title "connection"
+   :description "redis connection instance"}})
 
-        read-script
-        (fn []
-          (-> script ::rscript/path io/resource slurp))
+(sm/register!
+ {:type ::pubsub-connection
+  :pred #(satisfies? IPubSubConnection %)
+  :type-properties
+  {:title "connection"
+   :description "redis connection instance"}})
 
-        load-script
-        (fn []
-          (let [id (.scriptLoad ^RedisScriptingCommands cmd
-                                ^String (read-script))]
-            (swap! cache assoc sname id)
-            (l/trc :hint "load script" :name sname :id id)
+(defn redis?
+  [o]
+  (satisfies? IRedis o))
 
-            id))
+(sm/register!
+ {:type ::redis
+  :pred redis?})
 
-        eval-script
-        (fn [id]
-          (try
-            (let [tpoint  (ct/tpoint)
-                  result  (.evalsha ^RedisScriptingCommands cmd
-                                    ^String id
-                                    ^ScriptOutputType ScriptOutputType/MULTI
-                                    ^"[Ljava.lang.String;" keys
-                                    ^"[Ljava.lang.String;" vals)
-                  elapsed (tpoint)]
+(def ^:private schema:script
+  [:map {:title "script"}
+   [::rscript/name qualified-keyword?]
+   [::rscript/path ::sm/text]
+   [::rscript/keys {:optional true} [:vector :any]]
+   [::rscript/vals {:optional true} [:vector :any]]])
 
-              (mtx/run! metrics {:id :redis-eval-timing
-                                 :labels [(name sname)]
-                                 :val (inst-ms elapsed)})
+(def valid-script?
+  (sm/lazy-validator schema:script))
 
-              (l/trc :hint "eval script"
-                     :name (name sname)
-                     :id id
-                     :params (str/join "," (::rscript/vals script))
-                     :elapsed (ct/format-duration elapsed))
+(defmethod ig/expand-key ::redis
+  [k v]
+  (let [cpus    (px/get-available-processors)
+        threads (max 1 (int (* cpus 0.2)))]
+    {k (-> (d/without-nils v)
+           (assoc ::timeout (dt/duration "10s"))
+           (assoc ::io-threads (max 3 threads))
+           (assoc ::worker-threads (max 3 threads)))}))
 
-              result)
+(def ^:private schema:redis-params
+  [:map {:title "redis-params"}
+   ::wrk/executor
+   ::mtx/metrics
+   [::uri ::sm/uri]
+   [::worker-threads ::sm/int]
+   [::io-threads ::sm/int]
+   [::timeout ::dt/duration]])
 
-            (catch io.lettuce.core.RedisNoScriptException _cause
-              ::load)
+(defmethod ig/assert-key ::redis
+  [_ params]
+  (assert (sm/check schema:redis-params params)))
 
-            (catch Throwable cause
-              (when-let [on-error (::rscript/on-error script)]
-                (on-error cause))
-              (throw cause))))
+(defmethod ig/init-key ::redis
+  [_ params]
+  (initialize-resources params))
 
-        eval-script'
-        (fn [id]
-          (loop [id      id
-                 retries 0]
-            (if (> retries MAX-EVAL-RETRIES)
-              (ex/raise :type :internal
-                        :code ::max-eval-retries-reached
-                        :hint (str "unable to eval redis script " sname))
-              (let [result (eval-script id)]
-                (if (= result ::load)
-                  (recur (load-script)
-                         (inc retries))
-                  result)))))]
+(defmethod ig/halt-key! ::redis
+  [_ instance]
+  (d/close! instance))
 
-    (if-let [id (c/get @cache sname)]
-      (eval-script' id)
-      (-> (load-script)
-          (eval-script')))))
+(defn- initialize-resources
+  "Initialize redis connection resources"
+  [{:keys [::uri ::io-threads ::worker-threads ::wrk/executor ::mtx/metrics] :as params}]
 
-(deftype Connection [^StatefulRedisConnection conn
-                     ^RedisCommands cmd
-                     ^Duration timeout
-                     cache metrics]
-  AutoCloseable
-  (close [_]
-    (ex/ignoring (.close conn)))
+  (l/inf :hint "initialize redis resources"
+         :uri (str uri)
+         :io-threads io-threads
+         :worker-threads worker-threads)
 
-  IConnection
-  (-set-timeout [_ timeout]
-    (.setTimeout conn ^Duration timeout))
+  (let [timer     (HashedWheelTimer.)
+        resources (.. (DefaultClientResources/builder)
+                      (ioThreadPoolSize ^long io-threads)
+                      (computationThreadPoolSize ^long worker-threads)
+                      (timer ^Timer timer)
+                      (build))
 
-  (-reset-timeout [_]
-    (.setTimeout conn timeout))
+        redis-uri (RedisURI/create ^String (str uri))
 
-  (-get-timeout [_]
-    (.getTimeout conn))
+        shutdown  (fn [client conn]
+                    (ex/ignoring (.close ^StatefulConnection conn))
+                    (ex/ignoring (.close ^RedisClient client))
+                    (l/trc :hint "disconnect" :hid (hash client)))
 
-  IDefaultConnection
-  (-publish [_ topic message]
-    (.publish cmd ^String topic ^String message))
+        on-remove (fn [key val cause]
+                    (l/trace :hint "evict connection (cache)" :key key :reason cause)
+                    (some-> val d/close!))
 
-  (-rpush [_ key elements]
-    (try
-      (let [vals (make-array String (count elements))]
-        (loop [i 0 xs (seq elements)]
-          (when xs
-            (aset ^"[[Ljava.lang.String;" vals i ^String (first xs))
-            (recur (inc i) (next xs))))
+        cache     (cache/create :executor executor
+                                :on-remove on-remove
+                                :keepalive "5m")]
+    (reify
+      java.lang.AutoCloseable
+      (close [_]
+        (ex/ignoring (cache/invalidate! cache))
+        (ex/ignoring (.shutdown ^ClientResources resources))
+        (ex/ignoring (.stop ^Timer timer)))
 
-        (.rpush cmd
-                ^String key
-                ^"[[Ljava.lang.String;" vals))
+      IRedis
+      (-get-or-connect [this key options]
+        (let [create (fn [_] (-connect this options))]
+          (cache/get cache key create)))
 
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause))))))
+      (-connect [_ options]
+        (let [timeout (or (:timeout options) (::timeout params))
+              codec   (get options :codec default-codec)
+              type    (get options :type :default)
+              client  (RedisClient/create ^ClientResources resources
+                                          ^RedisURI redis-uri)]
 
-  (-blpop [_ keys timeout]
-    (try
-      (let [keys (into-array String keys)]
-        (when-let [res (.blpop cmd
-                               ^double timeout
-                               ^"[Ljava.lang.String;" keys)]
-          (MapEntry/create
-           (.getKey ^KeyValue res)
-           (.getValue ^KeyValue res))))
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause))))))
+          (l/trc :hint "connect" :hid (hash client))
+          (if (= type :pubsub)
+            (let [conn (.connectPubSub ^RedisClient client
+                                       ^RedisCodec codec)]
+              (.setTimeout ^StatefulConnection conn
+                           ^Duration timeout)
+              (reify
+                IPubSubConnection
+                (add-listener [_ listener]
+                  (assert (instance? RedisPubSubListener listener) "expected listener instance")
+                  (.addListener ^StatefulRedisPubSubConnection conn
+                                ^RedisPubSubListener listener))
 
-  (-get [_ key]
-    (assert (string? key) "key expected to be string")
-    (.get cmd ^String key))
+                (subscribe [_ topics]
+                  (try
+                    (let [topics (into-array String (map str topics))
+                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
+                      (.subscribe ^RedisPubSubCommands cmd topics))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
-  (-set [_ key val args]
-    (.set cmd
-          ^String key
-          ^bytes val
-          ^SetArgs args))
-
-  (-del [_ keys]
-    (let [keys (into-array String keys)]
-      (.del cmd ^String/1 keys)))
-
-  (-ping [_]
-    (.ping cmd))
-
-  (-eval [_ script]
-    (impl-eval cmd cache metrics script)))
+                (unsubscribe [_ topics]
+                  (try
+                    (let [topics (into-array String (map str topics))
+                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
+                      (.unsubscribe ^RedisPubSubCommands cmd topics))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
 
-(deftype SubscriptionConnection [^StatefulRedisPubSubConnection conn
-                                 ^RedisPubSubCommands cmd
-                                 ^Duration timeout]
-  AutoCloseable
-  (close [_]
-    (ex/ignoring (.close conn)))
+                AutoCloseable
+                (close [_] (shutdown client conn))))
 
-  IConnection
-  (-set-timeout [_ timeout]
-    (.setTimeout conn ^Duration timeout))
+            (let [conn (.connect ^RedisClient client ^RedisCodec codec)]
+              (.setTimeout ^StatefulConnection conn ^Duration timeout)
+              (reify
+                IConnection
+                (publish [_ topic message]
+                  (assert (string? topic) "expected topic to be string")
+                  (assert (bytes? message) "expected message to be a byte array")
 
-  (-reset-timeout [_]
-    (.setTimeout conn timeout))
+                  (let [pcomm (.async ^StatefulRedisConnection conn)]
+                    (.publish ^RedisAsyncCommands pcomm ^String topic ^bytes message)))
 
-  (-get-timeout [_]
-    (.getTimeout conn))
+                (rpush [_ key payload]
+                  (assert (or (and (vector? payload)
+                                   (every? bytes? payload))
+                              (bytes? payload)))
+                  (try
+                    (let [cmd  (.sync ^StatefulRedisConnection conn)
+                          data (if (vector? payload) payload [payload])
+                          vals (make-array (. Class (forName "[B")) (count data))]
 
-  IPubSubConnection
-  (-add-listener [_ listener]
-    (.addListener conn ^RedisPubSubListener listener))
+                      (loop [i 0 xs (seq data)]
+                        (when xs
+                          (aset ^"[[B" vals i ^bytes (first xs))
+                          (recur (inc i) (next xs))))
 
-  (-subscribe [_ topics]
-    (try
-      (let [topics (into-array String topics)]
-        (.subscribe cmd topics))
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause))))))
+                      (.rpush ^RedisCommands cmd
+                              ^String key
+                              ^"[[B" vals))
 
-  (-unsubscribe [_ topics]
-    (try
-      (let [topics (into-array String topics)]
-        (.unsubscribe cmd topics))
-      (catch RedisCommandInterruptedException cause
-        (throw (InterruptedException. (ex-message cause)))))))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; PUBLIC API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+                (blpop [_ timeout keys]
+                  (try
+                    (let [keys    (into-array Object (map str keys))
+                          cmd     (.sync ^StatefulRedisConnection conn)
+                          timeout (/ (double (inst-ms timeout)) 1000.0)]
+                      (when-let [res (.blpop ^RedisCommands cmd
+                                             ^double timeout
+                                             ^"[Ljava.lang.String;" keys)]
+                        (MapEntry/create
+                         (.getKey ^KeyValue res)
+                         (.getValue ^KeyValue res))))
+                    (catch RedisCommandInterruptedException cause
+                      (throw (InterruptedException. (ex-message cause))))))
 
-(defn build-set-args
-  [options]
-  (reduce-kv (fn [^SetArgs args k v]
-               (case k
-                 :ex (if (instance? Duration v)
-                       (.ex args ^Duration v)
-                       (.ex args (long v)))
-                 :px (.px args (long v))
-                 :nx (if v (.nx args) args)
-                 :keep-ttl (if v (.keepttl args) args)))
-             (SetArgs.)
-             options))
+                (eval [_ script]
+                  (assert (valid-script? script) "expected valid script")
+                  (impl-eval conn metrics script))
+
+                AutoCloseable
+                (close [_] (shutdown client conn))))))))))
+
+(defn connect
+  [instance & {:as opts}]
+  (assert (satisfies? IRedis instance) "expected valid redis instance")
+  (-connect instance opts))
+
+(defn get-or-connect
+  [instance key & {:as opts}]
+  (assert (satisfies? IRedis instance) "expected valid redis instance")
+  (-get-or-connect instance key opts))
 
 (defn pubsub-listener
   [& {:keys [on-message on-subscribe on-unsubscribe]}]
@@ -302,172 +311,61 @@
       (when on-unsubscribe
         (on-unsubscribe nil topic count)))))
 
-(defn connect
-  [cfg & {:as options}]
-  (assert (contains? cfg ::mtx/metrics) "missing ::mtx/metrics on provided system")
-  (assert (contains? cfg ::client) "missing ::rds/client on provided system")
+(def ^:private scripts-cache (atom {}))
 
-  (let [state   (::client cfg)
+(defn- impl-eval
+  [^StatefulRedisConnection connection metrics script]
+  (let [cmd   (.async ^StatefulRedisConnection connection)
+        keys  (into-array String (map str (::rscript/keys script)))
+        vals  (into-array String (map str (::rscript/vals script)))
+        sname (::rscript/name script)]
 
-        cache   (::cache state)
-        client  (::client state)
-        timeout (or (some-> (:timeout options) ct/duration)
-                    (::timeout state))
+    (letfn [(on-error [cause]
+              (if (instance? io.lettuce.core.RedisNoScriptException cause)
+                (do
+                  (l/error :hint "no script found" :name sname :cause cause)
+                  (->> (load-script)
+                       (p/mcat eval-script)))
+                (if-let [on-error (::rscript/on-error script)]
+                  (on-error cause)
+                  (p/rejected cause))))
 
-        conn    (.connect ^RedisClient client
-                          ^RedisCodec default-codec)
-        cmd     (.sync ^StatefulRedisConnection conn)]
+            (eval-script [sha]
+              (let [tpoint (dt/tpoint)]
+                (->> (.evalsha ^RedisScriptingAsyncCommands cmd
+                               ^String sha
+                               ^ScriptOutputType ScriptOutputType/MULTI
+                               ^"[Ljava.lang.String;" keys
+                               ^"[Ljava.lang.String;" vals)
+                     (p/fmap (fn [result]
+                               (let [elapsed (tpoint)]
+                                 (mtx/run! metrics {:id :redis-eval-timing
+                                                    :labels [(name sname)]
+                                                    :val (inst-ms elapsed)})
+                                 (l/trace :hint "eval script"
+                                          :name (name sname)
+                                          :sha sha
+                                          :params (str/join "," (::rscript/vals script))
+                                          :elapsed (dt/format-duration elapsed))
+                                 result)))
+                     (p/merr on-error))))
 
-    (.setTimeout ^StatefulRedisConnection conn ^Duration timeout)
-    (->Connection conn cmd timeout cache (::mtx/metrics cfg))))
+            (read-script []
+              (-> script ::rscript/path io/resource slurp))
 
-(defn connect-pubsub
-  [cfg & {:as options}]
-  (let [state   (::client cfg)
-        client  (::client state)
+            (load-script []
+              (l/trace :hint "load script" :name sname)
+              (->> (.scriptLoad ^RedisScriptingAsyncCommands cmd
+                                ^String (read-script))
+                   (p/fmap (fn [sha]
+                             (swap! scripts-cache assoc sname sha)
+                             sha))))]
 
-        timeout (or (some-> (:timeout options) ct/duration)
-                    (::timeout state))
-        conn    (.connectPubSub ^RedisClient client
-                                ^RedisCodec default-codec)
-        cmd     (.sync ^StatefulRedisPubSubConnection conn)]
-
-
-    (.setTimeout ^StatefulRedisPubSubConnection conn
-                 ^Duration timeout)
-    (->SubscriptionConnection conn cmd timeout)))
-
-(defn get
-  [conn key]
-  (assert (string? key) "key must be string instance")
-  (try
-    (-get conn key)
-    (catch RedisCommandTimeoutException cause
-      (l/err :hint "timeout on get redis key" :key key :cause cause)
-      nil)))
-
-(defn set
-  ([conn key val]
-   (set conn key val nil))
-  ([conn key val args]
-   (assert (string? key) "key must be string instance")
-   (assert (string? val) "val must be string instance")
-   (let [args (cond
-                (or (instance? SetArgs args)
-                    (nil? args))
-                args
-
-                (map? args)
-                (build-set-args args)
-
-                :else
-                (throw (IllegalArgumentException. "invalid args")))]
-
-     (try
-       (-set conn key val args)
-       (catch RedisCommandTimeoutException cause
-         (l/err :hint "timeout on set redis key" :key key :cause cause)
-         nil)))))
-
-(defn del
-  [conn key-or-keys]
-  (let [keys (if (vector? key-or-keys) key-or-keys [key-or-keys])]
-    (assert (every? string? keys) "only string keys allowed")
-    (try
-      (-del conn keys)
-      (catch RedisCommandTimeoutException cause
-        (l/err :hint "timeout on del redis key" :key key :cause cause)
-        nil))))
-
-(defn ping
-  [conn]
-  (-ping conn))
-
-(defn blpop
-  [conn key-or-keys timeout]
-  (let [keys    (if (vector? key-or-keys) key-or-keys [key-or-keys])
-        timeout (cond
-                  (ct/duration? timeout)
-                  (/ (double (inst-ms timeout)) 1000.0)
-
-                  (double? timeout)
-                  timeout
-
-                  (int? timeout)
-                  (/ (double timeout) 1000.0)
-
-                  :else
-                  0)]
-
-    (assert (every? string? keys) "only string keys allowed")
-    (-blpop conn keys timeout)))
-
-(defn rpush
-  [conn key elements]
-  (assert (string? key) "key must be string instance")
-  (assert (every? string? elements) "elements should be all strings")
-  (let [elements (vec elements)]
-    (-rpush conn key elements)))
-
-(defn publish
-  [conn topic payload]
-  (assert (string? topic) "expected topic to be string")
-  (assert (string? payload) "expected message to be a byte array")
-  (-publish conn topic payload))
-
-(def ^:private schema:script
-  [:map {:title "script"}
-   [::rscript/name qualified-keyword?]
-   [::rscript/path ::sm/text]
-   [::rscript/keys {:optional true} [:vector :any]]
-   [::rscript/vals {:optional true} [:vector :any]]])
-
-(def ^:private valid-script?
-  (sm/lazy-validator schema:script))
-
-(defn eval
-  [conn script]
-  (assert (valid-script? script) "expected valid script")
-  (-eval conn script))
-
-(defn add-listener
-  [conn listener]
-  (let [listener (cond
-                   (map? listener)
-                   (pubsub-listener listener)
-
-                   (instance? RedisPubSubListener listener)
-                   listener
-
-                   :else
-                   (throw (IllegalArgumentException. "invalid listener provided")))]
-
-    (-add-listener conn listener)))
-
-(defn subscribe
-  [conn topic-or-topics]
-  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
-    (assert (every? string? topics))
-    (-subscribe conn topics)))
-
-(defn unsubscribe
-  [conn topic-or-topics]
-  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
-    (assert (every? string? topics))
-    (-unsubscribe conn topics)))
-
-(defn set-timeout
-  [conn timeout]
-  (let [timeout (ct/duration timeout)]
-    (-set-timeout conn timeout)))
-
-(defn get-timeout
-  [conn]
-  (-get-timeout conn))
-
-(defn reset-timeout
-  [conn]
-  (-reset-timeout conn))
+      (p/await!
+       (if-let [sha (get @scripts-cache sname)]
+         (eval-script sha)
+         (->> (load-script)
+              (p/mapcat eval-script)))))))
 
 (defn timeout-exception?
   [cause]
@@ -476,121 +374,3 @@
 (defn exception?
   [cause]
   (instance? RedisException cause))
-
-(defn get-pooled
-  [cfg]
-  (let [pool (::pool cfg)]
-    (gpool/get pool)))
-
-(defn close
-  [o]
-  (.close ^AutoCloseable o))
-
-(defn pool
-  [cfg & {:as options}]
-  (gpool/create :create-fn (partial connect cfg options)
-                :destroy-fn close
-                :dispose-fn -reset-timeout))
-
-(defn run!
-  [cfg f & args]
-  (if (gpool/pool? cfg)
-    (apply f {::pool cfg} f args)
-    (let [pool (::pool cfg)]
-      (with-open [^AutoCloseable conn (gpool/get pool)]
-        (apply f (assoc cfg ::conn @conn) args)))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; INITIALIZATION
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmethod ig/expand-key ::client
-  [k v]
-  {k (-> (d/without-nils v)
-         (assoc ::timeout (ct/duration "10s")))})
-
-(def ^:private schema:client
-  [:map {:title "RedisClient"}
-   [::timer [:fn #(instance? HashedWheelTimer %)]]
-   [::cache ::sm/atom]
-   [::timeout ::ct/duration]
-   [::resources [:fn #(instance? DefaultClientResources %)]]])
-
-(def check-client
-  (sm/check-fn schema:client))
-
-(sm/register! ::client schema:client)
-(sm/register!
- {:type ::pool
-  :pred gpool/pool?})
-
-(def ^:private schema:client-params
-  [:map {:title "redis-params"}
-   ::wrk/netty-io-executor
-   ::wrk/netty-executor
-   [::uri ::sm/uri]
-   [::timeout ::ct/duration]])
-
-(def ^:private check-client-params
-  (sm/check-fn schema:client-params))
-
-(defmethod ig/assert-key ::client
-  [_ params]
-  (check-client-params params))
-
-(defmethod ig/init-key ::client
-  [_ {:keys [::uri ::wrk/netty-io-executor ::wrk/netty-executor] :as params}]
-
-  (l/inf :hint "initialize redis client" :uri (str uri))
-
-  (let [timer     (HashedWheelTimer.)
-        cache     (atom {})
-
-        resources (.. (DefaultClientResources/builder)
-                      (eventExecutorGroup ^EventExecutorGroup netty-executor)
-
-                      ;; We provide lettuce with a shared event loop
-                      ;; group instance instead of letting lettuce to
-                      ;; create its own
-                      (eventLoopGroupProvider
-                       (reify io.lettuce.core.resource.EventLoopGroupProvider
-                         (allocate [_ _] netty-io-executor)
-                         (threadPoolSize [_]
-                           (.executorCount ^NioEventLoopGroup netty-io-executor))
-                         (release [_ _ _ _ _]
-                           ;; Do nothing
-                           )
-                         (shutdown [_ _ _ _]
-                           ;; Do nothing
-                           )))
-
-                      (timer ^Timer timer)
-                      (build))
-
-        redis-uri (RedisURI/create ^String (str uri))
-        client    (RedisClient/create ^ClientResources resources
-                                      ^RedisURI redis-uri)]
-
-    {::client client
-     ::cache cache
-     ::timer timer
-     ::timeout default-timeout
-     ::resources resources}))
-
-(defmethod ig/halt-key! ::client
-  [_ {:keys [::client ::timer ::resources]}]
-  (ex/ignoring (.shutdown ^RedisClient client))
-  (ex/ignoring (.shutdown ^ClientResources resources))
-  (ex/ignoring (.stop ^Timer timer)))
-
-(defmethod ig/assert-key ::pool
-  [_ {:keys [::client]}]
-  (check-client client))
-
-(defmethod ig/init-key ::pool
-  [_ cfg]
-  (pool cfg {:timeout (ct/duration 2000)}))
-
-(defmethod ig/halt-key! ::pool
-  [_ instance]
-  (.close ^java.lang.AutoCloseable instance))

backend/src/app/rpc.clj

@@ -12,7 +12,6 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.spec :as us]
-   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
@@ -23,7 +22,6 @@
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.msgbus :as-alias mbus]
-   [app.redis :as rds]
    [app.rpc.climit :as climit]
    [app.rpc.cond :as cond]
    [app.rpc.helpers :as rph]
@@ -33,6 +31,7 @@
    [app.storage :as-alias sto]
    [app.util.inet :as inet]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
@@ -64,13 +63,9 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result  (rph/unwrap result)
-                         status  (::http/status mdata 200)
-                         headers (cond-> (::http/headers mdata {})
-                                   (yres/stream-body? result)
-                                   (assoc "content-type" "application/octet-stream"))]
-                     {::yres/status  status
-                      ::yres/headers headers
+                   (let [result (rph/unwrap result)]
+                     {::yres/status  (::http/status mdata 200)
+                      ::yres/headers (::http/headers mdata {})
                       ::yres/body    result}))]
     (-> response
         (handle-response-transformation request mdata)
@@ -108,7 +103,7 @@
         data         (-> params
                          (assoc ::handler-name handler-name)
                          (assoc ::ip-addr ip-addr)
-                         (assoc ::request-at (ct/now))
+                         (assoc ::request-at (dt/now))
                          (assoc ::external-session-id session-id)
                          (assoc ::external-event-origin event-origin)
                          (assoc ::session/id (::session/id request))
@@ -135,7 +130,7 @@
   [{:keys [::mtx/metrics ::metrics-id]} f mdata]
   (let [labels (into-array String [(::sv/name mdata)])]
     (fn [cfg params]
-      (let [tp (ct/tpoint)]
+      (let [tp (dt/tpoint)]
         (try
           (f cfg params)
           (finally
@@ -244,6 +239,7 @@
           'app.rpc.commands.files
           'app.rpc.commands.files-create
           'app.rpc.commands.files-share
+          'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
           'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails
@@ -266,7 +262,6 @@
    ::session/manager
    ::http.client/client
    ::db/pool
-   ::rds/pool
    ::mbus/msgbus
    ::sto/storage
    ::mtx/metrics

backend/src/app/rpc/climit.clj

@@ -11,16 +11,17 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
    [app.util.cache :as cache]
    [app.util.services :as-alias sv]
+   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.edn :as edn]
    [clojure.set :as set]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
+   [promesa.exec :as px]
    [promesa.exec.bulkhead :as pbh])
   (:import
    clojure.lang.ExceptionInfo
@@ -153,7 +154,7 @@
            :id limit-id
            :label limit-label
            :queue queue
-           :elapsed (some-> elapsed ct/format-duration)
+           :elapsed (some-> elapsed dt/format-duration)
            :params @limit-params)))
 
 (def ^:private idseq (AtomicLong. 0))
@@ -170,19 +171,19 @@
         mlabels     (into-array String [(id->str limit-id)])
         limit-id    (id->str limit-id limit-key)
         limiter     (cache/get cache limit-id (partial create-limiter config))
-        tpoint      (ct/tpoint)
+        tpoint      (dt/tpoint)
         req-id      (.incrementAndGet ^AtomicLong idseq)]
     (try
       (let [stats (pbh/get-stats limiter)]
         (measure metrics mlabels stats nil)
         (log "enqueued" req-id stats limit-id limit-label limit-params nil))
 
-      (pbh/invoke! limiter (fn []
-                             (let [elapsed (tpoint)
-                                   stats   (pbh/get-stats limiter)]
-                               (measure metrics mlabels stats elapsed)
-                               (log "acquired" req-id stats limit-id limit-label limit-params elapsed)
-                               (handler))))
+      (px/invoke! limiter (fn []
+                            (let [elapsed (tpoint)
+                                  stats   (pbh/get-stats limiter)]
+                              (measure metrics mlabels stats elapsed)
+                              (log "acquired" req-id stats limit-id limit-label limit-params elapsed)
+                              (handler))))
 
       (catch ExceptionInfo cause
         (let [{:keys [type code]} (ex-data cause)]
@@ -288,9 +289,13 @@
           (get-limits cfg)))
 
 (defn invoke!
-  "Run a function in context of climit."
-  [{:keys [::rpc/climit] :as cfg} f params]
+  "Run a function in context of climit.
+  Intended to be used in virtual threads."
+  [{:keys [::executor ::rpc/climit] :as cfg} f params]
   (let [f (if climit
-            (build-exec-chain cfg f)
+            (let [f (if (some? executor)
+                      (fn [cfg params] (px/await! (px/submit! executor (fn [] (f cfg params)))))
+                      f)]
+              (build-exec-chain cfg f))
             f)]
     (f cfg params)))

backend/src/app/rpc/commands/access_token.clj

@@ -7,7 +7,6 @@
 (ns app.rpc.commands.access-token
   (:require
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.main :as-alias main]
@@ -16,21 +15,22 @@
    [app.rpc.quotes :as quotes]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.util.time :as dt]))
 
 (defn- decode-row
   [row]
   (dissoc row :perms))
 
 (defn create-access-token
-  [{:keys [::db/conn] :as cfg} profile-id name expiration]
-  (let [token-id   (uuid/next)
-        expires-at (some-> expiration (ct/in-future))
-        created-at (ct/now)
-        token      (tokens/generate cfg {:iss "access-token"
-                                         :iat created-at
-                                         :tid token-id})
+  [{:keys [::db/conn ::setup/props]} profile-id name expiration]
+  (let [created-at (dt/now)
+        token-id   (uuid/next)
+        token      (tokens/generate props {:iss "access-token"
+                                           :tid token-id
+                                           :iat created-at})
 
+        expires-at (some-> expiration dt/in-future)
         token      (db/insert! conn :access-token
                                {:id token-id
                                 :name name
@@ -49,7 +49,7 @@
 (def ^:private schema:create-access-token
   [:map {:title "create-access-token"}
    [:name [:string {:max 250 :min 1}]]
-   [:expiration {:optional true} ::ct/duration]])
+   [:expiration {:optional true} ::dt/duration]])
 
 (sv/defmethod ::create-access-token
   {::doc/added "1.18"

backend/src/app/rpc/commands/audit.clj

@@ -10,7 +10,6 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -21,7 +20,8 @@
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
    [app.util.inet :as inet]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.util.time :as dt]))
 
 (def ^:private event-columns
   [:id
@@ -49,7 +49,7 @@
 
 (defn- adjust-timestamp
   [{:keys [timestamp created-at] :as event}]
-  (let [margin (inst-ms (ct/diff timestamp created-at))]
+  (let [margin (inst-ms (dt/diff timestamp created-at))]
     (if (or (neg? margin)
             (> margin 3600000))
       ;; If event is in future or lags more than 1 hour, we reasign
@@ -63,7 +63,7 @@
   [{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}]
   (let [request (-> params meta ::http/request)
         ip-addr (inet/parse-request request)
-        tnow    (ct/now)
+        tnow    (dt/now)
         xform   (comp
                  (map (fn [event]
                         (-> event
@@ -92,9 +92,9 @@
      [:string {:max 250}]
      [::sm/one-of {:format "string"} valid-event-types]]]
    [:props
-    [:map-of :keyword ::sm/any]]
+    [:map-of :keyword :any]]
    [:context {:optional true}
-    [:map-of :keyword ::sm/any]]])
+    [:map-of :keyword :any]]])
 
 (def schema:push-audit-events
   [:map {:title "push-audit-events"}

backend/src/app/rpc/commands/auth.clj

@@ -6,14 +6,12 @@
 
 (ns app.rpc.commands.auth
   (:require
-   [app.auth :as auth]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -32,6 +30,7 @@
    [app.setup.welcome-file :refer [create-welcome-file]]
    [app.tokens :as tokens]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]
    [cuerdas.core :as str]))
 
@@ -43,7 +42,7 @@
 
 (defn- elapsed-verify-threshold?
   [profile]
-  (let [elapsed (ct/diff (:modified-at profile) (ct/now))
+  (let [elapsed (dt/diff (:modified-at profile) (dt/now))
         verify-threshold (cf/get :email-verify-threshold)]
     (pos? (compare elapsed verify-threshold))))
 
@@ -63,7 +62,7 @@
               (ex/raise :type :validation
                         :code :account-without-password
                         :hint "the current account does not have password")
-              (let [result (auth/verify-password password (:password profile))]
+              (let [result (profile/verify-password cfg password (:password profile))]
                 (when (:update result)
                   (l/trc :hint "updating profile password"
                          :id (str (:id profile))
@@ -86,7 +85,7 @@
               (ex/raise :type :validation
                         :code :wrong-credentials))
             (when-let [deleted-at (:deleted-at profile)]
-              (when (ct/is-after? (ct/now) deleted-at)
+              (when (dt/is-after? (dt/now) deleted-at)
                 (ex/raise :type :validation
                           :code :wrong-credentials)))
 
@@ -99,7 +98,7 @@
                                   (profile/strip-private-attrs))
 
                   invitation (when-let [token (:invitation-token params)]
-                               (tokens/verify cfg {:token token :iss :team-invitation}))
+                               (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
 
                   ;; If invitation member-id does not matches the profile-id, we just proceed to ignore the
                   ;; invitation because invitations matches exactly; and user can't login with other email and
@@ -153,11 +152,11 @@
 (defn recover-profile
   [{:keys [::db/conn] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
-            (let [tdata (tokens/verify cfg {:token token :iss :password-recovery})]
+            (let [tdata (tokens/verify (::setup/props cfg) {:token token :iss :password-recovery})]
               (:profile-id tdata)))
 
           (update-password [conn profile-id]
-            (let [pwd (auth/derive-password password)]
+            (let [pwd (profile/derive-password cfg password)]
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
@@ -192,7 +191,7 @@
               :hint "registration disabled"))
 
   (when (contains? params :invitation-token)
-    (let [invitation (tokens/verify cfg
+    (let [invitation (tokens/verify (::setup/props cfg)
                                     {:token (:invitation-token params)
                                      :iss :team-invitation})]
       (when-not (= (:email params) (:member-email invitation))
@@ -232,34 +231,31 @@
               :hint "email has complaint reports")))
 
 (defn prepare-register
-  [{:keys [::db/pool] :as cfg} {:keys [fullname email accept-newsletter-updates] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [email accept-newsletter-updates] :as params}]
 
   (validate-register-attempt! cfg params)
 
   (let [email   (profile/clean-email email)
         profile (profile/get-profile-by-email pool email)
         params  {:email email
-                 :fullname fullname
                  :password (:password params)
                  :invitation-token (:invitation-token params)
                  :backend "penpot"
                  :iss :prepared-register
                  :profile-id (:id profile)
-                 :exp (ct/in-future {:days 7})
+                 :exp (dt/in-future {:days 7})
                  :props {:newsletter-updates (or accept-newsletter-updates false)}}
 
         params (d/without-nils params)
-        token  (tokens/generate cfg params)]
+        token  (tokens/generate (::setup/props cfg) params)]
 
     (with-meta {:token token}
       {::audit/profile-id uuid/zero})))
 
 (def schema:prepare-register-profile
   [:map {:title "prepare-register-profile"}
-   [:fullname ::sm/text]
    [:email ::sm/email]
    [:password schema:password]
-   [:create-welcome-file {:optional true} :boolean]
    [:invitation-token {:optional true} schema:token]])
 
 (sv/defmethod ::prepare-register-profile
@@ -343,17 +339,17 @@
 
 (defn send-email-verification!
   [{:keys [::db/conn] :as cfg} profile]
-  (let [vtoken (tokens/generate cfg
+  (let [vtoken (tokens/generate (::setup/props cfg)
                                 {:iss :verify-email
-                                 :exp (ct/in-future "72h")
+                                 :exp (dt/in-future "72h")
                                  :profile-id (:id profile)
                                  :email (:email profile)})
         ;; NOTE: this token is mainly used for possible complains
         ;; identification on the sns webhook
-        ptoken (tokens/generate cfg
+        ptoken (tokens/generate (::setup/props cfg)
                                 {:iss :profile-identity
                                  :profile-id (:id profile)
-                                 :exp (ct/in-future {:days 30})})]
+                                 :exp (dt/in-future {:days 30})})]
     (eml/send! {::eml/conn conn
                 ::eml/factory eml/register
                 :public-uri (cf/get :public-uri)
@@ -363,9 +359,13 @@
                 :extra-data ptoken})))
 
 (defn register-profile
-  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token] :as params}]
-  (let [claims     (tokens/verify cfg {:token token :iss :prepared-register})
-        params     (into claims params)
+  [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token fullname theme] :as params}]
+  (let [theme      (when (= theme "light") theme)
+        claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
+        params     (-> claims
+                       (into params)
+                       (assoc :fullname fullname)
+                       (assoc :theme theme))
 
         profile    (if-let [profile-id (:profile-id claims)]
                      (profile/get-profile conn profile-id)
@@ -379,7 +379,7 @@
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
-                                             (update :password auth/derive-password))
+                                             (update :password #(profile/derive-password cfg %)))
                                profile   (->> (create-profile! conn params)
                                               (create-profile-rels! conn))]
                            (vary-meta profile assoc :created true))))
@@ -387,7 +387,7 @@
         created?   (-> profile meta :created true?)
 
         invitation (when-let [token (:invitation-token params)]
-                     (tokens/verify cfg {:token token :iss :team-invitation}))
+                     (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
 
         props      (-> (audit/profile->props profile)
                        (assoc :from-invitation (some? invitation)))
@@ -420,7 +420,7 @@
            (= (:email profile)
               (:member-email invitation)))
       (let [claims (assoc invitation :member-id  (:id profile))
-            token  (tokens/generate cfg claims)]
+            token  (tokens/generate (::setup/props cfg) claims)]
         (-> {:invitation-token token}
             (rph/with-transform (session/create-fn cfg (:id profile)))
             (rph/with-meta {::audit/replace-props props
@@ -467,7 +467,7 @@
 
         (when (= action "resend-email-verification")
           (db/update! conn :profile
-                      {:modified-at (ct/now)}
+                      {:modified-at (dt/now)}
                       {:id (:id profile)})
           (send-email-verification! cfg profile))
 
@@ -479,7 +479,10 @@
 
 (def schema:register-profile
   [:map {:title "register-profile"}
-   [:token schema:token]])
+   [:token schema:token]
+   [:fullname [::sm/word-string {:max 100}]]
+   [:theme {:optional true} [:string {:max 10}]]
+   [:create-welcome-file {:optional true} :boolean]])
 
 (sv/defmethod ::register-profile
   {::rpc/auth false
@@ -494,17 +497,17 @@
 (defn- request-profile-recovery
   [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
   (letfn [(create-recovery-token [{:keys [id] :as profile}]
-            (let [token (tokens/generate cfg
+            (let [token (tokens/generate (::setup/props cfg)
                                          {:iss :password-recovery
-                                          :exp (ct/in-future "15m")
+                                          :exp (dt/in-future "15m")
                                           :profile-id id})]
               (assoc profile :token token)))
 
           (send-email-notification [conn profile]
-            (let [ptoken (tokens/generate cfg
+            (let [ptoken (tokens/generate (::setup/props cfg)
                                           {:iss :profile-identity
                                            :profile-id (:id profile)
-                                           :exp (ct/in-future {:days 30})})]
+                                           :exp (dt/in-future {:days 30})})]
               (eml/send! {::eml/conn conn
                           ::eml/factory eml/password-recovery
                           :public-uri (cf/get :public-uri)
@@ -545,7 +548,7 @@
         :else
         (do
           (db/update! conn :profile
-                      {:modified-at (ct/now)}
+                      {:modified-at (dt/now)}
                       {:id (:id profile)})
           (->> profile
                (create-recovery-token)

backend/src/app/rpc/commands/binfile.clj

@@ -13,7 +13,6 @@
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
@@ -25,10 +24,12 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
-   [app.rpc.helpers :as rph]
    [app.tasks.file-gc]
    [app.util.services :as sv]
-   [app.worker :as-alias wrk]))
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]
+   [yetti.response :as yres]))
 
 (set! *warn-on-reflection* true)
 
@@ -44,7 +45,7 @@
 
 (defn stream-export-v1
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (rph/stream
+  (yres/stream-body
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -59,7 +60,7 @@
 
 (defn stream-export-v3
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (rph/stream
+  (yres/stream-body
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -79,16 +80,21 @@
    ::sm/params schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (let [version (or version 1)]
-    (case (int version)
-      1 (stream-export-v1 cfg params)
-      2 (throw (ex-info "not-implemented" {}))
-      3 (stream-export-v3 cfg params))))
+  (fn [_]
+    (let [version (or version 1)
+          body    (case (int version)
+                    1 (stream-export-v1 cfg params)
+                    2 (throw (ex-info "not-implemented" {}))
+                    3 (stream-export-v3 cfg params))]
+
+      {::yres/status 200
+       ::yres/headers {"content-type" "application/octet-stream"}
+       ::yres/body body})))
 
 ;; --- Command: import-binfile
 
 (defn- import-binfile
-  [{:keys [::db/pool] :as cfg} {:keys [profile-id project-id version name file]}]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
   (let [team   (teams/get-team pool
                                :profile-id profile-id
                                :project-id project-id)
@@ -99,14 +105,17 @@
                    (assoc ::bfc/name name)
                    (assoc ::bfc/input (:path file)))
 
+        ;; NOTE: the importation process performs some operations that are
+        ;; not very friendly with virtual threads, and for avoid
+        ;; unexpected blocking of other concurrent operations we dispatch
+        ;; that operation to a dedicated executor.
         result (case (int version)
-                 1 (bf.v1/import-files! cfg)
-                 3 (bf.v3/import-files! cfg))]
+                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
+                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
 
     (db/update! pool :project
-                {:modified-at (ct/now)}
-                {:id project-id}
-                {::db/return-keys false})
+                {:modified-at (dt/now)}
+                {:id project-id})
 
     result))
 
@@ -115,42 +124,20 @@
    [:name [:or [:string {:max 250}]
            [:map-of ::sm/uuid [:string {:max 250}]]]]
    [:project-id ::sm/uuid]
-   [:file-id {:optional true} ::sm/uuid]
    [:version {:optional true} ::sm/int]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::import-binfile
-  "Import a penpot file in a binary format. If `file-id` is provided,
-  an in-place import will be performed instead of creating a new file.
-
-  The in-place imports are only supported for binfile-v3 and when a
-  .penpot file only contains one penpot file.
-  "
+  "Import a penpot file in a binary format."
   {::doc/added "1.15"
-   ::doc/changes ["1.20" "Add file-id param for in-place import"
-                  "1.20" "Set default version to 3"]
-
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [version  (or version 3)
-        params   (-> params
-                     (assoc :profile-id profile-id)
-                     (assoc :version version))
-
-        cfg      (cond-> cfg
-                   (uuid? file-id)
-                   (assoc ::bfc/file-id file-id))
-
-        manifest (case (int version)
-                   1 nil
-                   3 (bf.v3/get-manifest (:path file)))]
-
+  (let [params (-> params
+                   (assoc :profile-id profile-id)
+                   (assoc :version (or version 1)))]
     (with-meta
       (sse/response (partial import-binfile cfg params))
-      {::audit/props {:file nil
-                      :file-id file-id
-                      :generated-by (:generated-by manifest)
-                      :referer (:referer manifest)}})))
+      {::audit/props {:file nil}})))

backend/src/app/rpc/commands/comments.clj

@@ -6,13 +6,11 @@
 
 (ns app.rpc.commands.comments
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uri :as uri]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -31,6 +29,7 @@
    [app.rpc.retry :as rtry]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [clojure.set :as set]
    [cuerdas.core :as str]))
 
@@ -164,16 +163,34 @@
 (def xf-decode-row
   (map decode-row))
 
+(def ^:private
+  sql:get-file
+  "SELECT f.id, f.modified_at, f.revn, f.features, f.name,
+          f.project_id, p.team_id, f.data,
+          f.data_ref_id, f.data_backend
+     FROM file as f
+    INNER JOIN project as p on (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
+
 (defn- get-file
   "A specialized version of get-file for comments module."
   [cfg file-id page-id]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-    (let [file (bfc/get-file cfg file-id)
-          data (get file :data)]
-      (-> file
-          (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
-          (assoc :page-id page-id)
-          (dissoc :data)))))
+  (let [file (db/exec-one! cfg [sql:get-file file-id])]
+    (when-not file
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "file not found"))
+
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+      (let [file (->> file
+                      (files/decode-row)
+                      (feat.fdata/resolve-file-data cfg))
+            data (get file :data)]
+        (-> file
+            (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
+            (assoc :page-id page-id)
+            (dissoc :data))))))
 
 ;; FIXME: rename
 (defn- get-comment-thread
@@ -205,7 +222,7 @@
 
 (defn upsert-comment-thread-status!
   ([conn profile-id thread-id]
-   (upsert-comment-thread-status! conn profile-id thread-id (ct/in-future "1s")))
+   (upsert-comment-thread-status! conn profile-id thread-id (dt/in-future "1s")))
   ([conn profile-id thread-id mod-at]
    (db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id mod-at mod-at])))
 
@@ -234,39 +251,34 @@
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
-(defn- get-comment-threads-sql
-  [where]
-  (str/ffmt
-   "SELECT DISTINCT ON (ct.id)
-           ct.*,
-           pf.fullname AS owner_fullname,
-           pf.email AS owner_email,
-           pf.photo_id AS owner_photo_id,
-           p.team_id AS team_id,
-           f.name AS file_name,
-           f.project_id AS project_id,
-           first_value(c.content) OVER w AS content,
-           (SELECT count(1)
-              FROM comment AS c
-             WHERE c.thread_id = ct.id) AS count_comments,
-           (SELECT count(1)
-              FROM comment AS c
-             WHERE c.thread_id = ct.id
-               AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
-      FROM comment_thread AS ct
-     INNER JOIN comment AS c ON (c.thread_id = ct.id)
-     INNER JOIN file AS f ON (f.id = ct.file_id)
-     INNER JOIN project AS p ON (p.id = f.project_id)
-      LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
-      LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
-     WHERE f.deleted_at IS NULL
-       AND p.deleted_at IS NULL
-       %1
-    WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)"
-   where))
+(def ^:private sql:comment-threads
+  "SELECT DISTINCT ON (ct.id)
+          ct.*,
+          pf.fullname AS owner_fullname,
+          pf.email AS owner_email,
+          pf.photo_id AS owner_photo_id,
+          p.team_id AS team_id,
+          f.name AS file_name,
+          f.project_id AS project_id,
+          first_value(c.content) OVER w AS content,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id) AS count_comments,
+          (SELECT count(1)
+             FROM comment AS c
+            WHERE c.thread_id = ct.id
+              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+     FROM comment_thread AS ct
+    INNER JOIN comment AS c ON (c.thread_id = ct.id)
+    INNER JOIN file AS f ON (f.id = ct.file_id)
+    INNER JOIN project AS p ON (p.id = f.project_id)
+     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
 
 (def ^:private sql:comment-threads-by-file-id
-  (get-comment-threads-sql "AND ct.file_id = ?"))
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
@@ -275,30 +287,7 @@
 
 ;; --- COMMAND: Get Unread Comment Threads
 
-(def ^:private sql:unread-all-comment-threads-by-team
-  (str "WITH threads AS ("
-       (get-comment-threads-sql "AND p.team_id = ?")
-       ")"
-       "SELECT t.* FROM threads AS t
-         WHERE t.count_unread_comments > 0"))
-
-(def ^:private sql:unread-partial-comment-threads-by-team
-  (str "WITH threads AS ("
-       (get-comment-threads-sql "AND p.team_id = ? AND (ct.owner_id = ? OR ? = ANY(ct.mentions))")
-       ")"
-       "SELECT t.* FROM threads AS t
-         WHERE t.count_unread_comments > 0"))
-
-(defn- get-unread-comment-threads
-  [cfg profile-id team-id]
-  (let [profile (-> (db/get cfg :profile {:id profile-id} ::db/remove-deleted false)
-                    (profile/decode-row))
-        notify  (or (-> profile :props :notifications :dashboard-comments) :all)
-        result  (case notify
-                  :all     (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
-                  :partial (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-                  [])]
-    (into [] xf-decode-row result)))
+(declare ^:private get-unread-comment-threads)
 
 (def ^:private
   schema:get-unread-comment-threads
@@ -309,8 +298,41 @@
   {::doc/added "1.15"
    ::sm/params schema:get-unread-comment-threads}
   [cfg {:keys [::rpc/profile-id team-id] :as params}]
-  (teams/check-read-permissions! cfg profile-id team-id)
-  (get-unread-comment-threads cfg profile-id team-id))
+  (db/run!
+   cfg
+   (fn [{:keys [::db/conn]}]
+     (teams/check-read-permissions! conn profile-id team-id)
+     (get-unread-comment-threads conn profile-id team-id))))
+
+(def sql:unread-all-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
+
+;; The partial configuration will retrieve only comments created by the user and
+;; threads that have a mention to the user.
+(def sql:unread-partial-comment-threads-by-team
+  (str "WITH threads AS (" sql:comment-threads ")"
+       "SELECT * FROM threads
+         WHERE count_unread_comments > 0
+           AND team_id = ?
+           AND (owner_id = ? OR ? = ANY(mentions))"))
+
+(defn- get-unread-comment-threads
+  [conn profile-id team-id]
+  (let [profile (-> (db/get conn :profile {:id profile-id})
+                    (profile/decode-row))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
+
+    (case notify
+      :all
+      (->> (db/exec! conn [sql:unread-all-comment-threads-by-team profile-id team-id])
+           (into [] xf-decode-row))
+
+      :partial
+      (->> (db/exec! conn [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+           (into [] xf-decode-row))
+
+      [])))
 
 ;; --- COMMAND: Get Single Comment Thread
 
@@ -321,17 +343,16 @@
    [:id ::sm/uuid]
    [:share-id {:optional true} [:maybe ::sm/uuid]]])
 
-(def ^:private sql:get-comment-thread
-  (get-comment-threads-sql "AND ct.file_id = ? AND ct.id = ?"))
-
 (sv/defmethod ::get-comment-thread
   {::doc/added "1.15"
    ::sm/params schema:get-comment-thread}
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
-                         (decode-row)))))
+                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
+                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                   (-> (db/exec-one! conn [sql profile-id id file-id])
+                       (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
 

backend/src/app/rpc/commands/demo.clj

@@ -7,16 +7,16 @@
 (ns app.rpc.commands.demo
   "A demo specific mutations."
   (:require
-   [app.auth :refer [derive-password]]
    [app.common.exceptions :as ex]
-   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.auth :as auth]
+   [app.rpc.commands.profile :as profile]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [buddy.core.codecs :as bc]
    [buddy.core.nonce :as bn]))
 
@@ -45,14 +45,15 @@
         params   {:email email
                   :fullname fullname
                   :is-active true
-                  :is-demo true
-                  :deleted-at (ct/in-future (cf/get-deletion-delay))
-                  :password (derive-password password)
-                  :props {}}
-        profile  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                                   (->> (auth/create-profile! conn params)
-                                        (auth/create-profile-rels! conn))))]
-    (with-meta {:email email
-                :password password}
-      {::audit/profile-id (:id profile)})))
+                  :deleted-at (dt/in-future (cf/get-deletion-delay))
+                  :password (profile/derive-password cfg password)
+                  :props {}}]
+
+
+    (let [profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                                    (->> (auth/create-profile! conn params)
+                                         (auth/create-profile-rels! conn))))]
+      (with-meta {:email email
+                  :password password}
+        {::audit/profile-id (:id profile)}))))
 

backend/src/app/rpc/commands/files.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.files
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
@@ -16,8 +15,6 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as-alias smdj]
-   [app.common.time :as ct]
-   [app.common.transit :as t]
    [app.common.types.components-list :as ctkl]
    [app.common.types.file :as ctf]
    [app.common.uri :as uri]
@@ -25,11 +22,9 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
-   [app.features.logical-deletion :as ldel]
+   [app.features.file-migrations :as feat.fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
-   [app.msgbus :as mbus]
-   [app.redis :as rds]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
@@ -40,8 +35,10 @@
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
 
 ;; --- FEATURES
 
@@ -53,13 +50,15 @@
 ;; --- HELPERS
 
 (def long-cache-duration
-  (ct/duration {:days 7}))
+  (dt/duration {:days 7}))
 
 (defn decode-row
-  [{:keys [features] :as row}]
+  [{:keys [data changes features] :as row}]
   (when row
     (cond-> row
-      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
+      features (assoc :features (db/decode-pgarray features #{}))
+      changes  (assoc :changes (blob/decode changes))
+      data     (assoc :data (blob/decode data)))))
 
 (defn check-version!
   [file]
@@ -77,16 +76,13 @@
 
 ;; --- FILE PERMISSIONS
 
-
 (def ^:private sql:file-permissions
   "select fpr.is_owner,
           fpr.is_admin,
           fpr.can_edit
      from file_profile_rel as fpr
-    inner join file as f on (f.id = fpr.file_id)
     where fpr.file_id = ?
       and fpr.profile_id = ?
-      and f.deleted_at is null
    union all
    select tpr.is_owner,
           tpr.is_admin,
@@ -96,7 +92,6 @@
     inner join file as f on (p.id = f.project_id)
     where f.id = ?
       and tpr.profile_id = ?
-      and f.deleted_at is null
    union all
    select ppr.is_owner,
           ppr.is_admin,
@@ -104,8 +99,7 @@
      from project_profile_rel as ppr
     inner join file as f on (f.project_id = ppr.project_id)
     where f.id = ?
-      and ppr.profile_id = ?
-      and f.deleted_at is null")
+      and ppr.profile_id = ?")
 
 (defn get-file-permissions
   [conn profile-id file-id]
@@ -191,15 +185,15 @@
    [:name [:string {:max 250}]]
    [:revn [::sm/int {:min 0}]]
    [:vern [::sm/int {:min 0}]]
-   [:modified-at ::ct/inst]
+   [:modified-at ::dt/instant]
    [:is-shared ::sm/boolean]
    [:project-id ::sm/uuid]
-   [:created-at ::ct/inst]
-   [:data {:optional true} ::sm/any]])
+   [:created-at ::dt/instant]
+   [:data {:optional true} :any]])
 
 (def schema:permissions-mixin
   [:map {:title "PermissionsMixin"}
-   [:permissions perms/schema:permissions]])
+   [:permissions ::perms/permissions]])
 
 (def schema:file-with-permissions
   [:merge {:title "FileWithPermissions"}
@@ -210,11 +204,92 @@
   schema:get-file
   [:map {:title "get-file"}
    [:features {:optional true} ::cfeat/features]
-   [:id ::sm/uuid]])
+   [:id ::sm/uuid]
+   [:project-id {:optional true} ::sm/uuid]])
+
+(defn- migrate-file
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} {:keys [read-only?]}]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
+            pmap/*tracked* (pmap/create-tracked)]
+    (let [;; For avoid unnecesary overhead of creating multiple pointers and
+          ;; handly internally with objects map in their worst case (when
+          ;; probably all shapes and all pointers will be readed in any
+          ;; case), we just realize/resolve them before applying the
+          ;; migration to the file
+          file (-> file
+                   (update :data feat.fdata/process-pointers deref)
+                   (update :data feat.fdata/process-objects (partial into {}))
+                   (fmg/migrate-file))]
+
+      (if (or read-only? (db/read-only? conn))
+        file
+        (let [;; When file is migrated, we break the rule of no perform
+              ;; mutations on get operations and update the file with all
+              ;; migrations applied
+              file (if (contains? (:features file) "fdata/objects-map")
+                     (feat.fdata/enable-objects-map file)
+                     file)
+              file (if (contains? (:features file) "fdata/pointer-map")
+                     (feat.fdata/enable-pointer-map file)
+                     file)]
+
+          (db/update! conn :file
+                      {:data (blob/encode (:data file))
+                       :version (:version file)
+                       :features (db/create-array conn "text" (:features file))}
+                      {:id id}
+                      {::db/return-keys false})
+
+          (when (contains? (:features file) "fdata/pointer-map")
+            (feat.fdata/persist-pointers! cfg id))
+
+          (feat.fmigr/upsert-migrations! conn file)
+          (feat.fmigr/resolve-applied-migrations cfg file))))))
+
+(defn get-file
+  [{:keys [::db/conn ::wrk/executor] :as cfg} id
+   & {:keys [project-id
+             migrate?
+             include-deleted?
+             lock-for-update?
+             preload-pointers?]
+      :or {include-deleted? false
+           lock-for-update? false
+           migrate? true
+           preload-pointers? false}
+      :as options}]
+
+  (assert (db/connection? conn) "expected cfg with valid connection")
+
+  (let [params (merge {:id id}
+                      (when (some? project-id)
+                        {:project-id project-id}))
+        file   (->> (db/get conn :file params
+                            {::db/check-deleted (not include-deleted?)
+                             ::db/remove-deleted (not include-deleted?)
+                             ::sql/for-update lock-for-update?})
+                    (feat.fmigr/resolve-applied-migrations cfg)
+                    (feat.fdata/resolve-file-data cfg))
+
+        ;; NOTE: we perform the file decoding in a separate thread
+        ;; because it has heavy and synchronous operations for
+        ;; decoding file body that are not very friendly with virtual
+        ;; threads.
+        file   (px/invoke! executor #(decode-row file))
+
+        file   (if (and migrate? (fmg/need-migration? file))
+                 (migrate-file cfg file options)
+                 file)]
+
+    (if preload-pointers?
+      (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+        (update file :data feat.fdata/process-pointers deref))
+
+      file)))
 
 (defn get-minimal-file
   [cfg id & {:as opts}]
-  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern])]
+  (let [opts (assoc opts ::sql/columns [:id :modified-at :deleted-at :revn :vern :data-ref-id :data-backend])]
     (db/get cfg :file {:id id} opts)))
 
 (defn- get-minimal-file-with-perms
@@ -226,7 +301,7 @@
 (defn get-file-etag
   [{:keys [::rpc/profile-id]} {:keys [modified-at revn vern permissions]}]
   (str profile-id "/" revn "/" vern "/" (hash fmg/available-migrations) "/"
-       (ct/format-inst modified-at :iso)
+       (dt/format-instant modified-at :iso)
        "/"
        (uri/map->query-string permissions)))
 
@@ -254,32 +329,23 @@
                                :project-id project-id
                                :file-id id)
 
-          file (-> (bfc/get-file cfg id
-                                 :project-id project-id)
+          file (-> (get-file cfg id :project-id project-id)
                    (assoc :permissions perms)
+                   (assoc :team-id (:id team))
                    (check-version!))]
 
       (-> (cfeat/get-team-enabled-features cf/flags team)
           (cfeat/check-client-features! (:features params))
           (cfeat/check-file-features! (:features file)))
 
-      (as-> file file
-        ;; This operation is needed for backward comapatibility with
-        ;; frontends that does not support pointer-map resolution
-        ;; mechanism; this just resolves the pointers on backend and
-        ;; return a complete file
-        (if (and (contains? (:features file) "fdata/pointer-map")
-                 (not (contains? (:features params) "fdata/pointer-map")))
-          (feat.fdata/realize-pointers cfg file)
-          file)
-
-        ;; This operation is needed for backward comapatibility with
-        ;; frontends that does not support objects-map mechanism; this
-        ;; just converts all objects map instaces to plain maps
-        (if (and (contains? (:features file) "fdata/objects-map")
-                 (not (contains? (:features params) "fdata/objects-map")))
-          (feat.fdata/realize-objects cfg file)
-          file)))))
+      ;; This operation is needed for backward comapatibility with frontends that
+      ;; does not support pointer-map resolution mechanism; this just resolves the
+      ;; pointers on backend and return a complete file.
+      (if (and (contains? (:features file) "fdata/pointer-map")
+               (not (contains? (:features params) "fdata/pointer-map")))
+        (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+          (update file :data feat.fdata/process-pointers deref))
+        file))))
 
 ;; --- COMMAND QUERY: get-file-fragment (by id)
 
@@ -287,8 +353,8 @@
   [:map {:title "FileFragment"}
    [:id ::sm/uuid]
    [:file-id ::sm/uuid]
-   [:created-at ::ct/inst]
-   [:content ::sm/any]])
+   [:created-at ::dt/instant]
+   [:content any?]])
 
 (def schema:get-file-fragment
   [:map {:title "get-file-fragment"}
@@ -298,8 +364,10 @@
 
 (defn- get-file-fragment
   [cfg file-id fragment-id]
-  (some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"})
-          (update :data blob/decode)))
+  (let [resolve-file-data (partial feat.fdata/resolve-file-data cfg)]
+    (some-> (db/get cfg :file-data-fragment {:file-id file-id :id fragment-id})
+            (resolve-file-data)
+            (update :data blob/decode))))
 
 (sv/defmethod ::get-file-fragment
   "Retrieve a file fragment by its ID. Only authenticated users."
@@ -389,42 +457,8 @@
     (:has-libraries row)))
 
 
-;; --- COMMAND QUERY: get-library-usage
-
-
-(declare get-library-usage)
-
-(def schema:get-library-usage
-  [:map {:title "get-library-usage"}
-   [:file-id ::sm/uuid]])
-:sample
-(sv/defmethod ::get-library-usage
-  "Gets the number of files that use the specified library."
-  {::doc/added "2.10.0"
-   ::sm/params schema:get-library-usage
-   ::sm/result ::sm/int}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
-  (dm/with-open [conn (db/open pool)]
-    (check-read-permissions! pool profile-id file-id)
-    (get-library-usage conn file-id)))
-
-(def ^:private sql:get-library-usage
-  "SELECT COUNT(*) AS used
-     FROM file_library_rel AS flr
-     JOIN file AS fl ON (flr.library_file_id = fl.id)
-    WHERE flr.library_file_id = ?::uuid
-      AND (fl.deleted_at IS NULL OR
-           fl.deleted_at > now())")
-
-(defn- get-library-usage
-  [conn file-id]
-  (let [row (db/exec-one! conn [sql:get-library-usage file-id])]
-    {:used-in (:used row)}))
-
-
 ;; --- QUERY COMMAND: get-page
 
-
 (defn- prune-objects
   "Given the page data and the object-id returns the page data with all
   other not needed objects removed from the `:objects` data
@@ -458,7 +492,7 @@
 
   (let [perms (get-permissions conn profile-id file-id share-id)
 
-        file  (bfc/get-file cfg file-id :read-only? true)
+        file  (get-file cfg file-id :read-only? true)
 
         proj  (db/get conn :project {:id (:project-id file)})
 
@@ -514,136 +548,59 @@
 
 ;; --- COMMAND QUERY: get-team-shared-files
 
-(defn- get-components-with-variants
-  "Return a set with all the variant-ids, and a list of components, but
-  with only one component by variant.
-
-  Returns a vector of unique components and a set of all variant ids"
-  [fdata]
-  (loop [variant-ids #{}
-         components' []
-         components  (ctkl/components-seq fdata)]
-    (if-let [{:keys [variant-id] :as component} (first components)]
-      (cond
-        (nil? variant-id)
-        (recur variant-ids
-               (conj components' component)
-               (rest components))
-
-        (contains? variant-ids variant-id)
-        (recur variant-ids
-               components'
-               (rest components))
-
-        :else
-        (recur (conj variant-ids variant-id)
-               (conj components' component)
-               (rest components)))
-
-      [(d/index-by :id components') variant-ids])))
-
-(defn- sample-assets
-  [assets limit]
-  (let [assets (into [] (map val) assets)]
-    {:count (count assets)
-     :sample (->> assets
-                  (sort-by #(str/lower (:name %)))
-                  (into [] (take limit)))}))
-
-(defn- calculate-library-summary
-  "Calculate the file library summary (counters and samples)"
-  [{:keys [data] :as file}]
-  (let [load-objects
-        (fn [sample]
-          (mapv #(ctf/load-component-objects data %) sample))
-
-        [components variant-ids]
-        (get-components-with-variants data)
-
-        components-sample
-        (-> (sample-assets components 4)
-            (update :sample load-objects))]
-
-    {:components components-sample
-     :variants {:count (count variant-ids)}
-     :colors (sample-assets (:colors data) 3)
-     :typographies (sample-assets (:typographies data) 3)}))
-
-(def ^:private file-summary-cache-key-ttl
-  (ct/duration {:days 30}))
-
-(def file-summary-cache-key-prefix
-  "penpot.library-summary.")
-
-(defn- get-file-with-summary
-  "Get a file without data with a summary of its local library content"
-  [cfg id]
-  (let [get-from-cache
-        (fn [{:keys [::rds/conn]} cache-key]
-          (when-let [result (rds/get conn cache-key)]
-            (let [file    (bfc/get-file cfg id :load-data? false)
-                  summary (t/decode-str result)]
-              (-> (assoc file :library-summary summary)
-                  (dissoc :data)))))
-
-        calculate-from-db
-        (fn []
-          (let [file   (bfc/get-file cfg id)
-                result (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-                         (calculate-library-summary file))]
-            (-> file
-                (assoc :library-summary result)
-                (dissoc :legacy-data)
-                (dissoc :data))))
-
-        persist-to-cache
-        (fn [{:keys [::rds/conn]} data cache-key]
-          (rds/set conn cache-key (t/encode-str data)
-                   (rds/build-set-args {:ex file-summary-cache-key-ttl})))]
-
-    (if (contains? cf/flags :redis-cache)
-      (let [cache-key (str file-summary-cache-key-prefix id)]
-        (or (rds/run! cfg get-from-cache cache-key)
-            (let [file (calculate-from-db)]
-              (rds/run! cfg persist-to-cache (:library-summary file) cache-key)
-              file)))
-      (calculate-from-db))))
-
 (def ^:private sql:team-shared-files
-  "WITH file_library_agg AS (
-      SELECT flr.file_id,
-             coalesce(array_agg(flr.library_file_id) filter (WHERE flr.library_file_id IS NOT NULL), '{}') AS library_file_ids
-        FROM file_library_rel flr
-       GROUP BY flr.file_id
-   )
+  "select f.id,
+          f.revn,
+          f.vern,
+          f.data,
+          f.project_id,
+          f.created_at,
+          f.modified_at,
+          f.name,
+          f.is_shared,
+          ft.media_id,
+          p.team_id
+     from file as f
+    inner join project as p on (p.id = f.project_id)
+     left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn and ft.deleted_at is null)
+    where f.is_shared = true
+      and f.deleted_at is null
+      and p.deleted_at is null
+      and p.team_id = ?
+    order by f.modified_at desc")
 
-   SELECT f.id,
-          fla.library_file_ids,
-          ft.media_id AS thumbnail_id
-     FROM file AS f
-    INNER JOIN project AS p ON (p.id = f.project_id)
-     LEFT JOIN file_thumbnail AS ft ON (ft.file_id = f.id AND ft.revn = f.revn AND ft.deleted_at IS NULL)
-     LEFT JOIN file_library_agg AS fla ON (fla.file_id = f.id)
-    WHERE f.is_shared = true
-      AND f.deleted_at IS NULL
-      AND p.deleted_at IS NULL
-      AND p.team_id = ?
-    ORDER BY f.modified_at DESC")
+(defn- get-library-summary
+  [cfg {:keys [id data] :as file}]
+  (letfn [(assets-sample [assets limit]
+            (let [sorted-assets (->> (vals assets)
+                                     (sort-by #(str/lower (:name %))))]
+              {:count (count sorted-assets)
+               :sample (into [] (take limit sorted-assets))}))]
+
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+      (let [load-objects      (fn [component]
+                                (ctf/load-component-objects data component))
+            components-sample (-> (assets-sample (ctkl/components data) 4)
+                                  (update :sample #(mapv load-objects %)))]
+        {:components components-sample
+         :media (assets-sample (:media data) 3)
+         :colors (assets-sample (:colors data) 3)
+         :typographies (assets-sample (:typographies data) 3)}))))
 
 (defn- get-team-shared-files
   [{:keys [::db/conn] :as cfg} {:keys [team-id profile-id]}]
   (teams/check-read-permissions! conn profile-id team-id)
-
-  (let [process-row
-        (fn [{:keys [id library-file-ids]}]
-          (let [file (get-file-with-summary cfg id)]
-            (assoc file :library-file-ids (db/decode-pgarray library-file-ids #{}))))
-
-        xform
-        (map process-row)]
-
-    (->> (db/plan conn [sql:team-shared-files team-id] {:fetch-size 1})
-         (transduce xform conj #{}))))
+  (->> (db/exec! conn [sql:team-shared-files team-id])
+       (into #{} (comp
+                  (map decode-row)
+                  (map (fn [row]
+                         (if-let [media-id (:media-id row)]
+                           (-> row
+                               (dissoc :media-id)
+                               (assoc :thumbnail-id media-id))
+                           (dissoc row :media-id))))
+                  (map #(assoc % :library-summary (get-library-summary cfg %)))
+                  (map #(dissoc % :data))))))
 
 (def ^:private schema:get-team-shared-files
   [:map {:title "get-team-shared-files"}
@@ -656,30 +613,46 @@
   [cfg {:keys [::rpc/profile-id] :as params}]
   (db/tx-run! cfg get-team-shared-files (assoc params :profile-id profile-id)))
 
-;; --- COMMAND QUERY: get-file-summary
-
-(defn- get-file-summary
-  [cfg id]
-  (let [file (get-file-with-summary cfg id)]
-    (-> (:library-summary file)
-        (assoc :name (:name file)))))
-
-(def ^:private
-  schema:get-file-summary
-  [:map {:title "get-file-summary"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::get-file-summary
-  "Retrieve a file summary by its ID. Only authenticated users."
-  {::doc/added "1.20"
-   ::sm/params schema:get-file-summary}
-  [cfg {:keys [::rpc/profile-id id] :as params}]
-  (check-read-permissions! cfg profile-id id)
-  (get-file-summary cfg id))
-
-
 ;; --- COMMAND QUERY: get-file-libraries
 
+(def ^:private sql:get-file-libraries
+  "WITH RECURSIVE libs AS (
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+      WHERE flr.file_id = ?::uuid
+    UNION
+     SELECT fl.*, flr.synced_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+       JOIN libs AS l ON (flr.file_id = l.id)
+   )
+   SELECT l.id,
+          l.features,
+          l.project_id,
+          p.team_id,
+          l.created_at,
+          l.modified_at,
+          l.deleted_at,
+          l.name,
+          l.revn,
+          l.vern,
+          l.synced_at,
+          l.is_shared
+     FROM libs AS l
+    INNER JOIN project AS p ON (p.id = l.project_id)
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+
+(defn get-file-libraries
+  [conn file-id]
+  (into []
+        (comp
+         ;; FIXME: :is-indirect set to false to all rows looks
+         ;; completly useless
+         (map #(assoc % :is-indirect false))
+         (map decode-row))
+        (db/exec! conn [sql:get-file-libraries file-id])))
+
 (def ^:private schema:get-file-libraries
   [:map {:title "get-file-libraries"}
    [:file-id ::sm/uuid]])
@@ -691,7 +664,7 @@
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
   (dm/with-open [conn (db/open pool)]
     (check-read-permissions! conn profile-id file-id)
-    (bfc/get-file-libraries conn file-id)))
+    (get-file-libraries conn file-id)))
 
 
 ;; --- COMMAND QUERY: Files that use this File library
@@ -765,14 +738,47 @@
     (teams/check-read-permissions! conn profile-id team-id)
     (get-team-recent-files conn team-id)))
 
-;; --- COMMAND QUERY: get-file-info
 
+;; --- COMMAND QUERY: get-file-summary
+
+(defn- get-file-summary
+  [{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}]
+  (check-read-permissions! conn profile-id id)
+  (let [team (teams/get-team conn
+                             :profile-id profile-id
+                             :project-id project-id
+                             :file-id id)
+
+        file (get-file cfg id
+                       :project-id project-id
+                       :read-only? true)]
+
+    (-> (cfeat/get-team-enabled-features cf/flags team)
+        (cfeat/check-client-features! (:features params))
+        (cfeat/check-file-features! (:features file)))
+
+    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+      {:name             (:name file)
+       :components-count (count (ctkl/components-seq (:data file)))
+       :graphics-count   (count (get-in file [:data :media] []))
+       :colors-count     (count (get-in file [:data :colors] []))
+       :typography-count (count (get-in file [:data :typographies] []))})))
+
+(sv/defmethod ::get-file-summary
+  "Retrieve a file summary by its ID. Only authenticated users."
+  {::doc/added "1.20"
+   ::sm/params schema:get-file}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (db/tx-run! cfg get-file-summary (assoc params :profile-id profile-id)))
+
+
+;; --- COMMAND QUERY: get-file-info
 
 (defn- get-file-info
   [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
-  (db/get conn :file
-          {:id id}
-          {::sql/columns [:id :deleted-at]}))
+  (db/get* conn :file
+           {:id id}
+           {::sql/columns [:id]}))
 
 (sv/defmethod ::get-file-info
   "Retrieve minimal file info by its ID."
@@ -792,7 +798,7 @@
   [conn {:keys [id name]}]
   (db/update! conn :file
               {:name name
-               :modified-at (ct/now)}
+               :modified-at (dt/now)}
               {:id id}
               {::db/return-keys true}))
 
@@ -805,8 +811,8 @@
     [:id ::sm/uuid]
     [:project-id ::sm/uuid]
     [:name [:string {:max 250}]]
-    [:created-at ::ct/inst]
-    [:modified-at ::ct/inst]]
+    [:created-at ::dt/instant]
+    [:modified-at ::dt/instant]]
 
    ::sm/params
    [:map {:title "RenameFileParams"}
@@ -817,8 +823,8 @@
    [:map {:title "SimplifiedFile"}
     [:id ::sm/uuid]
     [:name [:string {:max 250}]]
-    [:created-at ::ct/inst]
-    [:modified-at ::ct/inst]]
+    [:created-at ::dt/instant]
+    [:modified-at ::dt/instant]]
 
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
@@ -832,7 +838,7 @@
 
 ;; --- MUTATION COMMAND: set-file-shared
 
-(def ^:private sql:get-referenced-files
+(def sql:get-referenced-files
   "SELECT f.id
      FROM file_library_rel AS flr
     INNER JOIN file AS f ON (f.id = flr.file_id)
@@ -843,51 +849,56 @@
 (defn- absorb-library-by-file!
   [cfg ldata file-id]
 
-  (assert (db/connection-map? cfg)
-          "expected cfg with valid connection")
+  (dm/assert!
+   "expected cfg with valid connection"
+   (db/connection-map? cfg))
 
   (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)
             pmap/*tracked* (pmap/create-tracked)]
-    (let [file (-> (bfc/get-file cfg file-id
-                                 :include-deleted? true
-                                 :lock-for-update? true)
+    (let [file (-> (get-file cfg file-id
+                             :include-deleted? true
+                             :lock-for-update? true)
                    (update :data ctf/absorb-assets ldata))]
 
       (l/trc :hint "library absorbed"
              :library-id (str (:id ldata))
              :file-id (str file-id))
 
-      (bfc/update-file! cfg {:id file-id
-                             :migrations (:migrations file)
-                             :revn (inc (:revn file))
-                             :data (:data file)
-                             :modified-at (ct/now)
-                             :has-media-trimmed false}))))
+      (db/update! cfg :file
+                  {:revn (inc (:revn file))
+                   :data (blob/encode (:data file))
+                   :modified-at (dt/now)
+                   :has-media-trimmed false}
+                  {:id file-id})
+
+      (feat.fdata/persist-pointers! cfg file-id))))
 
 (defn- absorb-library
   "Find all files using a shared library, and absorb all library assets
   into the file local libraries"
-  [cfg {:keys [id data] :as library}]
+  [cfg {:keys [id] :as library}]
 
-  (assert (db/connection-map? cfg)
-          "expected cfg with valid connection")
+  (dm/assert!
+   "expected cfg with valid connection"
+   (db/connection-map? cfg))
 
-  (let [ids (->> (db/exec! cfg [sql:get-referenced-files id])
-                 (sequence bfc/xf-map-id))]
+  (let [ldata (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+                (-> library :data (feat.fdata/process-pointers deref)))
+        ids   (->> (db/exec! cfg [sql:get-referenced-files id])
+                   (map :id))]
 
     (l/trc :hint "absorbing library"
            :library-id (str id)
            :files (str/join "," (map str ids)))
 
-    (run! (partial absorb-library-by-file! cfg data) ids)
+    (run! (partial absorb-library-by-file! cfg ldata) ids)
     library))
 
 (defn absorb-library!
   [{:keys [::db/conn] :as cfg} id]
-  (let [file (-> (bfc/get-file cfg id
-                               :realize? true
-                               :lock-for-update? true
-                               :include-deleted? true)
+  (let [file (-> (get-file cfg id
+                           :lock-for-update? true
+                           :include-deleted? true)
                  (check-version!))
 
         proj (db/get* conn :project {:id (:project-id file)}
@@ -917,7 +928,7 @@
                  (db/delete! conn :file-library-rel {:library-file-id id})
                  (db/update! conn :file
                              {:is-shared false
-                              :modified-at (ct/now)}
+                              :modified-at (dt/now)}
                              {:id id})
                  (select-keys file [:id :name :is-shared]))
 
@@ -926,7 +937,7 @@
                (let [file (assoc file :is-shared true)]
                  (db/update! conn :file
                              {:is-shared true
-                              :modified-at (ct/now)}
+                              :modified-at (dt/now)}
                              {:id id})
                  file)
 
@@ -959,13 +970,12 @@
 ;; --- MUTATION COMMAND: delete-file
 
 (defn- mark-file-deleted
-  [conn team file-id]
-  (let [delay (ldel/get-deletion-delay team)
-        file  (db/update! conn :file
-                          {:deleted-at (ct/in-future delay)}
-                          {:id file-id}
-                          {::db/return-keys [:id :name :is-shared :deleted-at
-                                             :project-id :created-at :modified-at]})]
+  [conn file-id]
+  (let [file (db/update! conn :file
+                         {:deleted-at (dt/now)}
+                         {:id file-id}
+                         {::db/return-keys [:id :name :is-shared :deleted-at
+                                            :project-id :created-at :modified-at]})]
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :file
@@ -981,18 +991,7 @@
 (defn- delete-file
   [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
   (check-edition-permissions! conn profile-id id)
-  (let [team (teams/get-team conn
-                             :profile-id profile-id
-                             :file-id id)
-        file (mark-file-deleted conn team id)
-        msgbus (::mbus/msgbus cfg)]
-
-    (mbus/pub! msgbus
-               :topic id
-               :message {:type :file-deleted
-                         :file-id id
-                         :profile-id profile-id})
-
+  (let [file (mark-file-deleted conn id)]
     (rph/with-meta (rph/wrap)
       {::audit/props {:project-id (:project-id file)
                       :name (:name file)
@@ -1024,7 +1023,6 @@
    [:library-id ::sm/uuid]])
 
 (sv/defmethod ::link-file-to-library
-  "Link a file to a library. Returns the recursive list of libraries used by that library"
   {::doc/added "1.17"
    ::webhooks/event? true
    ::sm/params schema:link-file-to-library}
@@ -1038,8 +1036,7 @@
               (fn [{:keys [::db/conn]}]
                 (check-edition-permissions! conn profile-id file-id)
                 (check-edition-permissions! conn profile-id library-id)
-                (link-file-to-library conn params)
-                (bfc/get-libraries cfg [library-id]))))
+                (link-file-to-library conn params))))
 
 ;; --- MUTATION COMMAND: unlink-file-from-library
 
@@ -1069,7 +1066,7 @@
 (defn update-sync
   [conn {:keys [file-id library-id] :as params}]
   (db/update! conn :file-library-rel
-              {:synced-at (ct/now)}
+              {:synced-at (dt/now)}
               {:file-id file-id
                :library-file-id library-id}
               {::db/return-keys true}))
@@ -1094,14 +1091,14 @@
   [conn {:keys [file-id date] :as params}]
   (db/update! conn :file
               {:ignore-sync-until date
-               :modified-at (ct/now)}
+               :modified-at (dt/now)}
               {:id file-id}
               {::db/return-keys true}))
 
 (def ^:private schema:ignore-file-library-sync-status
   [:map {:title "ignore-file-library-sync-status"}
    [:file-id ::sm/uuid]
-   [:date ::ct/inst]])
+   [:date ::dt/instant]])
 
 ;; TODO: improve naming
 (sv/defmethod ::ignore-file-library-sync-status

backend/src/app/rpc/commands/files_create.clj

@@ -7,10 +7,9 @@
 (ns app.rpc.commands.files-create
   (:require
    [app.binfile.common :as bfc]
+   [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
-   [app.common.files.migrations :as fmg]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.config :as cf]
    [app.db :as db]
@@ -24,13 +23,13 @@
    [app.rpc.quotes :as quotes]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [clojure.set :as set]))
 
 (defn create-file-role!
   [conn {:keys [file-id profile-id role]}]
   (let [params {:file-id file-id
                 :profile-id profile-id}]
-
     (->> (perms/assign-role-flags params role)
          (db/insert! conn :file-profile-rel))))
 
@@ -42,34 +41,34 @@
     :or {is-shared false revn 0 create-page true}
     :as params}]
 
-  (assert (db/connection? conn) "expected a valid connection")
+  (dm/assert!
+   "expected a valid connection"
+   (db/connection? conn))
 
   (binding [pmap/*tracked* (pmap/create-tracked)
             cfeat/*current* features]
-
     (let [file (ctf/make-file {:id id
                                :project-id project-id
                                :name name
                                :revn revn
                                :is-shared is-shared
                                :features features
-                               :migrations fmg/available-migrations
                                :ignore-sync-until ignore-sync-until
-                               :created-at modified-at
+                               :modified-at modified-at
                                :deleted-at deleted-at}
                               {:create-page create-page
-                               :page-id page-id})]
-
-      (bfc/insert-file! cfg file)
+                               :page-id page-id})
+          file (-> (bfc/insert-file! cfg file)
+                   (bfc/decode-row))]
 
       (->> (assoc params :file-id (:id file) :role :owner)
            (create-file-role! conn))
 
       (db/update! conn :project
-                  {:modified-at (ct/now)}
+                  {:modified-at (dt/now)}
                   {:id project-id})
 
-      (bfc/get-file cfg (:id file)))))
+      file)))
 
 (def ^:private schema:create-file
   [:map {:title "create-file"}
@@ -115,15 +114,14 @@
     ;; FIXME: IMPORTANT: this code can have race conditions, because
     ;; we have no locks for updating team so, creating two files
     ;; concurrently can lead to lost team features updating
+
     (when-let [features (-> features
                             (set/difference (:features team))
                             (set/difference cfeat/no-team-inheritable-features)
                             (not-empty))]
-      (let [features (-> features
-                         (set/union (:features team))
-                         (set/difference cfeat/no-team-inheritable-features)
-                         (into-array))]
-
+      (let [features (->> features
+                          (set/union (:features team))
+                          (db/create-array conn "text"))]
         (db/update! conn :team
                     {:features features}
                     {:id (:id team)}

backend/src/app/rpc/commands/files_snapshot.clj

@@ -8,20 +8,43 @@
   (:require
    [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.file-snapshots :as fsnap]
-   [app.features.logical-deletion :as ldel]
+   [app.features.fdata :as feat.fdata]
    [app.main :as-alias main]
    [app.msgbus :as mbus]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
-   [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.quotes :as quotes]
-   [app.util.services :as sv]))
+   [app.storage :as sto]
+   [app.util.blob :as blob]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [cuerdas.core :as str]))
+
+(def sql:get-file-snapshots
+  "WITH changes AS (
+      SELECT id, label, revn, created_at, created_by, profile_id
+        FROM file_change
+       WHERE file_id = ?
+         AND data IS NOT NULL
+         AND (deleted_at IS NULL OR deleted_at > now())
+   ), versions AS (
+      (SELECT * FROM changes WHERE created_by = 'system' LIMIT 1000)
+      UNION ALL
+      (SELECT * FROM changes WHERE created_by != 'system' LIMIT 1000)
+   )
+   SELECT * FROM versions
+    ORDER BY created_at DESC;")
+
+(defn get-file-snapshots
+  [conn file-id]
+  (db/exec! conn [sql:get-file-snapshots file-id]))
 
 (def ^:private schema:get-file-snapshots
   [:map {:title "get-file-snapshots"}
@@ -33,7 +56,72 @@
   [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-read-permissions! conn profile-id file-id)
-                 (fsnap/get-visible-snapshots conn file-id))))
+                 (get-file-snapshots conn file-id))))
+
+(defn- generate-snapshot-label
+  []
+  (let [ts (-> (dt/now)
+               (dt/format-instant)
+               (str/replace #"[T:\.]" "-")
+               (str/rtrim "Z"))]
+    (str "snapshot-" ts)))
+
+(defn create-file-snapshot!
+  [cfg file & {:keys [label created-by deleted-at profile-id]
+               :or {deleted-at :default
+                    created-by :system}}]
+
+  (assert (#{:system :user :admin} created-by)
+          "expected valid keyword for created-by")
+
+  (let [conn
+        (db/get-connection cfg)
+
+        created-by
+        (name created-by)
+
+        deleted-at
+        (cond
+          (= deleted-at :default)
+          (dt/plus (dt/now) (cf/get-deletion-delay))
+
+          (dt/instant? deleted-at)
+          deleted-at
+
+          :else
+          nil)
+
+        label
+        (or label (generate-snapshot-label))
+
+        snapshot-id
+        (uuid/next)
+
+        data
+        (blob/encode (:data file))
+
+        features
+        (db/encode-pgarray (:features file) conn "text")]
+
+    (l/debug :hint "creating file snapshot"
+             :file-id (str (:id file))
+             :id (str snapshot-id)
+             :label label)
+
+    (db/insert! cfg :file-change
+                {:id snapshot-id
+                 :revn (:revn file)
+                 :data data
+                 :version (:version file)
+                 :features features
+                 :profile-id profile-id
+                 :file-id (:id file)
+                 :label label
+                 :deleted-at deleted-at
+                 :created-by created-by}
+                {::db/return-keys false})
+
+    {:id snapshot-id :label label}))
 
 (def ^:private schema:create-file-snapshot
   [:map
@@ -46,7 +134,7 @@
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id file-id label]}]
   (files/check-edition-permissions! conn profile-id file-id)
-  (let [file    (bfc/get-file cfg file-id :realize? true)
+  (let [file    (bfc/get-file cfg file-id)
         project (db/get-by-id cfg :project (:project-id file))]
 
     (-> cfg
@@ -57,10 +145,82 @@
         (quotes/check! {::quotes/id ::quotes/snapshots-per-file}
                        {::quotes/id ::quotes/snapshots-per-team}))
 
-    (fsnap/create! cfg file
-                   {:label label
-                    :profile-id profile-id
-                    :created-by "user"})))
+    (create-file-snapshot! cfg file
+                           {:label label
+                            :profile-id profile-id
+                            :created-by :user})))
+
+(defn restore-file-snapshot!
+  [{:keys [::db/conn ::mbus/msgbus] :as cfg} file-id snapshot-id]
+  (let [storage  (sto/resolve cfg {::db/reuse-conn true})
+        file     (files/get-minimal-file conn file-id {::db/for-update true})
+        vern     (rand-int Integer/MAX_VALUE)
+        snapshot (some->> (db/get* conn :file-change
+                                   {:file-id file-id
+                                    :id snapshot-id}
+                                   {::db/for-share true})
+                          (feat.fdata/resolve-file-data cfg))]
+
+    (when-not snapshot
+      (ex/raise :type :not-found
+                :code :snapshot-not-found
+                :hint "unable to find snapshot with the provided label"
+                :snapshot-id snapshot-id
+                :file-id file-id))
+
+    (when-not (:data snapshot)
+      (ex/raise :type :validation
+                :code :snapshot-without-data
+                :hint "snapshot has no data"
+                :label (:label snapshot)
+                :file-id file-id))
+
+    (l/dbg :hint "restoring snapshot"
+           :file-id (str file-id)
+           :label (:label snapshot)
+           :snapshot-id (str (:id snapshot)))
+
+    ;; If the file was already offloaded, on restring the snapshot
+    ;; we are going to replace the file data, so we need to touch
+    ;; the old referenced storage object and avoid possible leaks
+    (when (feat.fdata/offloaded? file)
+      (sto/touch-object! storage (:data-ref-id file)))
+
+    (db/update! conn :file
+                {:data (:data snapshot)
+                 :revn (inc (:revn file))
+                 :vern vern
+                 :version (:version snapshot)
+                 :data-backend nil
+                 :data-ref-id nil
+                 :has-media-trimmed false
+                 :features (:features snapshot)}
+                {:id file-id})
+
+    ;; clean object thumbnails
+    (let [sql (str "update file_tagged_object_thumbnail "
+                   "   set deleted_at = now() "
+                   " where file_id=? returning media_id")
+          res (db/exec! conn [sql file-id])]
+      (doseq [media-id (into #{} (keep :media-id) res)]
+        (sto/touch-object! storage media-id)))
+
+    ;; clean file thumbnails
+    (let [sql (str "update file_thumbnail "
+                   "   set deleted_at = now() "
+                   " where file_id=? returning media_id")
+          res (db/exec! conn [sql file-id])]
+      (doseq [media-id (into #{} (keep :media-id) res)]
+        (sto/touch-object! storage media-id)))
+
+    ;; Send to the clients a notification to reload the file
+    (mbus/pub! msgbus
+               :topic (:id file)
+               :message {:type :file-restore
+                         :file-id (:id file)
+                         :vern vern})
+    {:id (:id snapshot)
+     :label (:label snapshot)}))
 
 (def ^:private schema:restore-file-snapshot
   [:map {:title "restore-file-snapshot"}
@@ -69,152 +229,75 @@
 
 (sv/defmethod ::restore-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:restore-file-snapshot
-   ::db/transaction true}
-  [{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id file-id id] :as params}]
-  (files/check-edition-permissions! conn profile-id file-id)
-  (let [file  (bfc/get-file cfg file-id)
-        team  (teams/get-team conn
-                              :profile-id profile-id
-                              :file-id file-id)
-        delay (ldel/get-deletion-delay team)]
-
-    (fsnap/create! cfg file
-                   {:profile-id profile-id
-                    :deleted-at (ct/in-future delay)
-                    :created-by "system"})
-
-    (let [vern (fsnap/restore! cfg file-id id)]
-      ;; Send to the clients a notification to reload the file
-      (mbus/pub! msgbus
-                 :topic (:id file)
-                 :message {:type :file-restore
-                           :file-id (:id file)
-                           :vern vern})
-      nil)))
+   ::sm/params schema:restore-file-snapshot}
+  [cfg {:keys [::rpc/profile-id file-id id] :as params}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (files/check-edition-permissions! conn profile-id file-id)
+                (let [file (bfc/get-file cfg file-id)]
+                  (create-file-snapshot! cfg file
+                                         {:profile-id profile-id
+                                          :created-by :system})
+                  (restore-file-snapshot! cfg file-id id)))))
 
 (def ^:private schema:update-file-snapshot
   [:map {:title "update-file-snapshot"}
    [:id ::sm/uuid]
    [:label ::sm/text]])
 
+(defn- update-file-snapshot!
+  [conn snapshot-id label]
+  (-> (db/update! conn :file-change
+                  {:label label
+                   :created-by "user"
+                   :deleted-at nil}
+                  {:id snapshot-id}
+                  {::db/return-keys true})
+      (dissoc :data :features)))
+
+(defn- get-snapshot
+  "Get a minimal snapshot from database and lock for update"
+  [conn id]
+  (db/get conn :file-change
+          {:id id}
+          {::sql/columns [:id :file-id :created-by :deleted-at]
+           ::db/for-update true}))
+
 (sv/defmethod ::update-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:update-file-snapshot
-   ::db/transaction true}
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id label]}]
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
-    (fsnap/update! conn (assoc snapshot :label label))))
+   ::sm/params schema:update-file-snapshot}
+  [cfg {:keys [::rpc/profile-id id label]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn]}]
+                (let [snapshot (get-snapshot conn id)]
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+                  (update-file-snapshot! conn id label)))))
 
 (def ^:private schema:remove-file-snapshot
   [:map {:title "remove-file-snapshot"}
    [:id ::sm/uuid]])
 
+(defn- delete-file-snapshot!
+  [conn snapshot-id]
+  (db/update! conn :file-change
+              {:deleted-at (dt/now)}
+              {:id snapshot-id}
+              {::db/return-keys false})
+  nil)
+
 (sv/defmethod ::delete-file-snapshot
   {::doc/added "1.20"
-   ::sm/params schema:remove-file-snapshot
-   ::db/transaction true}
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
+   ::sm/params schema:remove-file-snapshot}
+  [cfg {:keys [::rpc/profile-id id]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn]}]
+                (let [snapshot (get-snapshot conn id)]
+                  (files/check-edition-permissions! conn profile-id (:file-id snapshot))
 
-    (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
-                :code :system-snapshots-cant-be-deleted
-                :file-id (:file-id snapshot)
-                :snapshot-id id
-                :profile-id profile-id))
+                  (when (not= (:created-by snapshot) "user")
+                    (ex/raise :type :validation
+                              :code :system-snapshots-cant-be-deleted
+                              :snapshot-id id
+                              :profile-id profile-id))
 
-    (when (and (some? (:locked-by snapshot))
-               (not= (:locked-by snapshot) profile-id))
-      (ex/raise :type :validation
-                :code :snapshot-is-locked
-                :file-id (:file-id snapshot)
-                :snapshot-id id
-                :profile-id profile-id))
-
-    (let [team  (teams/get-team conn
-                                :profile-id profile-id
-                                :file-id (:file-id snapshot))
-          delay (ldel/get-deletion-delay team)]
-      (fsnap/delete! conn (assoc snapshot :deleted-at (ct/in-future delay))))))
-
-;;; Lock/unlock version endpoints
-
-(def ^:private schema:lock-file-snapshot
-  [:map {:title "lock-file-snapshot"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::lock-file-snapshot
-  {::doc/added "1.20"
-   ::sm/params schema:lock-file-snapshot
-   ::db/transaction true}
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
-
-    (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
-                :code :system-snapshots-cant-be-locked
-                :hint "Only user-created versions can be locked"
-                :snapshot-id id
-                :profile-id profile-id))
-
-    ;; Only the creator can lock their own version
-    (when (not= (:profile-id snapshot) profile-id)
-      (ex/raise :type :validation
-                :code :only-creator-can-lock
-                :hint "Only the version creator can lock it"
-                :snapshot-id id
-                :profile-id profile-id
-                :creator-id (:profile-id snapshot)))
-
-    ;; Check if already locked
-    (when (:locked-by snapshot)
-      (ex/raise :type :validation
-                :code :snapshot-already-locked
-                :hint "Version is already locked"
-                :snapshot-id id
-                :profile-id profile-id
-                :locked-by (:locked-by snapshot)))
-
-    (fsnap/lock-by! conn id profile-id)))
-
-(def ^:private schema:unlock-file-snapshot
-  [:map {:title "unlock-file-snapshot"}
-   [:id ::sm/uuid]])
-
-(sv/defmethod ::unlock-file-snapshot
-  {::doc/added "1.20"
-   ::sm/params schema:unlock-file-snapshot
-   ::db/transaction  true}
-  [{:keys [::db/conn]} {:keys [::rpc/profile-id id]}]
-  (let [snapshot (fsnap/get-minimal-snapshot conn id)]
-    (files/check-edition-permissions! conn profile-id (:file-id snapshot))
-
-    (when (not= (:created-by snapshot) "user")
-      (ex/raise :type :validation
-                :code :system-snapshots-cant-be-unlocked
-                :hint "Only user-created versions can be unlocked"
-                :snapshot-id id
-                :profile-id profile-id))
-
-    ;; Only the creator can unlock their own version
-    (when (not= (:profile-id snapshot) profile-id)
-      (ex/raise :type :validation
-                :code :only-creator-can-unlock
-                :hint "Only the version creator can unlock it"
-                :snapshot-id id
-                :profile-id profile-id
-                :creator-id (:profile-id snapshot)))
-
-    ;; Check if not locked
-    (when (not (:locked-by snapshot))
-      (ex/raise :type :validation
-                :code :snapshot-not-locked
-                :hint "Version is not locked"
-                :snapshot-id id
-                :profile-id profile-id))
-
-    (fsnap/unlock! conn id)))
+                  (delete-file-snapshot! conn id)))))

backend/src/app/rpc/commands/files_temp.clj

@@ -0,0 +1,176 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files-temp
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.features :as cfeat]
+   [app.common.files.changes :as cpc]
+   [app.common.schema :as sm]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as sql]
+   [app.features.components-v2 :as feat.compv2]
+   [app.features.fdata :as fdata]
+   [app.loggers.audit :as audit]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
+   [app.rpc.commands.files-create :as files.create]
+   [app.rpc.commands.files-update :as-alias files.update]
+   [app.rpc.commands.projects :as projects]
+   [app.rpc.commands.teams :as teams]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
+   [app.util.blob :as blob]
+   [app.util.pointer-map :as pmap]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.set :as set]))
+
+;; --- MUTATION COMMAND: create-temp-file
+
+(def ^:private schema:create-temp-file
+  [:map {:title "create-temp-file"}
+   [:name [:string {:max 250}]]
+   [:project-id ::sm/uuid]
+   [:id {:optional true} ::sm/uuid]
+   [:is-shared ::sm/boolean]
+   [:features ::cfeat/features]
+   [:create-page ::sm/boolean]])
+
+(sv/defmethod ::create-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:create-temp-file
+   ::db/transaction true}
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
+  (projects/check-edition-permissions! conn profile-id project-id)
+  (let [team (teams/get-team conn :profile-id profile-id :project-id project-id)
+        ;; When we create files, we only need to respect the team
+        ;; features, because some features can be enabled
+        ;; globally, but the team is still not migrated properly.
+        input-features
+        (:features params #{})
+
+        ;; If the imported project doesn't contain v2 we need to remove it
+        team-features
+        (cond-> (cfeat/get-team-enabled-features cf/flags team)
+          (not (contains? input-features "components/v2"))
+          (disj "components/v2"))
+
+        ;; We also include all no migration features declared by
+        ;; client; that enables the ability to enable a runtime
+        ;; feature on frontend and make it permanent on file
+        features
+        (-> input-features
+            (set/intersection cfeat/no-migration-features)
+            (set/union team-features))
+
+        params
+        (-> params
+            (assoc :profile-id profile-id)
+            (assoc :deleted-at (dt/in-future {:days 1}))
+            (assoc :features features))]
+
+    (files.create/create-file cfg params)))
+
+;; --- MUTATION COMMAND: update-temp-file
+
+
+(def ^:private schema:update-temp-file
+  [:map {:title "update-temp-file"}
+   [:changes [:vector ::cpc/change]]
+   [:revn [::sm/int {:min 0}]]
+   [:session-id ::sm/uuid]
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::update-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:update-temp-file}
+  [cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                    (db/insert! conn :file-change
+                                {:id (uuid/next)
+                                 :session-id session-id
+                                 :profile-id profile-id
+                                 :created-at (dt/now)
+                                 :file-id id
+                                 :revn revn
+                                 :data nil
+                                 :changes (blob/encode changes)})
+                    (rph/with-meta (rph/wrap nil)
+                      {::audit/replace-props {:file-id id
+                                              :revn revn}}))))
+
+;; --- MUTATION COMMAND: persist-temp-file
+
+(defn persist-temp-file
+  [{:keys [::db/conn] :as cfg} {:keys [id ::rpc/profile-id] :as params}]
+  (let [file (files/get-file cfg id
+                             :migrate? false
+                             :lock-for-update? true)]
+
+    (when (nil? (:deleted-at file))
+      (ex/raise :type :validation
+                :code :cant-persist-already-persisted-file))
+
+
+    (let [changes (->> (db/cursor conn
+                                  (sql/select :file-change {:file-id id}
+                                              {:order-by [[:revn :asc]]})
+                                  {:chunk-size 10})
+                       (sequence (mapcat (comp blob/decode :changes))))
+
+          file    (update file :data cpc/process-changes changes)
+
+          file    (if (contains? (:features file) "fdata/objects-map")
+                    (fdata/enable-objects-map file)
+                    file)
+
+          file    (if (contains? (:features file) "fdata/pointer-map")
+                    (binding [pmap/*tracked* (pmap/create-tracked)]
+                      (let [file (fdata/enable-pointer-map file)]
+                        (fdata/persist-pointers! cfg id)
+                        file))
+                    file)]
+
+      ;; Delete changes from the changes history
+      (db/delete! conn :file-change {:file-id id})
+
+      (db/update! conn :file
+                  {:deleted-at nil
+                   :revn 1
+                   :data (blob/encode (:data file))}
+                  {:id id})
+
+      (let [team          (teams/get-team conn :profile-id profile-id :project-id (:project-id file))
+            file-features (:features file)
+            team-features (cfeat/get-team-enabled-features cf/flags team)]
+        (when (and (contains? team-features "components/v2")
+                   (not (contains? file-features "components/v2")))
+          ;; Migrate components v2
+          (feat.compv2/migrate-file! cfg
+                                     (:id file)
+                                     :max-procs 2
+                                     :validate? true
+                                     :throw-on-validate? true)))
+
+      nil)))
+
+(def ^:private schema:persist-temp-file
+  [:map {:title "persist-temp-file"}
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::persist-temp-file
+  {::doc/added "1.17"
+   ::doc/module :files
+   ::sm/params schema:persist-temp-file}
+  [cfg {:keys [::rpc/profile-id id] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (persist-temp-file cfg params))))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.files-thumbnails
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.features :as cfeat]
@@ -14,7 +13,6 @@
    [app.common.geom.shapes :as gsh]
    [app.common.schema :as sm]
    [app.common.thumbnails :as thc]
-   [app.common.time :as ct]
    [app.common.types.shape-tree :as ctt]
    [app.config :as cf]
    [app.db :as db]
@@ -32,12 +30,13 @@
    [app.storage :as sto]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [cuerdas.core :as str]))
 
 ;; --- FEATURES
 
 (def long-cache-duration
-  (ct/duration {:days 7}))
+  (dt/duration {:days 7}))
 
 ;; --- COMMAND QUERY: get-file-object-thumbnails
 
@@ -186,7 +185,7 @@
   [:map {:title "PartialFile"}
    [:id ::sm/uuid]
    [:revn {:min 0} ::sm/int]
-   [:page [:map-of :keyword ::sm/any]]])
+   [:page :any]])
 
 (sv/defmethod ::get-file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
@@ -203,9 +202,9 @@
                                             :profile-id profile-id
                                             :file-id file-id)
 
-                       file (bfc/get-file cfg file-id
-                                          :realize? true
-                                          :read-only? true)]
+                       file (files/get-file cfg file-id
+                                            :preload-pointers? true
+                                            :read-only? true)]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
                        (cfeat/check-file-features! (:features file)))
@@ -248,7 +247,7 @@
 (defn- create-file-object-thumbnail!
   [{:keys [::sto/storage] :as cfg} file object-id media tag]
   (let [file-id   (:id file)
-        timestamp (ct/now)
+        timestamp (dt/now)
         media     (persist-thumbnail! storage media timestamp)
         [th1 th2] (db/tx-run! cfg (fn [{:keys [::db/conn]}]
                                     (let [th1 (db/exec-one! conn [sql:get-file-object-thumbnail file-id object-id tag])
@@ -272,7 +271,7 @@
   [:map {:title "create-file-object-thumbnail"}
    [:file-id ::sm/uuid]
    [:object-id [:string {:max 250}]]
-   [:media media/schema:upload]
+   [:media ::media/upload]
    [:tag {:optional true} [:string {:max 50}]]])
 
 (sv/defmethod ::create-file-object-thumbnail
@@ -303,7 +302,7 @@
                                              {::sql/for-update true})]
     (sto/touch-object! storage media-id)
     (db/update! conn :file-tagged-object-thumbnail
-                {:deleted-at (ct/now)}
+                {:deleted-at (dt/now)}
                 {:file-id file-id
                  :object-id object-id
                  :tag tag})))
@@ -339,8 +338,7 @@
         hash  (sto/calculate-hash path)
         data  (-> (sto/content path)
                   (sto/wrap-with-hash hash))
-        tnow  (ct/now)
-
+        tnow  (dt/now)
         media (sto/put-object! storage
                                {::sto/content data
                                 ::sto/deduplicate? true
@@ -383,7 +381,7 @@
   [:map {:title "create-file-thumbnail"}
    [:file-id ::sm/uuid]
    [:revn ::sm/int]
-   [:media media/schema:upload]])
+   [:media ::media/upload]])
 
 (sv/defmethod ::create-file-thumbnail
   "Creates or updates the file thumbnail. Mainly used for paint the

backend/src/app/rpc/commands/files_update.clj

@@ -15,29 +15,30 @@
    [app.common.files.validate :as val]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
-   [app.features.fdata :as fdata]
-   [app.features.file-snapshots :as fsnap]
-   [app.features.logical-deletion :as ldel]
+   [app.features.fdata :as feat.fdata]
+   [app.features.file-migrations :as feat.fmigr]
    [app.http.errors :as errors]
    [app.loggers.audit :as audit]
    [app.loggers.webhooks :as webhooks]
    [app.metrics :as mtx]
    [app.msgbus :as mbus]
-   [app.redis :as rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as climit]
    [app.rpc.commands.files :as files]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [clojure.set :as set]))
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.set :as set]
+   [promesa.exec :as px]))
 
 (declare ^:private get-lagged-changes)
 (declare ^:private send-notifications!)
@@ -45,7 +46,6 @@
 (declare ^:private update-file*)
 (declare ^:private process-changes-and-validate)
 (declare ^:private take-snapshot?)
-(declare ^:private invalidate-caches!)
 
 ;; PUBLIC API; intended to be used outside of this module
 (declare update-file!)
@@ -63,10 +63,10 @@
    [:revn {:min 0} ::sm/int]
    [:vern {:min 0} ::sm/int]
    [:features {:optional true} ::cfeat/features]
-   [:changes {:optional true} [:vector cpc/schema:change]]
+   [:changes {:optional true} [:vector ::cpc/change]]
    [:changes-with-metadata {:optional true}
     [:vector [:map
-              [:changes [:vector cpc/schema:change]]
+              [:changes [:vector ::cpc/change]]
               [:hint-origin {:optional true} :keyword]
               [:hint-events {:optional true} [:vector [:string {:max 250}]]]]]]
    [:skip-validate {:optional true} ::sm/boolean]])
@@ -75,7 +75,7 @@
   schema:update-file-result
   [:vector {:title "update-file-result"}
    [:map
-    [:changes [:vector cpc/schema:change]]
+    [:changes [:vector ::cpc/change]]
     [:file-id ::sm/uuid]
     [:id ::sm/uuid]
     [:revn {:min 0} ::sm/int]
@@ -122,84 +122,83 @@
                 [:update-file/global]]
 
    ::webhooks/event? true
-   ::webhooks/batch-timeout (ct/duration "2m")
+   ::webhooks/batch-timeout (dt/duration "2m")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
 
    ::sm/params schema:update-file
    ::sm/result schema:update-file-result
    ::doc/module :files
-   ::doc/added "1.17"
-   ::db/transaction true}
-  [{:keys [::mtx/metrics ::db/conn] :as cfg}
+   ::doc/added "1.17"}
+  [{:keys [::mtx/metrics] :as cfg}
    {:keys [::rpc/profile-id id changes changes-with-metadata] :as params}]
+  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                    (files/check-edition-permissions! conn profile-id id)
+                    (db/xact-lock! conn id)
 
-  (files/check-edition-permissions! conn profile-id id)
-  (db/xact-lock! conn id)
+                    (let [file     (get-file conn id)
+                          team     (teams/get-team conn
+                                                   :profile-id profile-id
+                                                   :team-id (:team-id file))
 
-  (let [file     (get-file cfg id)
-        team     (teams/get-team conn
-                                 :profile-id profile-id
-                                 :team-id (:team-id file))
+                          features (-> (cfeat/get-team-enabled-features cf/flags team)
+                                       (cfeat/check-client-features! (:features params))
+                                       (cfeat/check-file-features! (:features file)))
 
-        features (-> (cfeat/get-team-enabled-features cf/flags team)
-                     (cfeat/check-client-features! (:features params))
-                     (cfeat/check-file-features! (:features file)))
+                          changes  (if changes-with-metadata
+                                     (->> changes-with-metadata (mapcat :changes) vec)
+                                     (vec changes))
 
-        changes  (if changes-with-metadata
-                   (->> changes-with-metadata (mapcat :changes) vec)
-                   (vec changes))
+                          params   (-> params
+                                       (assoc :profile-id profile-id)
+                                       (assoc :features (set/difference features cfeat/frontend-only-features))
+                                       (assoc :team team)
+                                       (assoc :file file)
+                                       (assoc :changes changes))
 
-        params   (-> params
-                     (assoc :profile-id profile-id)
-                     (assoc :features (set/difference features cfeat/frontend-only-features))
-                     (assoc :team team)
-                     (assoc :file file)
-                     (assoc :changes changes))
+                          cfg      (assoc cfg ::timestamp (dt/now))
 
-        cfg      (assoc cfg ::timestamp (ct/now))
-
-        tpoint   (ct/tpoint)]
-
-    (when (not= (:vern params)
-                (:vern file))
-      (ex/raise :type :validation
-                :code :vern-conflict
-                :hint "A different version has been restored for the file."
-                :context {:incoming-revn (:revn params)
-                          :stored-revn (:revn file)}))
-
-    (when (> (:revn params)
-             (:revn file))
-      (ex/raise :type :validation
-                :code :revn-conflict
-                :hint "The incoming revision number is greater that stored version."
-                :context {:incoming-revn (:revn params)
-                          :stored-revn (:revn file)}))
-
-    ;; When newly computed features does not match exactly with the
-    ;; features defined on team row, we update it
-    (when-let [features (-> features
-                            (set/difference (:features team))
-                            (set/difference cfeat/no-team-inheritable-features)
-                            (not-empty))]
-      (let [features (-> features
-                         (set/union (:features team))
-                         (set/difference cfeat/no-team-inheritable-features)
-                         (into-array))]
-        (db/update! conn :team
-                    {:features features}
-                    {:id (:id team)}
-                    {::db/return-keys false})))
+                          tpoint   (dt/tpoint)]
 
 
-    (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+                      (when (not= (:vern params)
+                                  (:vern file))
+                        (ex/raise :type :validation
+                                  :code :vern-conflict
+                                  :hint "A different version has been restored for the file."
+                                  :context {:incoming-revn (:revn params)
+                                            :stored-revn (:revn file)}))
 
-    (binding [l/*context* (some-> (meta params)
-                                  (get :app.http/request)
-                                  (errors/request->context))]
-      (-> (update-file* cfg params)
-          (rph/with-defer #(let [elapsed (tpoint)]
-                             (l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))
+                      (when (> (:revn params)
+                               (:revn file))
+                        (ex/raise :type :validation
+                                  :code :revn-conflict
+                                  :hint "The incoming revision number is greater that stored version."
+                                  :context {:incoming-revn (:revn params)
+                                            :stored-revn (:revn file)}))
+
+                      ;; When newly computed features does not match exactly with
+                      ;; the features defined on team row, we update it
+                      (when-let [features (-> features
+                                              (set/difference (:features team))
+                                              (set/difference cfeat/no-team-inheritable-features)
+                                              (not-empty))]
+                        (let [features (->> features
+                                            (set/union (:features team))
+                                            (db/create-array conn "text"))]
+                          (db/update! conn :team
+                                      {:features features}
+                                      {:id (:id team)}
+                                      {::db/return-keys false})))
+
+
+                      (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
+                      (binding [l/*context* (some-> (meta params)
+                                                    (get :app.http/request)
+                                                    (errors/request->context))]
+                        (-> (update-file* cfg params)
+                            (rph/with-defer #(let [elapsed (tpoint)]
+                                               (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
 
 (defn- update-file*
   "Internal function, part of the update-file process, that encapsulates
@@ -209,41 +208,31 @@
   Follow the inner implementation to `update-file-data!` function.
 
   Only intended for internal use on this module."
-  [{:keys [::db/conn ::timestamp] :as cfg}
-   {:keys [profile-id file team features changes session-id skip-validate] :as params}]
+  [{:keys [::db/conn ::wrk/executor ::timestamp] :as cfg}
+   {:keys [profile-id file features changes session-id skip-validate] :as params}]
 
-  (binding [pmap/*tracked* (pmap/create-tracked)
-            pmap/*load-fn* (partial fdata/load-pointer cfg (:id file))]
+  (let [;; Retrieve the file data
+        file  (feat.fmigr/resolve-applied-migrations cfg file)
+        file  (feat.fdata/resolve-file-data cfg file)
+        file  (assoc file :features
+                     (-> features
+                         (set/difference cfeat/frontend-only-features)
+                         (set/union (:features file))))]
 
-    (let [file (assoc file :features
-                      (-> features
-                          (set/difference cfeat/frontend-only-features)
-                          (set/union (:features file))))
+    ;; We create a new lexycal scope for clearly delimit the result of
+    ;; executing this update file operation and all its side effects
+    (let [file (px/invoke! executor
+                           (fn []
+                             ;; Process the file data on separated thread for avoid to do
+                             ;; the CPU intensive operation on vthread.
+                             (binding [cfeat/*current*  features
+                                       cfeat/*previous* (:features file)]
+                               (update-file-data! cfg file
+                                                  process-changes-and-validate
+                                                  changes skip-validate))))]
 
-          ;; We need to preserve the original revn for the response
-          revn
-          (get file :revn)
-
-          file
-          (binding [cfeat/*current*  features
-                    cfeat/*previous* (:features file)]
-            (update-file-data! cfg file
-                               process-changes-and-validate
-                               changes skip-validate))
-
-          deleted-at
-          (ct/plus timestamp (ct/duration {:hours 1}))]
-
-      (when-let [file (::snapshot file)]
-        (let [deleted-at (ct/plus timestamp (ldel/get-deletion-delay team))
-              label      (str "internal/snapshot/" revn)]
-
-          (fsnap/create! cfg file
-                         {:label label
-                          :created-by "system"
-                          :deleted-at deleted-at
-                          :profile-id profile-id
-                          :session-id session-id})))
+      (feat.fmigr/upsert-migrations! conn file)
+      (persist-file! cfg file)
 
       ;; Insert change (xlog) with deleted_at in a future data for
       ;; make them automatically eleggible for GC once they expires
@@ -253,71 +242,87 @@
                    :profile-id profile-id
                    :created-at timestamp
                    :updated-at timestamp
-                   :deleted-at deleted-at
+                   :deleted-at (if (::snapshot-data file)
+                                 (dt/plus timestamp (cf/get-deletion-delay))
+                                 (dt/plus timestamp (dt/duration {:hours 1})))
                    :file-id (:id file)
                    :revn (:revn file)
                    :version (:version file)
-                   :features (into-array (:features file))
+                   :features (:features file)
+                   :label (::snapshot-label file)
+                   :data (::snapshot-data file)
                    :changes (blob/encode changes)}
                   {::db/return-keys false})
 
-      (persist-file! cfg file)
-
-      (when (contains? cf/flags :redis-cache)
-        (invalidate-caches! cfg file))
-
       ;; Send asynchronous notifications
-      (send-notifications! cfg params file)
+      (send-notifications! cfg params file))
 
-      (with-meta {:revn revn :lagged (get-lagged-changes conn params)}
-        {::audit/replace-props
-         {:id         (:id file)
-          :name       (:name file)
-          :features   (:features file)
-          :project-id (:project-id file)
-          :team-id    (:team-id file)}}))))
+    (when (feat.fdata/offloaded? file)
+      (let [storage (sto/resolve cfg ::db/reuse-conn true)]
+        (some->> (:data-ref-id file) (sto/touch-object! storage))))
+
+    (let [response {:revn (:revn file)
+                    :lagged (get-lagged-changes conn params)}]
+      (vary-meta response assoc ::audit/replace-props
+                 {:id         (:id file)
+                  :name       (:name file)
+                  :features   (:features file)
+                  :project-id (:project-id file)
+                  :team-id    (:team-id file)}))))
+
+(defn update-file!
+  "A public api that allows apply a transformation to a file with all context setup."
+  [{:keys [::db/conn] :as cfg} file-id update-fn & args]
+  (let [file (get-file cfg file-id)
+        file (apply update-file-data! cfg file update-fn args)]
+    (feat.fmigr/upsert-migrations! conn file)
+    (persist-file! cfg file)))
+
+(def ^:private sql:get-file
+  "SELECT f.*, p.team_id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+    WHERE f.id = ?
+      AND (f.deleted_at IS NULL OR
+           f.deleted_at > now())
+      FOR KEY SHARE")
 
 (defn get-file
   "Get not-decoded file, only decodes the features set."
-  [cfg id]
-  (bfc/get-file cfg id :decode? false :lock-for-share? true))
+  [conn id]
+  (let [file (db/exec-one! conn [sql:get-file id])]
+    (when-not file
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint (format "file with id '%s' does not exists" id)))
+    (update file :features db/decode-pgarray #{})))
 
 (defn persist-file!
   "Function responsible of persisting already encoded file. Should be
   used together with `get-file` and `update-file-data!`.
 
   It also updates the project modified-at attr."
-  [{:keys [::db/conn ::timestamp] :as cfg} file]
+  [{:keys [::db/conn ::timestamp]} file]
   (let [;; The timestamp can be nil because this function is also
         ;; intended to be used outside of this module
-        modified-at
-        (or timestamp (ct/now))
-
-        file
-        (-> file
-            (dissoc ::snapshot)
-            (assoc :modified-at modified-at)
-            (assoc :has-media-trimmed false))]
+        modified-at (or timestamp (dt/now))]
 
     (db/update! conn :project
                 {:modified-at modified-at}
                 {:id (:project-id file)}
                 {::db/return-keys false})
 
-    (bfc/update-file! cfg file)))
-
-(defn- invalidate-caches!
-  [cfg {:keys [id] :as file}]
-  (rds/run! cfg (fn [{:keys [::rds/conn]}]
-                  (let [key (str files/file-summary-cache-key-prefix id)]
-                    (rds/del conn key)))))
-
-(defn- attach-snapshot
-  "Attach snapshot data to the file. This should be called before the
-  upcoming file operations are applied to the file."
-  [cfg migrated? file]
-  (let [snapshot (if migrated? file (fdata/realize cfg file))]
-    (assoc file ::snapshot snapshot)))
+    (db/update! conn :file
+                {:revn (:revn file)
+                 :data (:data file)
+                 :version (:version file)
+                 :features (:features file)
+                 :data-backend nil
+                 :data-ref-id nil
+                 :modified-at modified-at
+                 :has-media-trimmed false}
+                {:id (:id file)}
+                {::db/return-keys false})))
 
 (defn- update-file-data!
   "Perform a file data transformation in with all update context setup.
@@ -329,35 +334,58 @@
   fdata/pointer-map modified fragments."
 
   [cfg {:keys [id] :as file} update-fn & args]
-  (let [file (update file :data (fn [data]
-                                  (-> data
-                                      (blob/decode)
-                                      (assoc :id id))))
-        libs (delay (bfc/get-resolved-file-libraries cfg file))
+  (binding [pmap/*tracked* (pmap/create-tracked)
+            pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+    (let [file (update file :data (fn [data]
+                                    (-> data
+                                        (blob/decode)
+                                        (assoc :id (:id file)))))
 
-        need-migration?
-        (fmg/need-migration? file)
+          ;; For avoid unnecesary overhead of creating multiple pointers
+          ;; and handly internally with objects map in their worst
+          ;; case (when probably all shapes and all pointers will be
+          ;; readed in any case), we just realize/resolve them before
+          ;; applying the migration to the file
+          file (if (fmg/need-migration? file)
+                 (-> file
+                     (update :data feat.fdata/process-pointers deref)
+                     (update :data feat.fdata/process-objects (partial into {}))
+                     (fmg/migrate-file))
+                 file)
 
-        take-snapshot?
-        (take-snapshot? file)
+          file (apply update-fn cfg file args)
 
-        ;; For avoid unnecesary overhead of creating multiple
-        ;; pointers and handly internally with objects map in their
-        ;; worst case (when probably all shapes and all pointers
-        ;; will be readed in any case), we just realize/resolve them
-        ;; before applying the migration to the file
-        file
-        (cond-> file
-          ;; need-migration?
-          ;; (->> (fdata/realize cfg))
+          ;; TODO: reuse operations if file is migrated
+          ;; TODO: move encoding to a separated thread
+          file (if (take-snapshot? file)
+                 (let [tpoint   (dt/tpoint)
+                       snapshot (-> (:data file)
+                                    (feat.fdata/process-pointers deref)
+                                    (feat.fdata/process-objects (partial into {}))
+                                    (blob/encode))
+                       elapsed (tpoint)
+                       label   (str "internal/snapshot/" (:revn file))]
 
-          need-migration?
-          (fmg/migrate-file libs)
+                   (l/trc :hint "take snapshot"
+                          :file-id (str (:id file))
+                          :revn (:revn file)
+                          :label label
+                          :elapsed (dt/format-duration elapsed))
 
-          take-snapshot?
-          (->> (attach-snapshot cfg need-migration?)))]
+                   (-> file
+                       (assoc ::snapshot-data snapshot)
+                       (assoc ::snapshot-label label)))
+                 file)]
 
-    (apply update-fn cfg file args)))
+      (bfc/encode-file cfg file))))
+
+(defn- get-file-libraries
+  "A helper for preload file libraries, mainly used for perform file
+  semantical and structural validation"
+  [{:keys [::db/conn] :as cfg} file]
+  (->> (files/get-file-libraries conn (:id file))
+       (into [file] (map #(bfc/get-file cfg (:id %))))
+       (d/index-by :id)))
 
 (defn- soft-validate-file-schema!
   [file]
@@ -383,7 +411,8 @@
         (when (and (or (contains? cf/flags :file-validation)
                        (contains? cf/flags :soft-file-validation))
                    (not skip-validate))
-          (bfc/get-resolved-file-libraries cfg file))
+          (get-file-libraries cfg file))
+
 
         ;; The main purpose of this atom is provide a contextual state
         ;; for the changes subsystem where optionally some hints can
@@ -428,11 +457,11 @@
   (when (contains? cf/flags :auto-file-snapshot)
     (let [freq    (or (cf/get :auto-file-snapshot-every) 20)
           timeout (or (cf/get :auto-file-snapshot-timeout)
-                      (ct/duration {:hours 1}))]
+                      (dt/duration {:hours 1}))]
 
       (or (= 1 freq)
           (zero? (mod revn freq))
-          (> (inst-ms (ct/diff modified-at (ct/now)))
+          (> (inst-ms (dt/diff modified-at (dt/now)))
              (inst-ms timeout))))))
 
 (def ^:private sql:lagged-changes
@@ -446,9 +475,8 @@
 (defn- get-lagged-changes
   [conn {:keys [id revn] :as params}]
   (->> (db/exec! conn [sql:lagged-changes id revn])
-       (filter :changes)
-       (mapv (fn [row]
-               (update row :changes blob/decode)))))
+       (map files/decode-row)
+       (vec)))
 
 (defn- send-notifications!
   [cfg {:keys [team changes session-id] :as params} file]
@@ -473,5 +501,5 @@
                            :file-id (:id file)
                            :session-id session-id
                            :revn (:revn file)
-                           :modified-at (ct/now)
+                           :modified-at (dt/now)
                            :changes lchanges}))))

backend/src/app/rpc/commands/fonts.clj

@@ -9,11 +9,9 @@
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -26,7 +24,10 @@
    [app.rpc.helpers :as rph]
    [app.rpc.quotes :as quotes]
    [app.storage :as sto]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 (def valid-weight #{100 200 300 400 500 600 700 800 900 950})
 (def valid-style #{"normal" "italic"})
@@ -35,13 +36,14 @@
 
 (def ^:private
   schema:get-font-variants
-  [:and
-   [:map {:title "get-font-variants"}
-    [:team-id {:optional true} ::sm/uuid]
-    [:file-id {:optional true} ::sm/uuid]
-    [:project-id {:optional true} ::sm/uuid]
-    [:share-id {:optional true} ::sm/uuid]]
-   [::sm/contains-any #{:team-id :file-id :project-id}]])
+  [:schema {:title "get-font-variants"}
+   [:and
+    [:map
+     [:team-id {:optional true} ::sm/uuid]
+     [:file-id {:optional true} ::sm/uuid]
+     [:project-id {:optional true} ::sm/uuid]
+     [:share-id {:optional true} ::sm/uuid]]
+    [::sm/contains-any #{:team-id :file-id :project-id}]]])
 
 (sv/defmethod ::get-font-variants
   {::doc/added "1.18"
@@ -78,9 +80,9 @@
 (def ^:private schema:create-font-variant
   [:map {:title "create-font-variant"}
    [:team-id ::sm/uuid]
-   [:data [:map-of ::sm/text ::sm/any]]
+   [:data [:map-of :string :any]]
    [:font-id ::sm/uuid]
-   [:font-family ::sm/text]
+   [:font-family :string]
    [:font-weight [::sm/one-of {:format "number"} valid-weight]]
    [:font-style [::sm/one-of {:format "string"} valid-style]]])
 
@@ -103,7 +105,7 @@
                 (create-font-variant cfg (assoc params :profile-id profile-id)))))
 
 (defn create-font-variant
-  [{:keys [::sto/storage ::db/conn]} {:keys [data] :as params}]
+  [{:keys [::sto/storage ::db/conn ::wrk/executor]} {:keys [data] :as params}]
   (letfn [(generate-missing! [data]
             (let [data (media/run {:cmd :generate-fonts :input data})]
               (when (and (not (contains? data "font/otf"))
@@ -121,7 +123,7 @@
                     content (-> (sto/content resource)
                                 (sto/wrap-with-hash hash))]
                 {::sto/content content
-                 ::sto/touched-at (ct/now)
+                 ::sto/touched-at (dt/now)
                  ::sto/deduplicate? true
                  :content-type mtype
                  :bucket "team-font-variant"})))
@@ -155,7 +157,7 @@
                          :otf-file-id (:id otf)
                          :ttf-file-id (:id ttf)}))]
 
-    (let [data   (generate-missing! data)
+    (let [data   (px/invoke! executor (partial generate-missing! data))
           assets (persist-fonts-files! data)
           result (insert-font-variant! assets)]
       (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))
@@ -200,40 +202,32 @@
 (sv/defmethod ::delete-font
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font
-   ::db/transaction true}
-  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
-  (let [team  (teams/get-team conn
-                              :profile-id profile-id
-                              :team-id team-id)
+   ::sm/params schema:delete-font}
+  [cfg {:keys [::rpc/profile-id id team-id]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (teams/check-edition-permissions! conn profile-id team-id)
+                (let [fonts   (db/query conn :team-font-variant
+                                        {:team-id team-id
+                                         :font-id id
+                                         :deleted-at nil}
+                                        {::sql/for-update true})
+                      tnow    (dt/now)]
 
-        fonts (db/query conn :team-font-variant
-                        {:team-id team-id
-                         :font-id id
-                         :deleted-at nil}
-                        {::sql/for-update true})
+                  (when-not (seq fonts)
+                    (ex/raise :type :not-found
+                              :code :object-not-found))
 
-        delay (ldel/get-deletion-delay team)
-        tnow  (ct/in-future delay)]
+                  (doseq [font fonts]
+                    (db/update! conn :team-font-variant
+                                {:deleted-at tnow}
+                                {:id (:id font)}))
 
-    (teams/check-edition-permissions! (:permissions team))
-
-    (when-not (seq fonts)
-      (ex/raise :type :not-found
-                :code :object-not-found))
-
-
-    (doseq [font fonts]
-      (db/update! conn :team-font-variant
-                  {:deleted-at tnow}
-                  {:id (:id font)}
-                  {::db/return-keys false}))
-
-    (rph/with-meta (rph/wrap)
-      {::audit/props {:id id
-                      :team-id team-id
-                      :name (:font-family (peek fonts))
-                      :profile-id profile-id}})))
+                  (rph/with-meta (rph/wrap)
+                    {::audit/props {:id id
+                                    :team-id team-id
+                                    :name (:font-family (peek fonts))
+                                    :profile-id profile-id}})))))
 
 ;; --- DELETE FONT VARIANT
 
@@ -245,23 +239,19 @@
 (sv/defmethod ::delete-font-variant
   {::doc/added "1.18"
    ::webhooks/event? true
-   ::sm/params schema:delete-font-variant
-   ::db/transaction true}
-  [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id team-id]}]
-  (let [team    (teams/get-team conn
-                                :profile-id profile-id
-                                :team-id team-id)
-        variant (db/get conn :team-font-variant
-                        {:id id :team-id team-id}
-                        {::sql/for-update true})
-        delay   (ldel/get-deletion-delay team)]
+   ::sm/params schema:delete-font-variant}
+  [cfg {:keys [::rpc/profile-id id team-id]}]
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (teams/check-edition-permissions! conn profile-id team-id)
+                (let [variant (db/get conn :team-font-variant
+                                      {:id id :team-id team-id}
+                                      {::sql/for-update true})]
 
-    (teams/check-edition-permissions! (:permissions team))
-    (db/update! conn :team-font-variant
-                {:deleted-at (ct/in-future delay)}
-                {:id (:id variant)}
-                {::db/return-keys false})
+                  (db/update! conn :team-font-variant
+                              {:deleted-at (dt/now)}
+                              {:id (:id variant)})
 
-    (rph/with-meta (rph/wrap)
-      {::audit/props {:font-family (:font-family variant)
-                      :font-id (:font-id variant)}})))
+                  (rph/with-meta (rph/wrap)
+                    {::audit/props {:font-family (:font-family variant)
+                                    :font-id (:font-id variant)}})))))

backend/src/app/rpc/commands/ldap.clj

@@ -38,7 +38,7 @@
    ::doc/added "1.15"
    ::doc/module :auth
    ::sm/params schema:login-with-ldap}
-  [{:keys [::ldap/provider] :as cfg} params]
+  [{:keys [::setup/props ::ldap/provider] :as cfg} params]
   (when-not provider
     (ex/raise :type :restriction
               :code :ldap-not-initialized
@@ -60,11 +60,11 @@
         ;; user comes from team-invitation process; in this case,
         ;; regenerate token and send back to the user a new invitation
         ;; token (and mark current session as logged).
-        (let [claims (tokens/verify cfg {:token token :iss :team-invitation})
+        (let [claims (tokens/verify props {:token token :iss :team-invitation})
               claims (assoc claims
                             :member-id  (:id profile)
                             :member-email (:email profile))
-              token  (tokens/generate cfg claims)]
+              token  (tokens/generate props claims)]
           (-> {:invitation-token token}
               (rph/with-transform (session/create-fn cfg (:id profile)))
               (rph/with-meta {::audit/props (:props profile)

backend/src/app/rpc/commands/management.clj

@@ -13,7 +13,6 @@
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -28,14 +27,17 @@
    [app.setup :as-alias setup]
    [app.setup.templates :as tmpl]
    [app.storage.tmp :as tmp]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
+   [promesa.exec :as px]))
 
 ;; --- COMMAND: Duplicate File
 
 (defn duplicate-file
   [{:keys [::db/conn ::bfc/timestamp] :as cfg} {:keys [profile-id file-id name reset-shared-flag] :as params}]
   (let [;; We don't touch the original file on duplication
-        file       (bfc/get-file cfg file-id :realize? true)
+        file       (bfc/get-file cfg file-id)
         project-id (:project-id file)
         file       (-> file
                        (update :id bfc/lookup-index)
@@ -54,7 +56,7 @@
     (vswap! bfc/*state* update :index bfc/update-index fmeds :id)
 
     ;; Process and persist file
-    (let [file (bfc/process-file cfg file)]
+    (let [file (bfc/process-file file)]
       (bfc/insert-file! cfg file ::db/return-keys false)
 
       ;; The file profile creation is optional, so when no profile is
@@ -102,7 +104,7 @@
                     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
 
                     (binding [bfc/*state* (volatile! {:index {file-id (uuid/next)}})]
-                      (duplicate-file (assoc cfg ::bfc/timestamp (ct/now))
+                      (duplicate-file (assoc cfg ::bfc/timestamp (dt/now))
                                       (-> params
                                           (assoc :profile-id profile-id)
                                           (assoc :reset-shared-flag true)))))))
@@ -162,7 +164,7 @@
   (db/tx-run! cfg (fn [cfg]
                     ;; Defer all constraints
                     (db/exec-one! cfg ["SET CONSTRAINTS ALL DEFERRED"])
-                    (-> (assoc cfg ::bfc/timestamp (ct/now))
+                    (-> (assoc cfg ::bfc/timestamp (dt/now))
                         (duplicate-project (assoc params :profile-id profile-id))))))
 
 (defn duplicate-team
@@ -311,14 +313,15 @@
 
     ;; Update the modification date of the all affected projects
     ;; ensuring that the destination project is the most recent one.
-    (loop [project-ids (into (list project-id) source)
-           modified-at (ct/now)]
-      (when-let [project-id (first project-ids)]
-        (db/update! conn :project
-                    {:modified-at modified-at}
-                    {:id project-id})
-        (recur (rest project-ids)
-               (ct/plus modified-at 10))))
+    (doseq [project-id (into (list project-id) source)]
+
+      ;; NOTE: as this is executed on virtual thread, sleeping does
+      ;; not causes major issues, and allows an easy way to set a
+      ;; trully different modification date to each file.
+      (px/sleep 10)
+      (db/update! conn :project
+                  {:modified-at (dt/now)}
+                  {:id project-id}))
 
     nil))
 
@@ -393,7 +396,12 @@
 ;; --- COMMAND: Clone Template
 
 (defn clone-template
-  [{:keys [::db/pool] :as cfg} {:keys [project-id profile-id] :as params} template]
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [project-id profile-id] :as params} template]
+
+  ;; NOTE: the importation process performs some operations
+  ;; that are not very friendly with virtual threads, and for
+  ;; avoid unexpected blocking of other concurrent operations
+  ;; we dispatch that operation to a dedicated executor.
   (let [template (tmp/tempfile-from template
                                     :prefix "penpot.template."
                                     :suffix ""
@@ -411,13 +419,13 @@
                      (assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
 
         result   (if (= format :binfile-v3)
-                   (bf.v3/import-files! cfg)
-                   (bf.v1/import-files! cfg))]
+                   (px/invoke! executor (partial bf.v3/import-files! cfg))
+                   (px/invoke! executor (partial bf.v1/import-files! cfg)))]
 
     (db/tx-run! cfg
                 (fn [{:keys [::db/conn] :as cfg}]
                   (db/update! conn :project
-                              {:modified-at (ct/now)}
+                              {:modified-at (dt/now)}
                               {:id project-id}
                               {::db/return-keys false})
 

backend/src/app/rpc/commands/media.clj

@@ -10,7 +10,6 @@
    [app.common.exceptions :as ex]
    [app.common.media :as cm]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -24,8 +23,11 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.services :as sv]
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
    [cuerdas.core :as str]
-   [datoteka.io :as io]))
+   [datoteka.io :as io]
+   [promesa.exec :as px]))
 
 (def default-max-file-size
   (* 1024 1024 10)) ; 10 MiB
@@ -46,7 +48,7 @@
    [:file-id ::sm/uuid]
    [:is-local ::sm/boolean]
    [:name [:string {:max 250}]]
-   [:content media/schema:upload]])
+   [:content ::media/upload]])
 
 (sv/defmethod ::upload-file-media-object
   {::doc/added "1.17"
@@ -65,7 +67,7 @@
                        mobj  (create-file-media-object cfg params)]
 
                    (db/update! conn :file
-                               {:modified-at (ct/now)
+                               {:modified-at (dt/now)
                                 :has-media-trimmed false}
                                {:id file-id}
                                {::db/return-keys false})
@@ -151,9 +153,9 @@
       (assoc ::image (process-main-image info)))))
 
 (defn- create-file-media-object
-  [{:keys [::sto/storage ::db/conn] :as cfg}
+  [{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
    {:keys [id file-id is-local name content]}]
-  (let [result (process-image content)
+  (let [result (px/invoke! executor (partial process-image content))
         image  (sto/put-object! storage (::image result))
         thumb  (when-let [params (::thumb result)]
                  (sto/put-object! storage params))]
@@ -190,7 +192,7 @@
         mobj (create-file-media-object-from-url cfg (assoc params :profile-id profile-id))]
 
     (db/update! pool :file
-                {:modified-at (ct/now)
+                {:modified-at (dt/now)
                  :has-media-trimmed false}
                 {:id file-id}
                 {::db/return-keys false})

backend/src/app/rpc/commands/profile.clj

@@ -10,7 +10,6 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.types.plugins :refer [schema:plugin-registry]]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -29,14 +28,18 @@
    [app.storage :as sto]
    [app.tokens :as tokens]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [promesa.exec :as px]))
 
 (declare check-profile-existence!)
 (declare decode-row)
+(declare derive-password)
 (declare filter-props)
 (declare get-profile)
 (declare strip-private-attrs)
+(declare verify-password)
 
 (def schema:props-notifications
   [:map {:title "props-notifications"}
@@ -67,8 +70,8 @@
    [:is-blocked {:optional true} ::sm/boolean]
    [:is-demo {:optional true} ::sm/boolean]
    [:is-muted {:optional true} ::sm/boolean]
-   [:created-at {:optional true} ::ct/inst]
-   [:modified-at {:optional true} ::ct/inst]
+   [:created-at {:optional true} ::sm/inst]
+   [:modified-at {:optional true} ::sm/inst]
    [:default-project-id {:optional true} ::sm/uuid]
    [:default-team-id {:optional true} ::sm/uuid]
    [:props {:optional true} schema:props]])
@@ -107,9 +110,7 @@
 (defn get-profile
   "Get profile by id. Throws not-found exception if no profile found."
   [conn id & {:as opts}]
-  ;; NOTE: We need to set ::db/remove-deleted to false because demo profiles
-  ;; are created with a set deleted-at value
-  (-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false))
+  (-> (db/get-by-id conn :profile id opts)
       (decode-row)))
 
 ;; --- MUTATION: Update Profile (own)
@@ -130,7 +131,9 @@
   ;; NOTE: we need to retrieve the profile independently if we use
   ;; it or not for explicit locking and avoid concurrent updates of
   ;; the same row/object.
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
+                    (decode-row))
+
         ;; Update the profile map with direct params
         profile (-> profile
                     (assoc :fullname fullname)
@@ -140,9 +143,9 @@
     (db/update! conn :profile
                 {:fullname fullname
                  :lang lang
-                 :theme theme}
-                {:id profile-id}
-                {::db/return-keys false})
+                 :theme theme
+                 :props (db/tjson (:props profile))}
+                {:id profile-id})
 
     (-> profile
         (strip-private-attrs)
@@ -191,7 +194,7 @@
   [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
   (let [profile (db/get-by-id conn :profile profile-id ::sql/for-update true)]
     (when (and (not= (:password profile) "!")
-               (not (:valid (auth/verify-password old-password (:password profile)))))
+               (not (:valid (verify-password cfg old-password (:password profile)))))
       (ex/raise :type :validation
                 :code :old-password-not-match))
     profile))
@@ -200,7 +203,7 @@
   [{:keys [::db/conn] :as cfg} {:keys [id password] :as profile}]
   (when-not (db/read-only? conn)
     (db/update! conn :profile
-                {:password (auth/derive-password password)}
+                {:password (derive-password cfg password)}
                 {:id id})
     nil))
 
@@ -225,22 +228,21 @@
 
 (defn- update-notifications!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
-  (let [profile
-        (get-profile conn profile-id ::db/for-update true)
+  (let [profile (get-profile conn profile-id)
 
         notifications
         {:dashboard-comments dashboard-comments
          :email-comments email-comments
-         :email-invites email-invites}
+         :email-invites email-invites}]
 
-        props
-        (-> (get profile :props)
-            (assoc :notifications notifications))]
+    (db/update!
+     conn :profile
+     {:props
+      (-> (:props profile)
+          (assoc :notifications notifications)
+          (db/tjson))}
+     {:id (:id profile)})
 
-    (db/update! conn :profile
-                {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
     nil))
 
 ;; --- MUTATION: Update Photo
@@ -251,7 +253,7 @@
 (def ^:private
   schema:update-profile-photo
   [:map {:title "update-profile-photo"}
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-profile-photo
   {:doc/added "1.1"
@@ -302,11 +304,12 @@
      :content-type (:mtype thumb)}))
 
 (defn upload-photo
-  [{:keys [::sto/storage] :as cfg} {:keys [file] :as params}]
+  [{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file] :as params}]
   (let [params (-> cfg
                    (assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
                                        [:process-image/global]])
                    (assoc ::climit/label "upload-photo")
+                   (assoc ::climit/executor executor)
                    (climit/invoke! generate-thumbnail! file))]
     (sto/put-object! storage params)))
 
@@ -347,15 +350,15 @@
 
 (defn- request-email-change!
   [{:keys [::db/conn] :as cfg} {:keys [profile email] :as params}]
-  (let [token   (tokens/generate cfg
+  (let [token   (tokens/generate (::setup/props cfg)
                                  {:iss :change-email
-                                  :exp (ct/in-future "15m")
+                                  :exp (dt/in-future "15m")
                                   :profile-id (:id profile)
                                   :email email})
-        ptoken  (tokens/generate cfg
+        ptoken  (tokens/generate (::setup/props cfg)
                                  {:iss :profile-identity
                                   :profile-id (:id profile)
-                                  :exp (ct/in-future {:days 30})})]
+                                  :exp (dt/in-future {:days 30})})]
 
     (when (not= email (:email profile))
       (check-profile-existence! conn params))
@@ -408,7 +411,7 @@
 
 (defn update-profile-props
   [{:keys [::db/conn] :as cfg} profile-id props]
-  (let [profile (get-profile conn profile-id ::db/for-update true)
+  (let [profile (get-profile conn profile-id ::sql/for-update true)
         props   (reduce-kv (fn [props k v]
                              ;; We don't accept namespaced keys
                              (if (simple-ident? k)
@@ -421,17 +424,16 @@
 
     (db/update! conn :profile
                 {:props (db/tjson props)}
-                {:id profile-id}
-                {::db/return-keys false})
+                {:id profile-id})
 
     (filter-props props)))
 
 (sv/defmethod ::update-profile-props
   {::doc/added "1.0"
-   ::sm/params schema:update-profile-props
-   ::db/transaction true}
+   ::sm/params schema:update-profile-props}
   [cfg {:keys [::rpc/profile-id props]}]
-  (update-profile-props cfg profile-id props))
+  (db/tx-run! cfg (fn [cfg]
+                    (update-profile-props cfg profile-id props))))
 
 ;; --- MUTATION: Delete Profile
 
@@ -442,7 +444,7 @@
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
   (let [teams      (get-owned-teams conn profile-id)
-        deleted-at (ct/now)]
+        deleted-at (dt/now)]
 
     ;; If we found owned teams with participants, we don't allow
     ;; delete profile until the user properly transfer ownership or
@@ -469,29 +471,6 @@
     (-> (rph/wrap nil)
         (rph/with-transform (session/delete-fn cfg)))))
 
-(def sql:get-subscription-editors
-  "SELECT DISTINCT
-          p.id,
-          p.fullname AS name,
-          p.email AS email
-     FROM team_profile_rel AS tpr1
-     JOIN team as t
-       ON tpr1.team_id = t.id
-     JOIN team_profile_rel AS tpr2
-       ON (tpr1.team_id = tpr2.team_id)
-     JOIN profile AS p
-       ON (tpr2.profile_id = p.id)
-    WHERE tpr1.profile_id = ?
-      AND tpr1.is_owner IS true
-      AND tpr2.can_edit IS true
-      AND t.deleted_at IS NULL")
-
-(sv/defmethod ::get-subscription-usage
-  {::doc/added "2.9"}
-  [cfg {:keys [::rpc/profile-id]}]
-  (let [editors (db/exec! cfg [sql:get-subscription-editors profile-id])]
-    {:editors editors}))
-
 ;; --- HELPERS
 
 (def sql:owned-teams
@@ -501,7 +480,8 @@
         JOIN team AS t ON (t.id = tpr.team_id)
        WHERE tpr.is_owner IS TRUE
          AND tpr.profile_id = ?
-         AND t.deleted_at IS NULL
+         AND (t.deleted_at IS NULL OR
+              t.deleted_at > now())
    )
    SELECT tpr.team_id AS id,
           count(tpr.profile_id) - 1 AS participants
@@ -549,6 +529,15 @@
   [props]
   (into {} (filter (fn [[k _]] (simple-ident? k))) props))
 
+(defn derive-password
+  [{:keys [::wrk/executor]} password]
+  (when password
+    (px/invoke! executor (partial auth/derive-password password))))
+
+(defn verify-password
+  [{:keys [::wrk/executor]} password password-data]
+  (px/invoke! executor (partial auth/verify-password password password-data)))
+
 (defn decode-row
   [{:keys [props] :as row}]
   (cond-> row

backend/src/app/rpc/commands/projects.clj

@@ -9,10 +9,8 @@
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.db :as db]
    [app.db.sql :as-alias sql]
-   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as webhooks]
    [app.rpc :as-alias rpc]
@@ -22,6 +20,7 @@
    [app.rpc.permissions :as perms]
    [app.rpc.quotes :as quotes]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]))
 
 ;; --- Check Project Permissions
@@ -218,7 +217,7 @@
 (sv/defmethod ::update-project-pin
   {::doc/added "1.18"
    ::sm/params schema:update-project-pin
-   ::webhooks/batch-timeout (ct/duration "5s")
+   ::webhooks/batch-timeout (dt/duration "5s")
    ::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
    ::webhooks/event? true
    ::db/transaction true}
@@ -254,10 +253,9 @@
 ;; --- MUTATION: Delete Project
 
 (defn- delete-project
-  [conn team project-id]
-  (let [delay   (ldel/get-deletion-delay team)
-        project (db/update! conn :project
-                            {:deleted-at (ct/in-future delay)}
+  [conn project-id]
+  (let [project (db/update! conn :project
+                            {:deleted-at (dt/now)}
                             {:id project-id}
                             {::db/return-keys true})]
 
@@ -274,6 +272,7 @@
 
     project))
 
+
 (def ^:private schema:delete-project
   [:map {:title "delete-project"}
    [:id ::sm/uuid]])
@@ -285,10 +284,7 @@
    ::db/transaction true}
   [{:keys [::db/conn]} {:keys [::rpc/profile-id id] :as params}]
   (check-edition-permissions! conn profile-id id)
-  (let [team    (teams/get-team conn
-                                :profile-id profile-id
-                                :project-id id)
-        project (delete-project conn team id)]
+  (let [project (delete-project conn id)]
     (rph/with-meta (rph/wrap)
       {::audit/props {:team-id (:team-id project)
                       :name (:name project)

backend/src/app/rpc/commands/teams.clj

@@ -11,14 +11,12 @@
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
-   [app.common.types.team :as types.team]
+   [app.common.types.team :as tt]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.email :as eml]
-   [app.features.logical-deletion :as ldel]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.media :as media]
@@ -31,20 +29,21 @@
    [app.setup :as-alias setup]
    [app.storage :as sto]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.set :as set]))
 
 ;; --- Helpers & Specs
 
 (def ^:private sql:team-permissions
-  "SELECT tpr.is_owner,
+  "select tpr.is_owner,
           tpr.is_admin,
           tpr.can_edit
-     FROM team_profile_rel AS tpr
-     JOIN team AS t ON (t.id = tpr.team_id)
-    WHERE tpr.profile_id = ?
-      AND tpr.team_id = ?
-      AND t.deleted_at IS NULL")
+     from team_profile_rel as tpr
+     join team as t on (t.id = tpr.team_id)
+    where tpr.profile_id = ?
+      and tpr.team_id = ?
+      and t.deleted_at is null")
 
 (defn get-permissions
   [conn profile-id team-id]
@@ -78,10 +77,9 @@
 
 (defn decode-row
   [{:keys [features subscription] :as row}]
-  (when row
-    (cond-> row
-      (some? features) (assoc :features (db/decode-pgarray features #{}))
-      (some? subscription) (assoc :subscription (db/decode-transit-pgobject subscription)))))
+  (cond-> row
+    (some? features) (assoc :features (db/decode-pgarray features #{}))
+    (some? subscription) (assoc :subscription (db/decode-transit-pgobject subscription))))
 
 ;; FIXME: move
 
@@ -116,6 +114,18 @@
 
 ;; --- Query: Teams
 
+(declare get-teams)
+
+(def ^:private schema:get-teams
+  [:map {:title "get-teams"}])
+
+(sv/defmethod ::get-teams
+  {::doc/added "1.17"
+   ::sm/params schema:get-teams}
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+  (dm/with-open [conn (db/open pool)]
+    (get-teams conn profile-id)))
+
 (def sql:get-teams-with-permissions
   "SELECT t.*,
           tp.is_owner,
@@ -140,8 +150,7 @@
             '~:status', CASE COALESCE(p.props->'~:subscription'->>'~:type', 'professional')
                           WHEN 'professional' THEN 'active'
                           ELSE COALESCE(p.props->'~:subscription'->>'~:status', 'incomplete')
-                       END,
-            '~:seats', p.props->'~:subscription'->'~:quantity'
+                       END
           ) AS subscription
      FROM team_profile_rel AS tp
      JOIN team AS t ON (t.id = tp.team_id)
@@ -151,7 +160,7 @@
        ON (tpr.profile_id = p.id)
     WHERE t.deleted_at IS null
       AND tp.profile_id = ?
-    ORDER BY tp.created_at ASC")
+    ORDER BY tp.created_at ASC;")
 
 (defn process-permissions
   [team]
@@ -182,38 +191,6 @@
     (->> (db/exec! conn [sql (:default-team-id profile) profile-id])
          (into [] xform:process-teams))))
 
-(def ^:private schema:get-teams
-  [:map {:title "get-teams"}])
-
-(sv/defmethod ::get-teams
-  {::doc/added "1.17"
-   ::sm/params schema:get-teams}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
-  (dm/with-open [conn (db/open pool)]
-    (get-teams conn profile-id)))
-
-(def ^:private sql:get-owned-teams
-  "SELECT t.id, t.name,
-          (SELECT count(*) FROM team_profile_rel WHERE team_id=t.id) AS total_members,
-          (SELECT count(*) FROM team_profile_rel WHERE team_id=t.id AND can_edit=true) AS total_editors
-     FROM team AS t
-     JOIN team_profile_rel AS tpr ON (tpr.team_id = t.id)
-    WHERE t.is_default IS false
-      AND tpr.is_owner IS true
-      AND tpr.profile_id = ?
-      AND t.deleted_at IS NULL")
-
-(defn- get-owned-teams
-  [cfg profile-id]
-  (->> (db/exec! cfg [sql:get-owned-teams profile-id])
-       (into [] (map decode-row))))
-
-(sv/defmethod ::get-owned-teams
-  {::doc/added "2.8.0"
-   ::sm/params schema:get-teams}
-  [cfg {:keys [::rpc/profile-id]}]
-  (get-owned-teams cfg profile-id))
-
 ;; --- Query: Team (by ID)
 
 (declare get-team)
@@ -237,43 +214,39 @@
 (defn get-team
   [conn & {:keys [profile-id team-id project-id file-id] :as params}]
 
-  (assert (uuid? profile-id) "profile-id is mandatory")
-  (assert (or (db/connection? conn)
-              (db/pool? conn))
-          "connection or pool is mandatory")
+  (dm/assert!
+   "connection or pool is mandatory"
+   (or (db/connection? conn)
+       (db/pool? conn)))
 
-  (let [{:keys [default-team-id] :as profile}
-        (profile/get-profile conn profile-id)
+  (dm/assert!
+   "profile-id is mandatory"
+   (uuid? profile-id))
 
-        sql
-        (if (contains? cf/flags :subscriptions)
-          sql:get-teams-with-permissions-and-subscription
-          sql:get-teams-with-permissions)
+  (let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
+        result (cond
+                 (some? team-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
+                                ") SELECT * FROM teams WHERE id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id team-id]))
 
-        result
-        (cond
-          (some? team-id)
-          (let [sql (str "WITH teams AS (" sql ") "
-                         "SELECT * FROM teams WHERE id=?")]
-            (db/exec-one! conn [sql default-team-id profile-id team-id]))
+                 (some? project-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
+                                "SELECT t.* FROM teams AS t "
+                                "  JOIN project AS p ON (p.team_id = t.id) "
+                                " WHERE p.id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id project-id]))
 
-          (some? project-id)
-          (let [sql (str "WITH teams AS (" sql ") "
-                         "SELECT t.* FROM teams AS t "
-                         "  JOIN project AS p ON (p.team_id = t.id) "
-                         " WHERE p.id=?")]
-            (db/exec-one! conn [sql default-team-id profile-id project-id]))
+                 (some? file-id)
+                 (let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
+                                "SELECT t.* FROM teams AS t "
+                                "  JOIN project AS p ON (p.team_id = t.id) "
+                                "  JOIN file AS f ON (f.project_id = p.id) "
+                                " WHERE f.id=?")]
+                   (db/exec-one! conn [sql default-team-id profile-id file-id]))
 
-          (some? file-id)
-          (let [sql (str "WITH teams AS (" sql ") "
-                         "SELECT t.* FROM teams AS t "
-                         "  JOIN project AS p ON (p.team_id = t.id) "
-                         "  JOIN file AS f ON (f.project_id = p.id) "
-                         " WHERE f.id=?")]
-            (db/exec-one! conn [sql default-team-id profile-id file-id]))
-
-          :else
-          (throw (IllegalArgumentException. "invalid arguments")))]
+                 :else
+                 (throw (IllegalArgumentException. "invalid arguments")))]
 
     (when-not result
       (ex/raise :type :not-found
@@ -443,18 +416,13 @@
    [:team-id ::sm/uuid]])
 
 (def sql:team-invitations
-  "SELECT email_to AS email,
-          role,
-          (valid_until < ?::timestamptz) AS expired
-     FROM team_invitation
-    WHERE team_id = ?
-    ORDER BY valid_until DESC, created_at DESC")
+  "select email_to as email, role, (valid_until < now()) as expired
+   from team_invitation where team_id = ? order by valid_until desc, created_at desc")
 
 (defn get-team-invitations
   [conn team-id]
-  (let [now (ct/now)]
-    (->> (db/exec! conn [sql:team-invitations now team-id])
-         (mapv #(update % :role keyword)))))
+  (->> (db/exec! conn [sql:team-invitations team-id])
+       (mapv #(update % :role keyword))))
 
 (sv/defmethod ::get-team-invitations
   {::doc/added "1.17"
@@ -467,12 +435,11 @@
 
 ;; --- COMMAND QUERY: get-team-info
 
-(defn get-team-info
+(defn- get-team-info
   [{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
-  (-> (db/get* conn :team
-               {:id id}
-               {::sql/columns [:id :is-default :features]})
-      (decode-row)))
+  (db/get* conn :team
+           {:id id}
+           {::sql/columns [:id :is-default]}))
 
 (sv/defmethod ::get-team-info
   "Retrieve minimal team info by its ID."
@@ -508,7 +475,7 @@
 
   (let [features (-> (cfeat/get-enabled-features cf/flags)
                      (set/difference cfeat/frontend-only-features)
-                     (set/difference cfeat/no-team-inheritable-features))
+                     (cfeat/check-client-features! (:features params)))
         params   (-> params
                      (assoc :profile-id profile-id)
                      (assoc :features features))
@@ -634,7 +601,7 @@
 
         ;; assign owner role to new profile
         (db/update! conn :team-profile-rel
-                    (get types.team/permissions-for-role :owner)
+                    (get tt/permissions-for-role :owner)
                     {:team-id id :profile-id reassign-to}))
 
       ;; and finally, if all other conditions does not match and the
@@ -667,13 +634,13 @@
 
 (defn- delete-team
   "Mark a team for deletion"
-  [conn {:keys [id] :as team}]
+  [conn team-id]
 
-  (let [delay (ldel/get-deletion-delay team)
-        team  (db/update! conn :team
-                          {:deleted-at (ct/in-future delay)}
-                          {:id id}
-                          {::db/return-keys true})]
+  (let [deleted-at (dt/now)
+        team       (db/update! conn :team
+                               {:deleted-at deleted-at}
+                               {:id team-id}
+                               {::db/return-keys true})]
 
     (when (:is-default team)
       (ex/raise :type :validation
@@ -683,8 +650,8 @@
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :team
-                                :deleted-at (:deleted-at team)
-                                :id id}})
+                                :deleted-at deleted-at
+                                :id team-id}})
     team))
 
 (def ^:private schema:delete-team
@@ -696,14 +663,12 @@
    ::sm/params schema:delete-team
    ::db/transaction true}
   [{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
-  (let [team  (get-team conn :profile-id profile-id :team-id id)
-        perms (get team :permissions)]
-
+  (let [perms (get-permissions conn profile-id id)]
     (when-not (:is-owner perms)
       (ex/raise :type :validation
                 :code :only-owner-can-delete-team))
 
-    (delete-team conn team)
+    (delete-team conn id)
     nil))
 
 ;; --- Mutation: Team Update Role
@@ -747,7 +712,7 @@
                          :team-id team-id
                          :role role})
 
-    (let [params (get types.team/permissions-for-role role)]
+    (let [params (get tt/permissions-for-role role)]
       ;; Only allow single owner on team
       (when (= role :owner)
         (db/update! conn :team-profile-rel
@@ -765,7 +730,7 @@
   [:map {:title "update-team-member-role"}
    [:team-id ::sm/uuid]
    [:member-id ::sm/uuid]
-   [:role types.team/schema:role]])
+   [:role ::tt/role]])
 
 (sv/defmethod ::update-team-member-role
   {::doc/added "1.17"
@@ -815,7 +780,7 @@
 (def ^:private schema:update-team-photo
   [:map {:title "update-team-photo"}
    [:team-id ::sm/uuid]
-   [:file media/schema:upload]])
+   [:file ::media/upload]])
 
 (sv/defmethod ::update-team-photo
   {::doc/added "1.17"

backend/src/app/rpc/commands/teams_invitations.clj

@@ -6,14 +6,12 @@
 
 (ns app.rpc.commands.teams-invitations
   (:require
-   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.types.team :as types.team]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -22,6 +20,7 @@
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.rpc :as-alias rpc]
+   [app.rpc.commands.files :as files]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
@@ -30,6 +29,7 @@
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [cuerdas.core :as str]))
 
 ;; --- Mutation: Create Team Invitation
@@ -43,7 +43,7 @@
 
 (defn- create-invitation-token
   [cfg {:keys [profile-id valid-until team-id member-id member-email role]}]
-  (tokens/generate cfg
+  (tokens/generate (::setup/props cfg)
                    {:iss :team-invitation
                     :exp valid-until
                     :profile-id profile-id
@@ -54,11 +54,15 @@
 
 (defn- create-profile-identity-token
   [cfg profile-id]
-  (assert (uuid? profile-id) "expected valid uuid for profile-id")
-  (tokens/generate cfg
+
+  (dm/assert!
+   "expected valid uuid for profile-id"
+   (uuid? profile-id))
+
+  (tokens/generate (::setup/props cfg)
                    {:iss :profile-identity
                     :profile-id profile-id
-                    :exp (ct/in-future {:days 30})}))
+                    :exp (dt/in-future {:days 30})}))
 
 (def ^:private schema:create-invitation
   [:map {:title "params:create-invitation"}
@@ -71,7 +75,7 @@
     [:map
      [:id ::sm/uuid]
      [:fullname :string]]]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:email ::sm/email]])
 
 (def ^:private check-create-invitation-params
@@ -122,7 +126,7 @@
         (teams/check-email-spam conn email true)
 
         (let [id         (uuid/next)
-              expire     (ct/in-future "168h") ;; 7 days
+              expire     (dt/in-future "168h") ;; 7 days
               invitation (db/exec-one! conn [sql:upsert-team-invitation id
                                              (:id team) (str/lower email)
                                              (:id profile)
@@ -220,112 +224,62 @@
 (def ^:private xf:map-email (map :email))
 
 (defn- create-team-invitations
-  "Unified function to handle both create and resend team invitations.
-  Accepts either:
-  - emails (set) + role (single role for all emails)
-  - invitations (vector of {:email :role} maps)"
-  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails invitations] :as params}]
-  (let [;; Normalize input to a consistent format: [{:email :role}]
-        invitation-data  (cond
-                           ;; Case 1: emails + single role (create invitations style)
-                           (and emails role)
-                           (map (fn [email] {:email email :role role}) emails)
+  [{:keys [::db/conn] :as cfg} {:keys [profile team role emails] :as params}]
+  (let [emails           (set emails)
 
-                           ;; Case 2: invitations with individual roles (resend invitations style)
-                           (some? invitations)
-                           invitations
+        join-requests    (->> (get-valid-access-request-profiles conn (:id team))
+                              (d/index-by :email))
 
-                           :else
-                           (throw (ex-info "Invalid parameters: must provide either emails+role or invitations" {})))
+        team-members     (into #{} xf:map-email
+                               (teams/get-team-members conn (:id team)))
 
-        invitation-emails (into #{} (map :email) invitation-data)
-
-        join-requests     (->> (get-valid-access-request-profiles conn (:id team))
-                               (d/index-by :email))
-
-        team-members      (into #{} xf:map-email
-                                (teams/get-team-members conn (:id team)))
-
-        invitations       (into #{}
-                                (comp
-                                ;; We don't re-send invitations to
-                                ;; already existing members
-                                 (remove #(contains? team-members (:email %)))
+        invitations      (into #{}
+                               (comp
+                                ;;  We don't re-send inviation to
+                                ;;  already existing members
+                                (remove team-members)
                                 ;; We don't send invitations to
                                 ;; join-requested members
-                                 (remove #(contains? join-requests (:email %)))
-                                 (map (fn [{:keys [email role]}]
-                                        (create-invitation cfg
-                                                           (-> params
-                                                               (assoc :email email)
-                                                               (assoc :role role)))))
-                                 (remove nil?))
-                                invitation-data)]
+                                (remove join-requests)
+                                (map (fn [email] (assoc params :email email)))
+                                (keep (partial create-invitation cfg)))
+                               emails)]
 
     ;; For requested invitations, do not send invitation emails, add
     ;; the user directly to the team
     (->> join-requests
-         (filter #(contains? invitation-emails (key %)))
-         (map (fn [[email member]]
-                (let [role (:role (first (filter #(= (:email %) email) invitation-data)))]
-                  (add-member-to-team conn profile team role member))))
-         (doall))
+         (filter #(contains? emails (key %)))
+         (map val)
+         (run! (partial add-member-to-team conn profile team role)))
 
     invitations))
 
 (def ^:private schema:create-team-invitations
-  [:and
-   [:map {:title "create-team-invitations"}
-    [:team-id ::sm/uuid]
-    ;; Support both formats:
-    ;; 1. emails (set) + role (single role for all)
-    ;; 2. invitations (vector of {:email :role} maps)
-    [:emails {:optional true} [::sm/set ::sm/email]]
-    [:role {:optional true} types.team/schema:role]
-    [:invitations {:optional true} [:vector [:map
-                                             [:email ::sm/email]
-                                             [:role types.team/schema:role]]]]]
-
-   ;; Ensure exactly one format is provided
-   [:fn (fn [params]
-          (let [has-emails-role (and (contains? params :emails)
-                                     (contains? params :role))
-                has-invitations (contains? params :invitations)]
-            (and (or has-emails-role has-invitations)
-                 (not (and has-emails-role has-invitations)))))]])
+  [:map {:title "create-team-invitations"}
+   [:team-id ::sm/uuid]
+   [:role ::types.team/role]
+   [:emails [::sm/set ::sm/email]]])
 
 (def ^:private max-invitations-by-request-threshold
   "The number of invitations can be sent in a single rpc request"
   25)
 
 (sv/defmethod ::create-team-invitations
-  "A rpc call that allows to send single or multiple invitations to join the team.
-
-  Supports two parameter formats:
-  1. emails (set) + role (single role for all emails)
-  2. invitations (vector of {:email :role} maps for individual roles)"
+  "A rpc call that allow to send a single or multiple invitations to
+  join the team."
   {::doc/added "1.17"
    ::doc/module :teams
    ::sm/params schema:create-team-invitations}
-  [cfg {:keys [::rpc/profile-id team-id role emails] :as params}]
+  [cfg {:keys [::rpc/profile-id team-id emails] :as params}]
   (let [perms    (teams/get-permissions cfg profile-id team-id)
         profile  (db/get-by-id cfg :profile profile-id)
-        ;; Determine which format is being used
-        using-emails-format? (and emails role)
-        ;; Handle both parameter formats
-        emails   (if using-emails-format?
-                   (into #{} (map profile/clean-email) emails)
-                   #{})
-        ;; Calculate total invitation count for both formats
-        invitation-count (if using-emails-format?
-                           (count emails)
-                           (count (:invitations params)))]
+        emails   (into #{} (map profile/clean-email) emails)]
 
     (when-not (:is-admin perms)
       (ex/raise :type :validation
                 :code :insufficient-permissions))
 
-    (when (> invitation-count max-invitations-by-request-threshold)
+    (when (> (count emails) max-invitations-by-request-threshold)
       (ex/raise :type :validation
                 :code :max-invitations-by-request
                 :hint "the maximum of invitation on single request is reached"
@@ -334,7 +288,7 @@
     (-> cfg
         (assoc ::quotes/profile-id profile-id)
         (assoc ::quotes/team-id team-id)
-        (assoc ::quotes/incr invitation-count)
+        (assoc ::quotes/incr (count emails))
         (quotes/check! {::quotes/id ::quotes/invitations-per-team}
                        {::quotes/id ::quotes/profiles-per-team}))
 
@@ -350,12 +304,7 @@
                                   (-> params
                                       (assoc :profile profile)
                                       (assoc :team team)
-                                      ;; Pass parameters in the correct format for the unified function
-                                      (cond-> using-emails-format?
-                                        ;; If using emails+role format, ensure both are present
-                                        (assoc :emails emails :role role)
-                                        ;; If using invitations format, the :invitations key is already in params
-                                        (not using-emails-format?) identity)))]
+                                      (assoc :emails emails)))]
 
       (with-meta {:total (count invitations)
                   :invitations invitations}
@@ -369,7 +318,7 @@
    [:features {:optional true} ::cfeat/features]
    [:id {:optional true} ::sm/uuid]
    [:emails [::sm/set ::sm/email]]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::create-team-with-invitations
   {::doc/added "1.17"
@@ -454,7 +403,7 @@
   [:map {:title "update-team-invitation-role"}
    [:team-id ::sm/uuid]
    [:email ::sm/email]
-   [:role types.team/schema:role]])
+   [:role ::types.team/role]])
 
 (sv/defmethod ::update-team-invitation-role
   {::doc/added "1.17"
@@ -469,7 +418,7 @@
                 :code :insufficient-permissions))
 
     (db/update! conn :team-invitation
-                {:role (name role) :updated-at (ct/now)}
+                {:role (name role) :updated-at (dt/now)}
                 {:team-id team-id :email-to (profile/clean-email email)})
 
     nil))
@@ -518,11 +467,11 @@
 
 (defn- check-existing-team-access-request
   "Checks if an existing team access request is still valid"
-  [{:keys [::db/conn]} team-id profile-id]
+  [conn team-id profile-id]
   (when-let [request (db/get* conn :team-access-request
                               {:team-id team-id
                                :requester-id profile-id})]
-    (when (ct/is-after? (:valid-until request) (ct/now))
+    (when (dt/is-after? (:valid-until request) (dt/now))
       (ex/raise :type :validation
                 :code :request-already-sent
                 :hint "you have already made a request to join this team less than 24 hours ago"))))
@@ -536,10 +485,10 @@
 
 (defn- upsert-team-access-request
   "Create or update team access request for provided team and profile-id"
-  [{:keys [::db/conn] :as cfg} team-id requester-id]
-  (check-existing-team-access-request cfg team-id requester-id)
-  (let [valid-until     (ct/in-future {:hours 24})
-        auto-join-until (ct/in-future {:days 7})
+  [conn team-id requester-id]
+  (check-existing-team-access-request conn team-id requester-id)
+  (let [valid-until     (dt/in-future {:hours 24})
+        auto-join-until (dt/in-future {:days 7})
         request-id      (uuid/next)]
     (db/exec-one! conn [sql:upsert-team-access-request
                         request-id team-id requester-id
@@ -550,7 +499,7 @@
   "A specific method for obtain a file with name and page-id used for
   team request access procediment"
   [cfg file-id]
-  (let [file (bfc/get-file cfg file-id :migrate? false)]
+  (let [file (files/get-file cfg file-id :migrate? false)]
     (-> file
         (dissoc :data)
         (dissoc :deleted-at)
@@ -599,7 +548,7 @@
     (teams/check-email-bounce conn (:email team-owner) false)
     (teams/check-email-spam conn (:email team-owner) true)
 
-    (let [request (upsert-team-access-request cfg team-id profile-id)
+    (let [request (upsert-team-access-request conn team-id profile-id)
           factory (cond
                     (and (some? file) (:is-default team) is-viewer)
                     eml/request-file-access-yourpenpot-view

backend/src/app/rpc/commands/verify_token.clj

@@ -8,7 +8,6 @@
   (:require
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.types.team :as types.team]
    [app.config :as cf]
    [app.db :as db]
@@ -24,7 +23,8 @@
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
    [app.tokens.spec.team-invitation :as-alias spec.team-invitation]
-   [app.util.services :as sv]))
+   [app.util.services :as sv]
+   [app.util.time :as dt]))
 
 (defmulti process-token (fn [_ _ claims] (:iss claims)))
 
@@ -38,7 +38,7 @@
    ::doc/module :auth
    ::sm/params schema:verify-token}
   [cfg {:keys [token] :as params}]
-  (let [claims (tokens/verify cfg {:token token})]
+  (let [claims (tokens/verify (::setup/props cfg) {:token token})]
     (db/tx-run! cfg process-token params claims)))
 
 (defmethod process-token :change-email
@@ -126,9 +126,9 @@
 (def schema:team-invitation-claims
   [:map {:title "TeamInvitationClaims"}
    [:iss :keyword]
-   [:exp ::ct/inst]
+   [:exp ::dt/instant]
    [:profile-id ::sm/uuid]
-   [:role types.team/schema:role]
+   [:role ::types.team/role]
    [:team-id ::sm/uuid]
    [:member-email ::sm/email]
    [:member-id {:optional true} ::sm/uuid]])

backend/src/app/rpc/commands/viewer.clj

@@ -6,7 +6,6 @@
 
 (ns app.rpc.commands.viewer
   (:require
-   [app.binfile.common :as bfc]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.schema :as sm]
@@ -51,7 +50,7 @@
 
 (defn- get-view-only-bundle
   [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
-  (let [file    (bfc/get-file cfg file-id)
+  (let [file    (files/get-file cfg file-id)
 
         project (db/get conn :project
                         {:id (:project-id file)}
@@ -79,9 +78,9 @@
                   :always
                   (update :data select-keys [:id :options :pages :pages-index :components]))
 
-        libs    (->> (bfc/get-file-libraries conn file-id)
+        libs    (->> (files/get-file-libraries conn file-id)
                      (mapv (fn [{:keys [id] :as lib}]
-                             (merge lib (bfc/get-file cfg id)))))
+                             (merge lib (files/get-file cfg id)))))
 
         links   (->> (db/query conn :share-link {:file-id file-id})
                      (mapv (fn [row]

backend/src/app/rpc/commands/webhooks.clj

@@ -9,7 +9,6 @@
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
-   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.common.uuid :as uuid]
    [app.db :as db]
@@ -20,6 +19,7 @@
    [app.rpc.doc :as-alias doc]
    [app.rpc.permissions :as perms]
    [app.util.services :as sv]
+   [app.util.time :as dt]
    [cuerdas.core :as str]))
 
 (defn get-webhooks-permissions
@@ -54,7 +54,7 @@
                     (http/req! cfg
                                {:method :head
                                 :uri (str (:uri params))
-                                :timeout (ct/duration "3s")}
+                                :timeout (dt/duration "3s")}
                                {:sync? true}))]
       (if (ex/exception? response)
         (if-let [hint (webhooks/interpret-exception response)]

backend/src/app/rpc/doc.clj

@@ -9,7 +9,6 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.json :as json]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
    [app.common.schema.desc-js-like :as smdj]
@@ -20,6 +19,7 @@
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
    [app.rpc :as-alias rpc]
+   [app.util.json :as json]
    [app.util.services :as sv]
    [app.util.template :as tmpl]
    [clojure.java.io :as io]
@@ -86,7 +86,7 @@
     (fn [request]
       (let [params  (:query-params request)
             pstyle  (:type params "js")
-            context (assoc @context :param-style pstyle)]
+            context (assoc context :param-style pstyle)]
 
         {::yres/status 200
          ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
@@ -166,6 +166,9 @@
      :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
                 ;; :description "penpot backend"
                 }]
+     :security
+     {:api_key []}
+
      :paths paths
      :components {:schemas @definitions}}))
 
@@ -175,7 +178,8 @@
     (fn [_]
       {::yres/status 200
        ::yres/headers {"content-type" "application/json; charset=utf-8"}
-       ::yres/body (json/encode @context)})
+       ::yres/body (json/encode context)})
+
     (fn [_]
       {::yres/status 404})))
 
@@ -205,7 +209,7 @@
 
 (defmethod ig/init-key ::routes
   [_ {:keys [::rpc/methods] :as cfg}]
-  [(let [context (delay (prepare-doc-context methods))]
+  [(let [context (prepare-doc-context methods)]
      [["/_doc"
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]
@@ -213,7 +217,7 @@
        {:handler (doc-handler context)
         :allowed-methods #{:get}}]])
 
-   (let [context (delay (prepare-openapi-context methods))]
+   (let [context (prepare-openapi-context methods)]
      [["/openapi"
        {:handler (openapi-handler)
         :allowed-methods #{:get}}]