🐛 Add proper input checking to font related RCP method

🐛 Fix issue with pdf render on exporter

.circleci/config.yml

@@ -1,342 +0,0 @@
-version: 2.1
-
-jobs:
-  lint:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    steps:
-      - checkout
-
-      - run:
-          name: "fmt check"
-          working_directory: "."
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-
-      - run:
-          name: "lint clj common"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:common
-
-      - run:
-          name: "lint clj frontend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:frontend
-
-      - run:
-          name: "lint clj backend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:backend
-
-      - run:
-          name: "lint clj exporter"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:exporter
-
-      - run:
-          name: "lint clj library"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:library
-
-  test-common:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
-
-      - run:
-          name: "JVM tests"
-          working_directory: "./common"
-          command: |
-            clojure -M:dev:test
-
-      - run:
-          name: "NODE tests"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run test
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
-
-  test-frontend:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: "install dependencies"
-          working_directory: "./frontend"
-          # We install playwright here because the dependent tasks
-          # uses the same cache as this task so we prepopulate it
-          command: |
-            yarn install
-            yarn run playwright install chromium
-
-      - run:
-          name: "lint scss on frontend"
-          working_directory: "./frontend"
-          command: |
-            yarn run lint:scss
-
-      - run:
-          name: "unit tests"
-          working_directory: "./frontend"
-          command: |
-            yarn run test
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-  test-library:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx6g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: Install dependencies and build
-          working_directory: "./library"
-          command: |
-            yarn install
-
-      - run:
-          name: Build and Test
-          working_directory: "./library"
-          command: |
-            ./scripts/build
-            yarn run test
-
-  test-components:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx6g -Xms2g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: Install dependencies
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn run playwright install chromium
-
-      - run:
-          name: Build Storybook
-          working_directory: "./frontend"
-          command: yarn run build:storybook
-
-      - run:
-          name: Serve Storybook and run tests
-          working_directory: "./frontend"
-          command: |
-            npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-              "npx http-server storybook-static --port 6006 --silent" \
-              "npx wait-on tcp:6006 && yarn test:storybook"
-
-  test-integration:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: large
-
-    environment:
-      JAVA_OPTS: -Xmx6g -Xms2g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: "integration tests"
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn run build:app:assets
-            yarn run build:app
-            yarn run build:app:libs
-            yarn run playwright install chromium
-            yarn run test:e2e -x --workers=4
-
-  test-backend:
-    docker:
-      - image: penpotapp/devenv:latest
-      - image: cimg/postgres:14.5
-        environment:
-          POSTGRES_USER: penpot_test
-          POSTGRES_PASSWORD: penpot_test
-          POSTGRES_DB: penpot_test
-      - image: cimg/redis:7.0.5
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "backend/deps.edn" }}
-
-      - run:
-          name: "tests"
-          working_directory: "./backend"
-          command: |
-            clojure -M:dev:test --reporter kaocha.report/documentation
-
-          environment:
-            PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
-            PENPOT_TEST_DATABASE_USERNAME: penpot_test
-            PENPOT_TEST_DATABASE_PASSWORD: penpot_test
-            PENPOT_TEST_REDIS_URI: "redis://localhost/1"
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.gitlibs
-          key: v1-dependencies-{{ checksum "backend/deps.edn" }}
-
-  test-render-wasm:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-    environment:
-
-    steps:
-      - checkout
-
-      - run:
-          name: "fmt check"
-          working_directory: "./render-wasm"
-          command: |
-            cargo fmt --check
-
-      - run:
-          name: "lint"
-          working_directory: "./render-wasm"
-          command: |
-            ./lint
-
-      - run:
-          name: "cargo tests"
-          working_directory: "./render-wasm"
-          command: |
-            ./test
-
-workflows:
-  penpot:
-    jobs:
-      - lint
-      - test-frontend:
-          requires:
-            - lint: success
-
-      - test-library:
-          requires:
-            - test-frontend: success
-            - lint: success
-
-      - test-components:
-          requires:
-            - test-frontend: success
-            - lint: success
-
-      - test-integration:
-          requires:
-            - test-frontend: success
-            - lint: success
-
-      - test-backend:
-          requires:
-            - lint: success
-
-      - test-common:
-          requires:
-            - lint: success
-
-      - test-render-wasm

.clj-kondo/config.edn

@@ -45,10 +45,25 @@
   :potok/reify-type
   {:level :error}
 
+  :redundant-primitive-coercion
+  {:level :off}
+
+  :unused-excluded-var
+  {:level :off}
+
+  :unresolved-excluded-var
+  {:level :off}
+
+  :missing-protocol-method
+  {:level :off}
+
   :unresolved-namespace
   {:level :warning
    :exclude [data_readers]}
 
+  :unused-value
+  {:level :off}
+
   :single-key-in
   {:level :warning}
 
@@ -64,6 +79,9 @@
   :redundant-nested-call
   {:level :off}
 
+  :redundant-str-call
+  {:level :off}
+
   :earmuffed-var-not-dynamic
   {:level :off}
 

.github/ISSUE_TEMPLATE/new-render-bug-report.md

@@ -0,0 +1,38 @@
+---
+name: New Render Bug Report
+about: Create a report about the bugs you have found in the new render
+title: ''
+labels: new render
+assignees: claragvinola
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Steps to Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots or screen recordings**
+If applicable, add screenshots or screen recording to help illustrate your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,6 +13,7 @@
 - [ ] Add a detailed explanation of how to reproduce the issue and/or verify the fix, if applicable.
 - [ ] Include screenshots or videos, if applicable.
 - [ ] Add or modify existing integration tests in case of bugs or new features, if applicable.
+- [ ] Refactor any modified SCSS files following the refactor guide.
 - [ ] Check CI passes successfully.
 - [ ] Update the `CHANGES.md` file, referencing the related GitHub issue, if applicable.
 

.github/workflows/build-bundle.yml

@@ -0,0 +1,93 @@
+name: Bundles Builder
+
+on:
+  # Create bundle from manual action
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch or ref'
+        type: string
+        required: true
+        default: 'develop'
+      build_wasm:
+        description: 'BUILD_WASM. Valid values: yes, no'
+        type: string
+        required: false
+        default: 'yes'
+      build_storybook:
+        description: 'BUILD_STORYBOOK. Valid values: yes, no'
+        type: string
+        required: false
+        default: 'yes'
+  workflow_call:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch or ref'
+        type: string
+        required: true
+        default: 'develop'
+      build_wasm:
+        description: 'BUILD_WASM. Valid values: yes, no'
+        type: string
+        required: false
+        default: 'yes'
+      build_storybook:
+        description: 'BUILD_STORYBOOK. Valid values: yes, no'
+        type: string
+        required: false
+        default: 'yes'
+
+jobs:
+  build-bundle:
+    name: Build and Upload Penpot Bundle
+    runs-on: penpot-runner-01
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gh_ref }}
+
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+          echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT
+
+      - name: Build bundle
+        env:
+          BUILD_WASM: ${{ inputs.build_wasm }}
+          BUILD_STORYBOOK: ${{ inputs.build_storybook }}
+        run: ./manage.sh build-bundle
+
+      - name: Prepare directories for zipping
+        run: |
+          mkdir zips
+          mv bundles penpot
+
+      - name: Create zip bundle
+        run: |
+          echo "📦 Packaging Penpot bundle..."
+          zip -r zips/penpot.zip penpot
+
+      - name: Upload Penpot bundle to S3
+        run: |
+          aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip --metadata bundle-version=${{ steps.vars.outputs.bundle_version }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 📦 *[PENPOT] Error building penpot bundles.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+               Bundle version: `${{ steps.vars.outputs.bundle_version }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/build-develop.yml

@@ -0,0 +1,21 @@
+name: _DEVELOP
+
+on:
+  schedule:
+    - cron: '16 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "develop"
+      build_wasm: "yes"
+      build_storybook: "yes"
+
+  build-docker:
+    needs: build-bundle
+    uses: ./.github/workflows/build-docker.yml
+    secrets: inherit
+    with:
+      gh_ref: "develop"

.github/workflows/build-docker-devenv.yml

@@ -0,0 +1,41 @@
+name: DevEnv Docker Image Builder
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and push DevEnv Docker image
+    environment: release-admins
+    runs-on: penpot-runner-02
+
+    steps:
+      - name: Set common environment variables
+        run: |
+          # Each job execution will use its own docker configuration.
+          echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.PUB_DOCKER_USERNAME }}
+          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
+
+      - name: Build and push DevEnv Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'penpotapp/devenv'
+        with:
+          context: ./docker/devenv/
+          file: ./docker/devenv/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.DOCKER_IMAGE }}:latest
+          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ env.DOCKER_IMAGE }}:buildcache,mode=max

.github/workflows/build-docker.yml

@@ -0,0 +1,183 @@
+name: Docker Images Builder
+
+on:
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch or ref'
+        type: string
+        required: true
+        default: 'develop'
+  workflow_call:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch or ref'
+        type: string
+        required: true
+        default: 'develop'
+
+jobs:
+  build-and-push:
+    name: Build and Push Penpot Docker Images
+    runs-on: penpot-runner-02
+
+    steps:
+      - name: Set common environment variables
+        run: |
+          # Each job execution will use its own docker configuration.
+          echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gh_ref }}
+
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Download Penpot Bundles
+        id: bundles
+        env:
+          FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+        run: |
+          tmp=$(aws s3api head-object \
+            --bucket ${{ secrets.S3_BUCKET }} \
+            --key "$FILE_NAME" \
+            --query 'Metadata."bundle-version"' \
+            --output text)
+          echo "bundle_version=$tmp" >> $GITHUB_OUTPUT
+          pushd docker/images
+          aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
+          unzip $FILE_NAME > /dev/null
+          mv penpot/backend bundle-backend
+          mv penpot/frontend bundle-frontend
+          mv penpot/exporter bundle-exporter
+          mv penpot/storybook bundle-storybook
+          mv penpot/mcp bundle-mcp
+          popd
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # To avoid the “429 Too Many Requests” error when downloading
+      # images from DockerHub for unregistered users.
+      # https://docs.docker.com/docker-hub/usage/
+      - name: Login to DockerHub Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.PUB_DOCKER_USERNAME }}
+          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images:
+            frontend
+            backend
+            exporter
+            storybook
+            mcp
+          labels: |
+            bundle_version=${{ steps.bundles.outputs.bundle_version }}
+
+      - name: Build and push Backend Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'backend'
+          BUNDLE_PATH: './bundle-backend'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.backend
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push Frontend Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'frontend'
+          BUNDLE_PATH: './bundle-frontend'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.frontend
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push Exporter Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'exporter'
+          BUNDLE_PATH: './bundle-exporter'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.exporter
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push Storybook Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'storybook'
+          BUNDLE_PATH: './bundle-storybook'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.storybook
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push MCP Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'mcp'
+          BUNDLE_PATH: './bundle-mcp'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.mcp
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🐳 *[PENPOT] Error building penpot docker images.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            📦 Bundle: `${{ steps.bundles.outputs.bundle_version }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/build-nitrate-module.yml

@@ -0,0 +1,21 @@
+name: _NITRATE MODULE
+
+on:
+  schedule:
+    - cron: '36 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "nitrate-module"
+      build_wasm: "yes"
+      build_storybook: "yes"
+
+  build-docker:
+    needs: build-bundle
+    uses: ./.github/workflows/build-docker.yml
+    secrets: inherit
+    with:
+      gh_ref: "nitrate-module"

.github/workflows/build-staging-render.yml

@@ -0,0 +1,14 @@
+name: _STAGING RENDER
+
+on:
+  schedule:
+    - cron: '36 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "staging-render"
+      build_wasm: "yes"
+      build_storybook: "yes"

.github/workflows/build-staging.yml

@@ -0,0 +1,21 @@
+name: _STAGING
+
+on:
+  schedule:
+    - cron: '36 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "staging"
+      build_wasm: "yes"
+      build_storybook: "yes"
+
+  build-docker:
+    needs: build-bundle
+    uses: ./.github/workflows/build-docker.yml
+    secrets: inherit
+    with:
+      gh_ref: "staging"

.github/workflows/build-tag.yml

@@ -0,0 +1,46 @@
+name: _TAG
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: ${{ github.ref_name }}
+      build_wasm: "yes"
+      build_storybook: "yes"
+
+  build-docker:
+    needs: build-bundle
+    uses: ./.github/workflows/build-docker.yml
+    secrets: inherit
+    with:
+      gh_ref: ${{ github.ref_name }}
+
+  notify:
+    name: Notifications
+    runs-on: ubuntu-24.04
+    needs: build-docker
+
+    steps:
+      - name: Notify Mattermost
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            🐳 *[PENPOT] Docker image available: ${{ github.ref_name }}*
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra
+
+  publish-final-tag:
+    if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta')  && contains(github.ref_name, '.') }}
+    needs: build-docker
+    uses: ./.github/workflows/release.yml
+    secrets: inherit
+    with:
+      gh_ref: ${{ github.ref_name }}

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(Merge|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
+          pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert|Reapply).+[^.])$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/plugins-deploy-api-doc.yml

@@ -0,0 +1,125 @@
+name: Plugins/api-doc deployer
+
+on:
+  push:
+    branches:
+      - develop
+      - staging
+      - main
+    paths:
+      - 'plugins/libs/plugin-types/index.d.ts'
+      - 'plugins/libs/plugin-types/REAME.md'
+      - 'plugins/tools/typedoc.css'
+      - 'plugins/CHANGELOG.md'
+      - 'plugins/wrangler-penpot-plugins-api-doc.toml'
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: choice
+        required: true
+        default: 'develop'
+        options:
+          - develop
+          - staging
+          - main
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.vars.outputs.gh_ref }}
+
+      # START: Setup Node and PNPM enabling cache
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+
+      - name: Enable PNPM
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          corepack enable;
+          corepack install;
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        working-directory: ./plugins
+        shell: bash
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+      # END: Setup Node and PNPM enabling cache
+
+      - name: Install deps
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          pnpm install --no-frozen-lockfile;
+          pnpm add -D -w wrangler@latest;
+
+      - name: Build docs
+        working-directory: plugins
+        shell: bash
+        run: pnpm run build:doc
+
+      - name: Select Worker name
+        run: |
+          REF="${{ steps.vars.outputs.gh_ref }}"
+          case "$REF" in
+            main)
+              echo "WORKER_NAME=penpot-plugins-api-doc-pro" >> $GITHUB_ENV
+              echo "WORKER_URI=doc.plugins.penpot.app" >> $GITHUB_ENV ;;
+            staging)
+              echo "WORKER_NAME=penpot-plugins-api-doc-pre" >> $GITHUB_ENV
+              echo "WORKER_URI=doc.plugins.penpot.dev" >> $GITHUB_ENV ;;
+            develop)
+              echo "WORKER_NAME=penpot-plugins-api-doc-hourly" >> $GITHUB_ENV
+              echo "WORKER_URI=doc.plugins.hourly.penpot.dev" >> $GITHUB_ENV ;;
+            *) echo "Unsupported branch ${REF}" && exit 1 ;;
+          esac
+
+      - name: Set the custom url
+        working-directory: plugins
+        shell: bash
+        run: |
+          sed -i "s/WORKER_URI/${{ env.WORKER_URI }}/g" wrangler-penpot-plugins-api-doc.toml
+
+      - name: Deploy to Cloudflare Workers
+        uses: cloudflare/wrangler-action@v3
+        with:
+          workingDirectory: plugins
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          command: deploy --config wrangler-penpot-plugins-api-doc.toml --name ${{ env.WORKER_NAME }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🧩📚 *[PENPOT PLUGINS] Error deploying API documentation.*
+            📄 Triggered from ref: `${{ inputs.gh_ref }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/plugins-deploy-package.yml

@@ -0,0 +1,127 @@
+name: Plugins/package deployer
+
+on:
+  # Deploy package from manual action
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: choice
+        required: true
+        default: 'develop'
+        options:
+          - develop
+          - staging
+          - main
+      plugin_name:
+        description: 'Pluging name (like plugins/apps/<plugin_name>-plugin)'
+        type: string
+        required: true
+  workflow_call:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: string
+        required: true
+        default: 'develop'
+      plugin_name:
+        description: 'Publig name (from plugins/apps/<plugin_name>-plugin)'
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: penpot-runner-01
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gh_ref }}
+
+      # START: Setup Node and PNPM enabling cache
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+
+      - name: Enable PNPM
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          corepack enable;
+          corepack install;
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        working-directory: ./plugins
+        shell: bash
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+      # END: Setup Node and PNPM enabling cache
+
+      - name: Install deps
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          pnpm install --no-frozen-lockfile;
+          pnpm add -D -w wrangler@latest;
+
+      - name: "Build package for ${{ inputs.plugin_name }}-plugin"
+        working-directory: plugins
+        shell: bash
+        run: npx nx build ${{ inputs.plugin_name }}-plugin
+
+      - name: Select Worker name
+        run: |
+          REF="${{ inputs.gh_ref }}"
+          case "$REF" in
+            main)
+              echo "WORKER_NAME=${{ inputs.plugin_name }}-plugin-pro" >> $GITHUB_ENV
+              echo "WORKER_URI=${{ inputs.plugin_name }}.plugins.penpot.app" >> $GITHUB_ENV ;;
+            staging)
+              echo "WORKER_NAME=${{ inputs.plugin_name }}-plugin-pre" >> $GITHUB_ENV
+              echo "WORKER_URI=${{ inputs.plugin_name }}.plugins.penpot.dev" >> $GITHUB_ENV ;;
+            develop)
+              echo "WORKER_NAME=${{ inputs.plugin_name }}-plugin-hourly" >> $GITHUB_ENV
+              echo "WORKER_URI=${{ inputs.plugin_name }}.plugins.hourly.penpot.dev" >> $GITHUB_ENV ;;
+            *) echo "Unsupported branch ${REF}" && exit 1 ;;
+          esac
+
+      - name: Set the custom url
+        working-directory: plugins
+        shell: bash
+        run: |
+          sed -i "s/WORKER_URI/${{ env.WORKER_URI }}/g" apps/${{ inputs.plugin_name }}-plugin/wrangler.toml
+
+      - name: Deploy to Cloudflare Workers
+        uses: cloudflare/wrangler-action@v3
+        with:
+          workingDirectory: plugins
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          command: deploy --config apps/${{ inputs.plugin_name }}-plugin/wrangler.toml --name ${{ env.WORKER_NAME }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🧩📦 *[PENPOT PLUGINS] Error deploying ${{ env.WORKER_NAME }}.*
+            📄 Triggered from ref: `${{ inputs.gh_ref }}`
+               Plugin name: `${{ inputs.plugin_name }}-plugin`
+               Cloudflare worker name: `${{ env.WORKER_NAME }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/plugins-deploy-packages.yml

@@ -0,0 +1,143 @@
+name: Plugins/packages deployer
+
+on:
+  push:
+    branches:
+      - develop
+      - staging
+      - main
+    paths:
+      - 'plugins/apps/*-plugin/**'
+      - 'libs/plugins-styles/**'
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: choice
+        required: true
+        default: 'develop'
+        options:
+          - develop
+          - staging
+          - main
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      colors_to_tokens: ${{ steps.filter.outputs.colors_to_tokens }}
+      create_palette: ${{ steps.filter.outputs.create_palette }}
+      lorem_ipsum: ${{ steps.filter.outputs.lorem_ipsum }}
+      rename_layers: ${{ steps.filter.outputs.rename_layers }}
+      contrast: ${{ steps.filter.outputs.contrast }}
+      icons: ${{ steps.filter.outputs.icons }}
+      poc_state: ${{ steps.filter.outputs.poc_state }}
+      table: ${{ steps.filter.outputs.table }}
+      # [For new plugins]
+      #   Add more outputs here
+    steps:
+      - uses: actions/checkout@v4
+      - id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            colors_to_tokens:
+              - 'plugins/apps/colors-to-tokens-plugin/**'
+              - 'libs/plugins-styles/**'
+            contrast:
+              - 'plugins/apps/contrast-plugin/**'
+              - 'libs/plugins-styles/**'
+            create_palette:
+              - 'plugins/apps/create-palette-plugin/**'
+              - 'libs/plugins-styles/**'
+            icons:
+              - 'plugins/apps/icons-plugin/**'
+              - 'libs/plugins-styles/**'
+            lorem_ipsum:
+              - 'plugins/apps/lorem-ipsum-plugin/**'
+              - 'libs/plugins-styles/**'
+            rename_layers:
+              - 'plugins/apps/rename-layers-plugin/**'
+              - 'libs/plugins-styles/**'
+            table:
+              - 'plugins/apps/table-plugin/**'
+              - 'libs/plugins-styles/**'
+           # [For new plugins]
+           #   Add more plugin filters here
+           #     another_plugin:
+           #       - 'plugins/apps/another-plugin/**'
+           #       - 'libs/plugins-styles/**'
+
+  colors-to-tokens-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.colors_to_tokens == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: colors-to-tokens
+
+  contrast-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.contrast == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: contrast
+
+  create-palette-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.create_palette == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: create-palette
+
+  icons-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.icons == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: icons
+
+  lorem-ipsum-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.lorem_ipsum == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: lorem-ipsum
+
+  rename-layers-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.rename_layers == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: rename-layers
+
+  table-plugin:
+    needs: detect-changes
+    if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.table == 'true'
+    uses: ./.github/workflows/plugins-deploy-package.yml
+    secrets: inherit
+    with:
+      gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+      plugin_name: table
+
+  # [For new plugins]
+  #   Add more jobs for other plugins below, following the same pattern
+  #     another-plugin:
+  #       needs: detect-changes
+  #       if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.another_plugin == 'true'
+  #       uses: ./.github/workflows/plugins-deploy-package.yml
+  #       secrets: inherit
+  #       with:
+  #         gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
+  #         plugin_name: another

.github/workflows/plugins-deploy-styles-doc.yml

@@ -0,0 +1,123 @@
+name: Plugins/styles-doc deployer
+
+on:
+  push:
+    branches:
+      - develop
+      - staging
+      - main
+    paths:
+      - 'plugins/apps/example-styles/**'
+      - 'plugins/libs/plugins-styles/**'
+      - 'plugins/wrangler-penpot-plugins-styles-doc.toml'
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Name of the branch'
+        type: choice
+        required: true
+        default: 'develop'
+        options:
+          - develop
+          - staging
+          - main
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.vars.outputs.gh_ref }}
+
+      # START: Setup Node and PNPM enabling cache
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+
+      - name: Enable PNPM
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          corepack enable;
+          corepack install;
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        working-directory: ./plugins
+        shell: bash
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+      # END: Setup Node and PNPM enabling cache
+
+      - name: Install deps
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          pnpm install --no-frozen-lockfile;
+          pnpm add -D -w wrangler@latest;
+
+      - name: Build styles
+        working-directory: plugins
+        shell: bash
+        run: npx nx run example-styles:build
+
+      - name: Select Worker name
+        run: |
+          REF="${{ steps.vars.outputs.gh_ref }}"
+          case "$REF" in
+            main)
+              echo "WORKER_NAME=penpot-plugins-styles-doc-pro" >> $GITHUB_ENV
+              echo "WORKER_URI=styles-doc.plugins.penpot.app" >> $GITHUB_ENV ;;
+            staging)
+              echo "WORKER_NAME=penpot-plugins-styles-doc-pre" >> $GITHUB_ENV
+              echo "WORKER_URI=styles-doc.plugins.penpot.dev" >> $GITHUB_ENV ;;
+            develop)
+              echo "WORKER_NAME=penpot-plugins-styles-doc-hourly" >> $GITHUB_ENV
+              echo "WORKER_URI=styles-doc.plugins.hourly.penpot.dev" >> $GITHUB_ENV ;;
+            *) echo "Unsupported branch ${REF}" && exit 1 ;;
+          esac
+
+      - name: Set the custom url
+        working-directory: plugins
+        shell: bash
+        run: |
+          sed -i "s/WORKER_URI/${{ env.WORKER_URI }}/g" wrangler-penpot-plugins-styles-doc.toml
+
+      - name: Deploy to Cloudflare Workers
+        uses: cloudflare/wrangler-action@v3
+        with:
+          workingDirectory: plugins
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          command: deploy --config wrangler-penpot-plugins-styles-doc.toml --name ${{ env.WORKER_NAME }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🧩💅 *[PENPOT PLUGINS] Error deploying Styles documentation.*
+            📄 Triggered from ref: `${{ inputs.gh_ref }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/release.yml

@@ -0,0 +1,114 @@
+name: Release Publisher
+
+on:
+  workflow_dispatch:
+    inputs:
+      gh_ref:
+        description: 'Tag to release'
+        type: string
+        required: true
+  workflow_call:
+    inputs:
+      gh_ref:
+        description: 'Tag to release'
+        type: string
+        required: true
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    environment: release-admins
+    runs-on: ubuntu-24.04
+    outputs:
+      version: ${{ steps.vars.outputs.gh_ref }}
+      release_notes: ${{ steps.extract_release_notes.outputs.release_notes }}
+    steps:
+      - name: Extract some useful variables
+        id: vars
+        run: |
+          echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.vars.outputs.gh_ref }}
+
+      # --- Publicly release the docker images ---
+      - name: Configure ECR credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
+          aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Install Skopeo
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y skopeo
+
+      - name: Copy images from AWS ECR to Docker Hub
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+          PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
+          PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
+          TAG: ${{ steps.vars.outputs.gh_ref }}
+        run: |
+          aws ecr get-login-password --region $AWS_REGION | \
+            skopeo login --username AWS --password-stdin \
+            $DOCKER_REGISTRY
+
+          echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io
+
+          IMAGES=("frontend" "backend" "exporter" "storybook")
+
+          for image in "${IMAGES[@]}"; do
+            skopeo copy --all \
+              docker://$DOCKER_REGISTRY/$image:$TAG \
+              docker://docker.io/penpotapp/$image:$TAG
+
+            for alias in main latest; do
+              skopeo copy --all \
+                docker://$DOCKER_REGISTRY/$image:$TAG \
+                docker://docker.io/penpotapp/$image:$alias
+            done
+          done
+
+      # --- Release notes extraction ---
+      - name: Extract release notes from CHANGES.md
+        id: extract_release_notes
+        env:
+          TAG: ${{ steps.vars.outputs.gh_ref }}
+        run: |
+          RELEASE_NOTES=$(awk "/^## $TAG$/{flag=1; next} /^## /{flag=0} flag" CHANGES.md | awk '{$1=$1};1')
+          if [ -z "$RELEASE_NOTES" ]; then
+            RELEASE_NOTES="No changes for $TAG according to CHANGES.md"
+          fi
+          echo "release_notes<<EOF" >> $GITHUB_OUTPUT
+          echo "$RELEASE_NOTES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      # --- Create GitHub release ---
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.vars.outputs.gh_ref }}
+          name: ${{ steps.vars.outputs.gh_ref }}
+          body: ${{ steps.extract_release_notes.outputs.release_notes }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🚀 *[PENPOT] Error releasing penpot.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/tests-mcp.yml

@@ -0,0 +1,45 @@
+name: "MCP CI"
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - staging
+      - main
+
+    types:
+      - opened
+      - synchronize
+
+    paths:
+      - 'mcp/**'
+
+  push:
+    branches:
+      - develop
+      - staging
+      - main
+
+    paths:
+      - 'mcp/**'
+
+jobs:
+  test:
+    name: "Test"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup
+        working-directory: ./mcp
+        run: ./scripts/setup
+
+      - name: Check
+        working-directory: ./mcp
+        run: |
+          pnpm run fmt:check;
+          pnpm -r run build;
+          pnpm -r run types:check;

.github/workflows/tests.yml

@@ -0,0 +1,337 @@
+name: "CI"
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - develop
+      - staging
+
+concurrency:
+  group: ${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: "Linter"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check clojure code format
+        run: |
+          ./scripts/lint
+
+  test-common:
+    name: "Common Tests"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests on JVM
+        working-directory: ./common
+        run: |
+          clojure -M:dev:test
+
+      - name: Run tests on NODE
+        working-directory: ./common
+        run: |
+          ./scripts/test
+
+  test-plugins:
+    name: Plugins Runtime Linter & Tests
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node
+        id: setup-node
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install deps
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          corepack enable;
+          corepack install;
+          pnpm install;
+
+      - name: Run Lint
+        working-directory: ./plugins
+        run: pnpm run lint
+
+      - name: Run Format Check
+        working-directory: ./plugins
+        run: pnpm run format:check
+
+      - name: Run Test
+        working-directory: ./plugins
+        run: pnpm run test
+
+      - name: Build runtime
+        working-directory: ./plugins
+        run: pnpm run build
+
+      - name: Build plugins
+        working-directory: ./plugins
+        run: pnpm run build:plugins
+
+      - name: Build styles
+        working-directory: ./plugins
+        run: pnpm run build:styles-example
+
+  test-frontend:
+    name: "Frontend Tests"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Unit Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test
+
+      - name: Component Tests
+        working-directory: ./frontend
+        env:
+          VITEST_BROWSER_TIMEOUT: 120000
+        run: |
+          ./scripts/test-components
+
+  test-render-wasm:
+    name: "Render WASM Tests"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Format
+        working-directory: ./render-wasm
+        run: |
+          cargo fmt --check
+
+      - name: Lint
+        working-directory: ./render-wasm
+        run: |
+          ./lint
+
+      - name: Test
+        working-directory: ./render-wasm
+        run: |
+          ./test
+
+  test-backend:
+    name: "Backend Tests"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    services:
+      postgres:
+        image: postgres:17
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: penpot_test
+          POSTGRES_PASSWORD: penpot_test
+          POSTGRES_DB: penpot_test
+
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      redis:
+        image: valkey/valkey:9
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        working-directory: ./backend
+        env:
+          PENPOT_TEST_DATABASE_URI: "postgresql://postgres/penpot_test"
+          PENPOT_TEST_DATABASE_USERNAME: penpot_test
+          PENPOT_TEST_DATABASE_PASSWORD: penpot_test
+          PENPOT_TEST_REDIS_URI: "redis://redis/1"
+
+        run: |
+          clojure -M:dev:test --reporter kaocha.report/documentation
+
+  test-library:
+    name: "Library Tests"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        working-directory: ./library
+        run: |
+          ./scripts/test
+
+  build-integration:
+    name: "Build Integration Bundle"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build Bundle
+        working-directory: ./frontend
+        run: |
+          ./scripts/build 0.0.0
+
+      - name: Store Bundle Cache
+        uses: actions/cache@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+
+  test-integration-1:
+    name: "Integration Tests 1/4"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="1/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-1
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-2:
+    name: "Integration Tests 2/4"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="2/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-2
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-3:
+    name: "Integration Tests 3/4"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="3/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-3
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-4:
+    name: "Integration Tests 4/4"
+    runs-on: penpot-runner-02
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="4/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-4
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3

.gitignore

@@ -5,6 +5,7 @@
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+.pnpm-store
 *-init.clj
 *.css.json
 *.jar
@@ -31,6 +32,7 @@
 /.clj-kondo/.cache
 /_dump
 /notes
+/playground/
 /backend/*.md
 /backend/*.sql
 /backend/*.txt
@@ -41,6 +43,7 @@
 /backend/resources/public/assets
 /backend/resources/public/media
 /backend/target/
+/backend/experiments
 /bundle*
 /cd.md
 /clj-profiler/
@@ -51,9 +54,6 @@
 /exporter/target
 /frontend/.storybook/preview-body.html
 /frontend/.storybook/preview-head.html
-/frontend/cypress/fixtures/validuser.json
-/frontend/cypress/videos/*/
-/frontend/cypress/videos/*/
 /frontend/dist/
 /frontend/npm-debug.log
 /frontend/out/
@@ -70,6 +70,8 @@
 /vendor/svgclean/bundle*.js
 /web
 /library/target/
+/library/*.zip
+/external
 
 clj-profiler/
 node_modules
@@ -79,3 +81,4 @@ node_modules
 /playwright/.cache/
 /render-wasm/target/
 /**/.yarn/*
+/.pnpm-store

.nvmrc

@@ -1 +1 @@
-v22.13.1
+v22.22.0

.travis.yml

@@ -1,40 +0,0 @@
-dist: xenial
-
-language: generic
-sudo: required
-
-cache:
-  directories:
-    - $HOME/.m2
-
-services:
-  - docker
-
-branches:
-  only:
-    - master
-    - develop
-
-install:
-  - curl -O https://download.clojure.org/install/linux-install-1.10.1.447.sh
-  - chmod +x linux-install-1.10.1.447.sh
-  - sudo ./linux-install-1.10.1.447.sh
-
-before_script:
-  - env | sort
-
-script:
-  - ./manage.sh build-devenv
-  - ./manage.sh run-frontend-tests
-  - ./manage.sh run-backend-tests
-  - ./manage.sh build-images
-  - ./manage.sh run
-
-after_script:
-  - docker images
-
-notifications:
-  email: false
-
-env:
-  - NODE_VERSION=10.16.0

CHANGES.md

@@ -1,6 +1,382 @@
 # CHANGELOG
 
-## 2.8.0 (Next / Unreleased)
+## 2.13.1
+
+### :bug: Bugs fixed
+
+- Fix PDF Exporter outputs empty page when board has A4 format [Taiga #13181](https://tree.taiga.io/project/penpot/issue/13181)
+
+## 2.13.0
+
+### :heart: Community contributions (Thank you!)
+
+- Fix mask issues with component swap (by @dfelinto) [Github #7675](https://github.com/penpot/penpot/issues/7675)
+
+### :sparkles: New features & Enhancements
+
+- Add new Box Shadow Tokens [Taiga #10201](https://tree.taiga.io/project/penpot/us/10201)
+- Make i18n translation files load on-demand [Taiga #11474](https://tree.taiga.io/project/penpot/us/11474)
+- Add deleted files to dashboard [Taiga #8149](https://tree.taiga.io/project/penpot/us/8149)
+
+### :bug: Bugs fixed
+
+- Fix problem when drag+duplicate a full grid [Taiga #12565](https://tree.taiga.io/project/penpot/issue/12565)
+- Fix problem when pasting elements in reverse flex layout [Taiga #12460](https://tree.taiga.io/project/penpot/issue/12460)
+- Fix wrong board size presets in Android [Taiga #12339](https://tree.taiga.io/project/penpot/issue/12339)
+- Fix problem with grid layout components and auto sizing [Github #7797](https://github.com/penpot/penpot/issues/7797)
+- Fix some alignments on inspect tab [Taiga #12915](https://tree.taiga.io/project/penpot/issue/12915)
+- Fix color assets from shared libraries not appearing as assets in Selected colors panel [Taiga #12957](https://tree.taiga.io/project/penpot/issue/12957)
+- Fix CSS generated box-shadow property [Taiga #12997](https://tree.taiga.io/project/penpot/issue/12997)
+- Fix inner shadow selector on shadow token [Taiga #12951](https://tree.taiga.io/project/penpot/issue/12951)
+- Fix missing text color token from selected shapes in selected colors list [Taiga #12956](https://tree.taiga.io/project/penpot/issue/12956)
+- Fix dropdown option width in Guides columns dropdown [Taiga #12959](https://tree.taiga.io/project/penpot/issue/12959)
+- Fix typos on download modal [Taiga #12865](https://tree.taiga.io/project/penpot/issue/12865)
+- Fix unhandled exception tokens creation dialog [Github #8110](https://github.com/penpot/penpot/issues/8110)
+- Fix allow negative spread values on shadow token creation [Taiga #13167](https://tree.taiga.io/project/penpot/issue/13167)
+- Fix spanish translations on import export token modal [Taiga #13171](https://tree.taiga.io/project/penpot/issue/13171)
+- Remove whitespaces from asset export filename [Github #8133](https://github.com/penpot/penpot/pull/8133)
+- Fix exception on uploading large fonts [Github #8135](https://github.com/penpot/penpot/pull/8135)
+- Fix unhandled exception on open-new-window helper [Github #7787](https://github.com/penpot/penpot/issues/7787)
+- Fix incorrect handling of input values on layout gap and padding inputs [Github #8113](https://github.com/penpot/penpot/issues/8113)
+- Fix several race conditions on path editor [Github #8187](https://github.com/penpot/penpot/pull/8187)
+- Fix app freeze when introducing an error on a very long token name [Taiga #13214](https://tree.taiga.io/project/penpot/issue/13214)
+- Fix import a file with shadow tokens [Taiga #13229](https://tree.taiga.io/project/penpot/issue/13229)
+- Fix allow spaces on token description [Taiga #13184](https://tree.taiga.io/project/penpot/issue/13184)
+- Fix error when creating a token with an invalid name [Taiga #13219](https://tree.taiga.io/project/penpot/issue/13219)
+
+## 2.12.1
+
+### :bug: Bugs fixed
+
+- Fix setting a portion of text as bold or underline messes things up [Github #7980](https://github.com/penpot/penpot/issues/7980)
+- Fix problem with style in fonts input [Taiga #12935](https://tree.taiga.io/project/penpot/issue/12935)
+- Fix problem with path editor and right click [Github #7917](https://github.com/penpot/penpot/issues/7917)
+
+
+## 2.12.0
+
+### :boom: Breaking changes & Deprecations
+
+#### Backend RPC API changes
+
+The backend RPC API URLS are changed from `/api/rpc/command/<name>` to
+`/api/main/methods/<name>`. The previous PATH is preserved for backward
+compatibility; however, if you are a user of this API, it is strongly
+recommended that you adapt your code to use the new PATH.
+
+
+#### Updated SSO Callback URL
+
+The OAuth / Single Sign-On (SSO) callback endpoint has changed to
+align with the new OpenID Connect (OIDC) implementation.
+
+Old callback URL:
+
+```
+https://<your_domain>/api/auth/oauth/<oauth_provider>/callback
+```
+
+New callback URL:
+
+```
+https://<your_domain>/api/auth/oidc/callback
+```
+
+**Action required:**
+
+If you have SSO/Social-Auth configured on your on-premise instance,
+the following actions are required before update:
+
+Update your OAuth or SSO provider configuration (e.g., Okta, Google,
+Azure AD, etc.) to use the new callback URL. Failure to update may
+result in authentication failures after upgrading.
+
+**Reason for change:**
+
+This update standardizes all authentication flows under the single URL
+and makis it more modular, enabling the ability to configure SSO auth
+provider dinamically.
+
+
+#### Changes on default docker compose
+
+We have updated the `docker/images/docker-compose.yaml` with a small
+change related to the `PENPOT_SECRET_KEY`. Since this version, this
+environment variable is also required on exporter. So if you are using
+penpot on-premise you will need to apply the same changes on your own
+`docker-compose.yaml` file.
+
+We have removed the Minio server from the `docker/images/docker-compose.yml`
+example. It's still usable as before, we just removed the example.
+
+### :rocket: Epics and highlights
+
+### :heart: Community contributions (Thank you!)
+
+- Ensure consistent snap behavior across all zoom levels [Github #7774](https://github.com/penpot/penpot/pull/7774) by [@Tokytome](https://github.com/Tokytome)
+- Fix crash in token grid view due to tooltip validation (by @dfelinto) [Github #7887](https://github.com/penpot/penpot/pull/7887)
+- Enable Hindi translations on the application
+
+### :sparkles: New features & Enhancements
+
+- Add the ability to select boards to export as PDF [Taiga #12320](https://tree.taiga.io/project/penpot/issue/12320)
+- Add toggle for switching boolean property values [Taiga #12341](https://tree.taiga.io/project/penpot/us/12341)
+- Make the file export process more reliable [Taiga #12555](https://tree.taiga.io/project/penpot/us/12555)
+- Add auth flow changes [Taiga #12333](https://tree.taiga.io/project/penpot/us/12333)
+- Add new shape validation mechanism for shapes [Github #7696](https://github.com/penpot/penpot/pull/7696)
+- Apply color tokens from sidebar [Taiga #11353](https://tree.taiga.io/project/penpot/us/11353)
+- Display tokens in the inspect tab [Taiga #9313](https://tree.taiga.io/project/penpot/us/9313)
+- Refactor clipboard behavior to assess some minor inconsistencies and make pasting binary data faster. [Taiga #12571](https://tree.taiga.io/project/penpot/task/12571)
+
+### :bug: Bugs fixed
+
+- Fix text line-height values are wrong [Taiga #12252](https://tree.taiga.io/project/penpot/issue/12252)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
+- Fix pan cursor not disabling viewport guides [Github #6985](https://github.com/penpot/penpot/issues/6985)
+- Fix viewport resize on locked shapes [Taiga #11974](https://tree.taiga.io/project/penpot/issue/11974)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix on copy instance inside a components chain touched are missing [Taiga #12371](https://tree.taiga.io/project/penpot/issue/12371)
+- Fix problem with multiple selection and shadows [Github #7437](https://github.com/penpot/penpot/issues/7437)
+- Fix search shortcut [Taiga #10265](https://tree.taiga.io/project/penpot/issue/10265)
+- Fix shortcut conflict in text editor (increase/decrease font size vs word selection)
+- Fix problem with plugins generating code for pages different than current one [Taiga #12312](https://tree.taiga.io/project/penpot/issue/12312)
+- Fix input confirmation behavior is not uniform [Taiga #12294](https://tree.taiga.io/project/penpot/issue/12294)
+- Fix copy/pasting application/transit+json [Taiga #12721](https://tree.taiga.io/project/penpot/issue/12721)
+- Fix problem with plugins content attribute [Plugins #209](https://github.com/penpot/penpot-plugins/issues/209)
+- Fix U and E icon displayed in project list [Taiga #12806](https://tree.taiga.io/project/penpot/issue/12806)
+- Fix unpublish library modal not scrolling a long file list [Taiga #12285](https://tree.taiga.io/project/penpot/issue/12285)
+- Fix incorrect interaction betwen hower and scroll on assets sidebar [Taiga #12389](https://tree.taiga.io/project/penpot/issue/12389)
+- Fix switch variants with paths [Taiga #12841](https://tree.taiga.io/project/penpot/issue/12841)
+- Fix referencing typography tokens on font-family tokens [Taiga #12492](https://tree.taiga.io/project/penpot/issue/12492)
+- Fix horizontal scroll on layer panel [Taiga #12843](https://tree.taiga.io/project/penpot/issue/12843)
+- Fix unicode handling on email template abbreviation filter [Github #7966](https://github.com/penpot/penpot/pull/7966)
+
+## 2.11.1
+
+- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
+
+## 2.11.0
+
+### :boom: Breaking changes & Deprecations
+
+- Deprecated configuration variables with the prefix `PENPOT_ASSETS_*`, and will be
+  removed in future versions:
+
+  - The `PENPOT_ASSETS_STORAGE_BACKEND` becomes `PENPOT_OBJECTS_STORAGE_BACKEND` and its
+    values passes from (`assets-fs` or `assets-s3`) to (`fs` or `s3`)
+  - The `PENPOT_STORAGE_ASSETS_FS_DIRECTORY` becomes `PENPOT_OBJECTS_STORAGE_FS_DIRECTORY`
+  - The `PENPOT_STORAGE_ASSETS_S3_BUCKET` becomes `PENPOT_OBJECTS_STORAGE_S3_BUCKET`
+  - The `PENPOT_STORAGE_ASSETS_S3_REGION` becomes `PENPOT_OBJECTS_STORAGE_S3_REGION`
+  - The `PENPOT_STORAGE_ASSETS_S3_ENDPOINT` becomes `PENPOT_OBJECTS_STORAGE_S3_ENDPOINT`
+  - The `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` replaced (see below)
+
+- Add `PENPOT_NETTY_IO_THREADS` and `PENPOT_EXECUTOR_THREADS` variables to provide the
+  control over concurrency of the shared resources used by netty. Penpot uses the netty IO
+  threads for AWS S3 SDK and Redis/Valkey communication, and the EXEC threads to perform
+  out of HTTP serving threads tasks such that cache invalidation, S3 response completion,
+  configuration reloading and many other auxiliar tasks. By default they use a half number
+  if available cpus with a minumum of 2 for both executors. You should not touch that
+  variables unless you are know what you are doing.
+
+- Replace the `PENPOT_STORAGE_ASSETS_S3_IO_THREADS` with a more general configuration
+  `PENPOT_NETTY_IO_THREADS` used to configure a shared netty resources across different
+  services which use netty internally (redis connection, S3 SDK client). This
+  configuration is not very commonly used so don't expected real impact on any user.
+
+### :sparkles: New features & Enhancements
+
+- New composite token: Typography [Taiga #10200](https://tree.taiga.io/project/penpot/us/10200)
+- Show current Penpot version [Taiga #11603](https://tree.taiga.io/project/penpot/us/11603)
+- Switch several variant copies at the same time [Taiga #11411](https://tree.taiga.io/project/penpot/us/11411)
+- Invitations management improvements [Taiga #3479](https://tree.taiga.io/project/penpot/us/3479)
+- Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
+- Reorder properties for a component [Taiga #10225](https://tree.taiga.io/project/penpot/us/10225)
+- File Data storage layout refactor [Github #7345](https://github.com/penpot/penpot/pull/7345)
+- Make several queries optimization on comment threads [Github #7506](https://github.com/penpot/penpot/pull/7506)
+
+### :bug: Bugs fixed
+
+- Fix selection problems when devtools open [Taiga #11950](https://tree.taiga.io/project/penpot/issue/11950)
+- Fix long font names overlap [Taiga #11844](https://tree.taiga.io/project/penpot/issue/11844)
+- Fix paste behavior according to the selected element [Taiga #11979](https://tree.taiga.io/project/penpot/issue/11979)
+- Fix problem with export size [Github #7160](https://github.com/penpot/penpot/issues/7160)
+- Fix multi level library dependencies [Taiga #12155](https://tree.taiga.io/project/penpot/issue/12155)
+- Fix component context menu options order in assets tab [Taiga #11941](https://tree.taiga.io/project/penpot/issue/11941)
+- Fix error updating library [Taiga #12218](https://tree.taiga.io/project/penpot/issue/12218)
+- Fix restoring a variant in another file makes it overlap the existing variant [Taiga #12049](https://tree.taiga.io/project/penpot/issue/12049)
+- Fix auto-width changes to fixed when switching variants [Taiga #12172](https://tree.taiga.io/project/penpot/issue/12172)
+- Fix component number has no singular translation string [Taiga #12106](https://tree.taiga.io/project/penpot/issue/12106)
+- Fix adding/removing identical text fills [Taiga #12287](https://tree.taiga.io/project/penpot/issue/12287)
+- Fix scroll on the inspect tab [Taiga #12293](https://tree.taiga.io/project/penpot/issue/12293)
+- Fix lock proportion tooltip [Taiga #12326](https://tree.taiga.io/project/penpot/issue/12326)
+- Fix internal Error when selecting a set by name in the token theme editor [Taiga #12310](https://tree.taiga.io/project/penpot/issue/12310)
+- Fix drag & drop functionality is swapping instead or reordering [Taiga #12254](https://tree.taiga.io/project/penpot/issue/12254)
+- Fix variants not syncronizing tokens on switch [Taiga #12290](https://tree.taiga.io/project/penpot/issue/12290)
+- Fix incorrect behavior of Alt + Drag for variants [Taiga #12309](https://tree.taiga.io/project/penpot/issue/12309)
+- Fix text override is lost after switch [Taiga #12269](https://tree.taiga.io/project/penpot/issue/12269)
+- Fix exporting a board crashing the app [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12384)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix selected colors not showing colors from children shapes in multiple selection [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12385)
+- Fix scrollbar issue in design tab [Taiga #12367](https://tree.taiga.io/project/penpot/issue/12367)
+- Fix library update notificacions showing when they should not [Taiga #12397](https://tree.taiga.io/project/penpot/issue/12397)
+- Fix remove flex button doesn’t work within variant [Taiga #12314](https://tree.taiga.io/project/penpot/issue/12314)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
+- Fix problem with certain text input in some editable labels (pages, components, tokens...) being in conflict with the drag/drop functionality [Taiga #12316](https://tree.taiga.io/project/penpot/issue/12316)
+- Fix not controlled theme renaming [Taiga #12411](https://tree.taiga.io/project/penpot/issue/12411)
+- Fix paste without selection sends the new element in the back [Taiga #12382](https://tree.taiga.io/project/penpot/issue/12382)
+- Fix options button does not work for comments created in the lower part of the screen [Taiga #12422](https://tree.taiga.io/project/penpot/issue/12422)
+- Fix problem when checking usage with removed teams [Taiga #12442](https://tree.taiga.io/project/penpot/issue/12442)
+- Fix focus mode persisting across page/file navigation [Taiga #12469](https://tree.taiga.io/project/penpot/issue/12469)
+- Fix shadow color validation [Github #7705](https://github.com/penpot/penpot/pull/7705)
+- Fix exception on selection blend-mode using keyboard [Github #7710](https://github.com/penpot/penpot/pull/7710)
+- Fix crash when using decimal (floating-point) values for X/Y or width/height [Taiga #12543](https://tree.taiga.io/project/penpot/issue/12543)
+
+## 2.10.1
+
+### :sparkles: New features & Enhancements
+
+- Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
+
+### :bug: Bugs fixed
+
+- Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
+
+## 2.10.0
+
+### :rocket: Epics and highlights
+
+- Variants
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+### :sparkles: New features & Enhancements
+
+- Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
+- Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
+- Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
+- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
+- New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
+- New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
+- New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
+- New letter spacing token [Taiga #10940](https://tree.taiga.io/project/penpot/us/10940)
+- New font weight token [Taiga #10939](https://tree.taiga.io/project/penpot/us/10939)
+- Upgrade Node to v22.18.0 [Github #7283](https://github.com/penpot/penpot/pull/7283)
+- Upgrade the base docker image for penpot frontend to v1.29.1 [Github #7283](https://github.com/penpot/penpot/pull/7283)
+- Create variant from an existing component [Taiga #2088](https://tree.taiga.io/project/penpot/us/2088)
+- Create variant from an existing variant [Taiga #8282](https://tree.taiga.io/project/penpot/us/8282)
+- Actions over a component with variants [Taiga #10503](https://tree.taiga.io/project/penpot/us/10503)
+- Create a variant by dragging a component into a component with variants [Taiga #8134](https://tree.taiga.io/project/penpot/us/8134)
+- Transform a variant into an individual component [Taiga #8141](https://tree.taiga.io/project/penpot/us/8141)
+- Delete variant [Taiga #6890](https://tree.taiga.io/project/penpot/us/6890)
+- Restore an orphaned copy of a variant [Taiga #10446](https://tree.taiga.io/project/penpot/us/10446)
+- Add, Edit & Delete variant properties name and value [Taiga #6892](https://tree.taiga.io/project/penpot/us/6892)
+- Retrieve variants [Taiga #6888](https://tree.taiga.io/project/penpot/us/6888)
+- Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
+- Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
+- Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
+- Fix problem with component swapping panel [Taiga #12175](https://tree.taiga.io/project/penpot/issue/12175)
+
+### :bug: Bugs fixed
+
+- Display strokes information in inspect tab [Taiga #11154](https://tree.taiga.io/project/penpot/issue/11154)
+- Fix problem with booleans selection [Taiga #11627](https://tree.taiga.io/project/penpot/issue/11627)
+- Fix missing font when copy&paste a chunk of text [Taiga #11522](https://tree.taiga.io/project/penpot/issue/11522)
+- Fix bad swap slot after two swaps [Taiga #11659](https://tree.taiga.io/project/penpot/issue/11659)
+- Fix missing package for the `penpot_exporter` Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
+- Fix issue where multiple dropdown menus could be opened simultaneously on the dashboard page [Taiga #11500](https://tree.taiga.io/project/penpot/issue/11500)
+- Fix font size/variant not updated when editing a text [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11552)
+- Fix issue where Alt + arrow keys shortcut interferes with letter-spacing when moving text layers [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11771)
+- Fix consistency issues on how font variants are visualized [Taiga #11499](https://tree.taiga.io/project/penpot/us/11499)
+- Fix parsing rx and ry SVG values for rect radius [Taiga #11861](https://tree.taiga.io/project/penpot/issue/11861)
+- Fix misleading affordance in saved versions [Taiga #11887](https://tree.taiga.io/project/penpot/issue/11887)
+- Fix pasting RTF text crashes penpot [Taiga #11717](https://tree.taiga.io/project/penpot/issue/11717)
+- Fix navigation arrows in Libraries & Templates carousel [Taiga #10609](https://tree.taiga.io/project/penpot/issue/10609)
+- Fix applying tokens with zero value to size [Taiga #11618](https://tree.taiga.io/project/penpot/issue/11618)
+- Fix typo [Taiga #11969](https://tree.taiga.io/project/penpot/issue/11969)
+- Fix typo [Taiga #11970](https://tree.taiga.io/project/penpot/issue/11970)
+- Fix typos [Taiga #11971](https://tree.taiga.io/project/penpot/issue/11971)
+- Fix inconsistent naming for "Flatten" [Taiga #8371](https://tree.taiga.io/project/penpot/issue/8371)
+- Layout item tokens should be unapplied when moving out of a layout [Taiga #11012](https://tree.taiga.io/project/penpot/issue/11012)
+- Fix incorrect date displayed for support plan [Taiga #11986](https://tree.taiga.io/project/penpot/issue/11986)
+- Fix can't import 'borderWidth' type token [#132](https://github.com/tokens-studio/penpot/issues/132)
+- Fix moving elements up or down while pressing alt [Taiga Issue #11992](https://tree.taiga.io/project/penpot/issue/11992)
+- Fix conflicting shortcuts (remove dec/inc line height and letter spacing) [Taiga #12102](https://tree.taiga.io/project/penpot/issue/12102)
+- Fix conflicting shortcuts (remove text-align shortcuts) [Taiga #12047](https://tree.taiga.io/project/penpot/issue/12047)
+- Fix export file with empty tokens library [Taiga #12137](https://tree.taiga.io/project/penpot/issue/12137)
+- Fix context menu on spacing tokens [Taiga #12141](https://tree.taiga.io/project/penpot/issue/12141)
+
+## 2.9.0
+
+### :rocket: Epics and highlights
+
+### :boom: Breaking changes & Deprecations
+
+### :heart: Community contributions (Thank you!)
+
+- Clarify message when inviting existing team members to make it more user-friendly and clear which invitations will be sent. [Taiga #11441](https://tree.taiga.io/project/penpot/issue/11441) by [@iprithvitharun](https://github.com/iprithvitharun)
+- Update email change confirmation message for clarity and correct grammar. [GitHub #6786](https://github.com/penpot/penpot/issues/6786) by [@iprithvitharun](https://github.com/iprithvitharun)
+
+### :sparkles: New features & Enhancements
+
+- Add visual indicator for new comments in the workspace [Taiga #11328](https://tree.taiga.io/project/penpot/issue/11328)
+- On components overrides, separate the content of the text from the rest of properties [Taiga #7434](https://tree.taiga.io/project/penpot/us/7434)
+- Improve dashboard's sidebar [Taiga #10700](https://tree.taiga.io/project/penpot/us/10700)
+- Change "Save color" button to primary button [Taiga #9410](https://tree.taiga.io/project/penpot/issue/9410)
+- Support for exif rotated images [GitHub #6767](https://github.com/penpot/penpot/issues/6767)
+- Display Blend Mode and Layer Opacity properties in the Inspect tab [Taiga #11283](https://tree.taiga.io/project/penpot/issue/11283)
+- Provide CSS `mix-blend-mode` property in code editor when present on shape [Taiga #11282](https://tree.taiga.io/project/penpot/issue/11282)
+- Add the option to import tokens in a .zip file. [Taiga #11378](https://tree.taiga.io/project/penpot/us/11378)
+- New typography token type - font size token [Taiga #10938](https://tree.taiga.io/project/penpot/us/10938)
+- Hide bounding box while editing visual effects [Taiga #11576](https://tree.taiga.io/project/penpot/issue/11576)
+- Improved text layer resizing: Allow double-click on text bounding box to set auto-width/auto-height [Taiga #11577](https://tree.taiga.io/project/penpot/issue/11577)
+- Improve text layer auto-resize: auto-width switches to auto-height on horizontal resize, and only switches to fixed on vertical resize [Taiga #11578](https://tree.taiga.io/project/penpot/issue/11578)
+- Add the ability to show login dialog on profile settings [Github #6871](https://github.com/penpot/penpot/pull/6871)
+- Improve the application of tokens with object specific tokens [Taiga #10209](https://tree.taiga.io/project/penpot/us/10209)
+- Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
+- Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
+
+### :bug: Bugs fixed
+
+- Copying font size does not copy the unit [Taiga #11143](https://tree.taiga.io/project/penpot/issue/11143)
+- Fix text-decoration line-through that displays a wrong property value [Taiga #11145](https://tree.taiga.io/project/penpot/issue/11145)
+- Fix display error message on register form [Taiga #11444](https://tree.taiga.io/project/penpot/issue/11444)
+- Fix toggle focus mode did not restore viewport and selection upon exit [GitHub #6280](https://github.com/penpot/penpot/issues/6820)
+- Fix problem when creating a layout from an existing layout [Taiga #11554](https://tree.taiga.io/project/penpot/issue/11554)
+- Fix title button from Title Case to Capitalize [Taiga #11476](https://tree.taiga.io/project/penpot/issue/11476)
+- Fix touchpad swipe leading to navigating back/forth [GitHub #4246](https://github.com/penpot/penpot/issues/4246)
+- Keep color data when copying from info tab into CSS [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
+- Update HSL values to modern syntax as defined in W3C CSS Color Module Level 4 [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
+- Fix main component receives focus and is selected when using 'Show Main Component' [Taiga #11402](https://tree.taiga.io/project/penpot/issue/11402)
+- Fix UI theme selection from main menu [Taiga #11567](https://tree.taiga.io/project/penpot/issue/11567)
+- Fix duplicating pages with mainInstance shapes nested inside groups [Taiga #10774](https://tree.taiga.io/project/penpot/issue/10774)
+- Fix ESC key not closing Add/Manage Libraries modal [Taiga #11523](https://tree.taiga.io/project/penpot/issue/11523)
+- Fix copying a shadow color from info tab [Taiga #11211](https://tree.taiga.io/project/penpot/issue/11211)
+- Fix remove color button in the gradient editor [Taiga #11623](https://tree.taiga.io/project/penpot/issue/11623)
+- Fix "Copy as SVG" generates different code from the Inspect panel [Taiga #11519](https://tree.taiga.io/project/penpot/issue/11519)
+- Fix overriden tokens in text copies are not preserved [Taiga #11486](https://tree.taiga.io/project/penpot/issue/11486)
+- Fix problem when changing between flex/grid layout [Taiga #11625](https://tree.taiga.io/project/penpot/issue/11625)
+- Fix opacity on stroke gradients [Taiga #11646](https://tree.taiga.io/project/penpot/issue/11646)
+- Fix change from gradient to solid color [Taiga #11648](https://tree.taiga.io/project/penpot/issue/11648)
+- Fix the context menu always closes after any action [Taiga #11624](https://tree.taiga.io/project/penpot/issue/11624)
+- Fix X & Y position do not sincronize with tokens [Taiga #11617](https://tree.taiga.io/project/penpot/issue/11617)
+- Fix tooltip position after first time [Taiga #11688](https://tree.taiga.io/project/penpot/issue/11688)
+- Fix inconsistent ordering of pinned projects on dashboard sidebar [Taiga #11674](https://tree.taiga.io/project/penpot/issue/11674)
+- Fix export button width on inspect tab [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
+- Fix stroke width token application [Taiga #11724](https://tree.taiga.io/project/penpot/issue/11724)
+- Fix number token application on shape [Taiga #11331](https://tree.taiga.io/project/penpot/task/11331)
+- Fix auto height is fixed in the HTML inspect tab for text elements [Taiga #11680](https://tree.taiga.io/project/penpot/task/11680)
+
+## 2.8.1
+
+### :bug: Bugs fixed
+
+- Fix unexpected exception on processing old texts [Github #6889](https://github.com/penpot/penpot/pull/6889)
+- Fix error on inspect tab when selecting multiple shapes [Taiga #11655](https://tree.taiga.io/project/penpot/issue/11655)
+- Fix missing package for the penport_exporter Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
+
+## 2.8.0
 
 ### :rocket: Epics and highlights
 
@@ -8,18 +384,19 @@
 
 **Penpot Library**
 
-The initial prototype is completly reworked for provide a more consistent API
+The initial prototype is completly reworked to provide a more consistent API
 and to have proper validation and params decoding. All the details can be found
 on [its own changelog](library/CHANGES.md)
 
 **Penpot migrate from Redis to Valkey**
 
 As [Valkey](https://valkey.io/) is an opne-souce fork of [Redis](https://redis.io/)
-version 7.2.4, this version of Penpot will be compatible with Redis but may diverge 
+version 7.2.4, this version of Penpot will be compatible with Redis but may diverge
-in future versions. Therefore, **migration from Redis to ValKey is recommended for all 
+in future versions. Therefore, **migration from Redis to ValKey is recommended for all
 on-premises instances** that want to keep up to date.
 
 ### :heart: Community contributions (Thank you!)
+
 - Add Serbian language [GitHub #5002](https://github.com/penpot/penpot/issues/5002) by [crnobog69](https://github.com/crnobog69)
 
 ### :sparkles: New features & Enhancements
@@ -39,6 +416,7 @@ on-premises instances** that want to keep up to date.
 - Update google fonts (at 2025/05/19) [Taiga 10792](https://tree.taiga.io/project/penpot/us/10792)
 - Add tooltip component to DS [Taiga 9220](https://tree.taiga.io/project/penpot/us/9220)
 - Allow multi file token export [Taiga #10144](https://tree.taiga.io/project/penpot/us/10144)
+- Fix problem when double click on hidden shapes [Taiga #11314](https://tree.taiga.io/project/penpot/issue/11314)
 
 ### :bug: Bugs fixed
 
@@ -57,8 +435,12 @@ on-premises instances** that want to keep up to date.
 - Fix button width [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
 - Fix mixed letter spacing and line height [Taiga #11178](https://tree.taiga.io/project/penpot/issue/11178)
 - Fix snap nodes shortcut [Taiga #11054](https://tree.taiga.io/project/penpot/issue/11054)
-- Fix changing a text property in a text layer does not unapply the previously applied token in the same property [Taiga #11337}(https://tree.taiga.io/project/penpot/issue/11337)
+- Fix changing a text property in a text layer does not unapply the previously applied token in the same property [Taiga #11337](https://tree.taiga.io/project/penpot/issue/11337)
 - Fix shortcut error pressing G+W from the View Mode [Taiga #11061](https://tree.taiga.io/project/penpot/issue/11061)
+- Fix entering long project name [Taiga #11417](https://tree.taiga.io/project/penpot/issue/11417)
+- Fix slow color picker [Taiga #11019](https://tree.taiga.io/project/penpot/issue/11019)
+- Fix tooltip position after click [Taiga #11405](https://tree.taiga.io/project/penpot/issue/11405)
+- Fix incorrect media translation on paste text with fill images [Github #6845](https://github.com/penpot/penpot/pull/6845)
 
 ## 2.7.2
 
@@ -70,7 +452,6 @@ on-premises instances** that want to keep up to date.
 - Fix copy in error message [GitHub #6615](https://github.com/penpot/penpot/pull/6615)
 - Fix url on invitation link [Taiga #11284](https://tree.taiga.io/project/penpot/issue/11284)
 
-
 ## 2.7.1
 
 ### :bug: Bugs fixed
@@ -78,7 +459,6 @@ on-premises instances** that want to keep up to date.
 - Fix incorrect handling of strokes with images on importing files
 - Fix tokens disappearing after manual additions [Taiga #11063](https://tree.taiga.io/project/penpot/issue/11063)
 
-
 ## 2.7.0
 
 ### :rocket: Epics and highlights
@@ -210,7 +590,6 @@ on-premises instances** that want to keep up to date.
 - Add character limitation to asset inputs [Taiga #10669](https://tree.taiga.io/project/penpot/issue/10669)
 - Fix Storybook link 'list of all available icons' wrong path [Taiga #10705](https://tree.taiga.io/project/penpot/issue/10705)
 
-
 ## 2.5.4
 
 ### :heart: Community contributions (Thank you!)
@@ -255,7 +634,7 @@ on-premises instances** that want to keep up to date.
 
 ### :boom: Breaking changes & Deprecations
 
-Although this is not a breaking change, we believe it’s important to highlight it in this
+Although this is not a breaking change, we believe it's important to highlight it in this
 section:
 
 This release includes a fix for an internal bug in Penpot that caused incorrect handling
@@ -263,9 +642,9 @@ of media assets (e.g., fill images). The issue has been resolved since version 2
 no new incorrect references will be generated. However, existing files may still contain
 incorrect references.
 
-To address this, we’ve provided a script to correct these references in existing files.
+To address this, we've provided a script to correct these references in existing files.
 
-While having incorrect references generally doesn’t result in visible issues, there are
+While having incorrect references generally doesn't result in visible issues, there are
 rare cases where it can cause problems. For example, if a component library (containing
 images) is deleted, and that library is being used in other files, running the FileGC task
 (responsible for freeing up space and performing logical deletions) could leave those
@@ -340,7 +719,6 @@ is a number of cores)
 - Fix missing methods reference on API Docs
 - Fix memory usage issue on file-gc asynchronous task (related to snapshots feature)
 
-
 ## 2.4.1
 
 ### :bug: Bugs fixed
@@ -348,7 +726,6 @@ is a number of cores)
 - Fix error when importing files with touched components [Taiga #9625](https://tree.taiga.io/project/penpot/issue/9625)
 - Fix problem when changing color libraries [Plugins #184](https://github.com/penpot/penpot-plugins/issues/184)
 
-
 ## 2.4.0
 
 ### :rocket: Epics and highlights
@@ -402,7 +779,6 @@ is a number of cores)
 
 - Add initial documentation for Kubernetes
 
-
 ## 2.3.1
 
 ### :bug: Bugs fixed
@@ -410,7 +786,6 @@ is a number of cores)
 - Fix unexpected issue on interaction between plugins sandbox and
   internal impl of promise
 
-
 ## 2.3.0
 
 ### :rocket: Epics and highlights
@@ -436,7 +811,6 @@ is a number of cores)
 
   You can enable it with the `enable-feature-text-editor-v2` configuration flag.
 
-
 ### :bug: Bugs fixed
 
 - Fix problem with constraints buttons [Taiga #8465](https://tree.taiga.io/project/penpot/issue/8465)
@@ -476,8 +850,8 @@ is a number of cores)
 ### :boom: Breaking changes & Deprecations
 
 - Removed "merge assets" option when exporting ".svg + .json" files. After the components changes the option wasn't
-working properly and we're planning to change the format soon. We think it's better to deprecate the option for the
+  working properly and we're planning to change the format soon. We think it's better to deprecate the option for the
-time being.
+  time being.
 
 ### :heart: Community contributions (Thank you!)
 
@@ -493,7 +867,7 @@ time being.
   freeing up space in the database. It can be enabled with the
   `enable-enable-tiered-file-data-storage` flag.
 
-  *(On-Premise feature, EXPERIMENTAL).*
+  _(On-Premise feature, EXPERIMENTAL)._
 
 - **JSON Interoperability for HTTP API** [Taiga #8372](https://tree.taiga.io/project/penpot/us/8372)
 
@@ -536,7 +910,7 @@ time being.
 
 - **Design System**
 
-  We implemented and subbed in new components from our Design System: `loader*` ([Taiga #8355](https://tree.taiga.io/project/penpot/task/8355))  and `tab-switcher*` ([Taiga #8518](https://tree.taiga.io/project/penpot/task/8518)).
+  We implemented and subbed in new components from our Design System: `loader*` ([Taiga #8355](https://tree.taiga.io/project/penpot/task/8355)) and `tab-switcher*` ([Taiga #8518](https://tree.taiga.io/project/penpot/task/8518)).
 
 - **Storybook** [Taiga #6329](https://tree.taiga.io/project/penpot/us/6329)
 
@@ -591,11 +965,11 @@ time being.
 
 ### :sparkles: New features
 
-- Consolidate templates new order and naming  [Taiga #8392](https://tree.taiga.io/project/penpot/task/8392)
+- Consolidate templates new order and naming [Taiga #8392](https://tree.taiga.io/project/penpot/task/8392)
 
 ### :bug: Bugs fixed
 
-- Fix the “search” label in translations [Taiga #8402](https://tree.taiga.io/project/penpot/issue/8402)
+- Fix the "search" label in translations [Taiga #8402](https://tree.taiga.io/project/penpot/issue/8402)
 - Fix pencil loader [Taiga #8348](https://tree.taiga.io/project/penpot/issue/8348)
 - Fix several issues on the OIDC.
 - Fix regression on the `email-verification` flag [Taiga #8398](https://tree.taiga.io/project/penpot/issue/8398)
@@ -675,22 +1049,21 @@ time being.
 - Fix color palette sorting [Taiga #7458](https://tree.taiga.io/project/penpot/issue/7458)
 - Fix style scoping problem with imported SVG [Taiga #7671](https://tree.taiga.io/project/penpot/issue/7671)
 
-
 ## 2.0.1
 
 ### :bug: Bugs fixed
 
 - Fix different issues related to components v2 migrations including [Github #4443](https://github.com/penpot/penpot/issues/4443)
 
-
 ## 2.0.0 - I Just Can't Get Enough
 
 ### :rocket: Epics and highlights
+
 - Grid CSS layout [Taiga #4915](https://tree.taiga.io/project/penpot/epic/4915)
 - UI redesign [Taiga #4958](https://tree.taiga.io/project/penpot/epic/4958)
 - New components System [Taiga #2662](https://tree.taiga.io/project/penpot/epic/2662)
 - Swap components [Taiga #1331](https://tree.taiga.io/project/penpot/us/1331)
-- Images as fill  [Taiga #2983](https://tree.taiga.io/project/penpot/us/2983)
+- Images as fill [Taiga #2983](https://tree.taiga.io/project/penpot/us/2983)
 - HTML code generation [Taiga #5277](https://tree.taiga.io/project/penpot/us/5277)
 - Light and dark themes [Taiga #2287](https://tree.taiga.io/project/penpot/us/2287)
 
@@ -699,9 +1072,9 @@ time being.
 - New strokes default to inside border [Taiga #6847](https://tree.taiga.io/project/penpot/issue/6847)
 - Change default z ordering on layers in flex layout. The previous behavior was inconsistent with how HTML works and we changed it to be more consistent. Previous layers that overlapped could be hidden, the fastest way to fix this is changing the z-index property but a better way is to change the order of your layers.
 
-
 ### :heart: Community contributions (Thank you!)
-- New Hausa, Yoruba and Igbo translations and  update translation files (by All For Tech Empowerment Foundation) [Taiga #6950](https://tree.taiga.io/project/penpot/us/6950), [Taiga #6534](https://tree.taiga.io/project/penpot/us/6534)
+
+- New Hausa, Yoruba and Igbo translations and update translation files (by All For Tech Empowerment Foundation) [Taiga #6950](https://tree.taiga.io/project/penpot/us/6950), [Taiga #6534](https://tree.taiga.io/project/penpot/us/6534)
 - Hide bounding-box when editing shape (by @VasilevsVV) [#3930](https://github.com/penpot/penpot/pull/3930)
 - CTRL + "+" to zoom into canvas instead of browser (by @audriu) [#3848](https://github.com/penpot/penpot/pull/3848)
 - Add dev deps.edn in the project root (by @PEZ) [#3794](https://github.com/penpot/penpot/pull/3794)
@@ -710,6 +1083,7 @@ time being.
 - Typo (by StephanEggermont) [#157](https://github.com/penpot/penpot-docs/pull/157)
 
 ### :sparkles: New features
+
 - Send comments with Ctrl+Enter / Cmd + Enter [Taiga #6085](https://tree.taiga.io/project/penpot/issue/6085)
 - Select through stroke only rectangle [Taiga #5484](https://tree.taiga.io/project/penpot/issue/5484)
 - Stroke default position [Taiga #6847](https://tree.taiga.io/project/penpot/issue/6847)
@@ -777,6 +1151,7 @@ time being.
 - [REDESIGN] Onboarding slides [Taiga #6678](https://tree.taiga.io/project/penpot/us/6678)
 
 ### :bug: Bugs fixed
+
 - Fix pixelated thumbnails [Github #3681](https://github.com/penpot/penpot/issues/3681), [Github #3661](https://github.com/penpot/penpot/issues/3661)
 - Fix problem with not applying colors to boards [Github #3941](https://github.com/penpot/penpot/issues/3941)
 - Fix problem with path editor undoing changes [Github #3998](https://github.com/penpot/penpot/issues/3998)
@@ -785,7 +1160,7 @@ time being.
 - Selecting from Color Palette does not work for board when there is no existing fill [Taiga #6464](https://tree.taiga.io/project/penpot/issue/6464)
 - Color thumbnails are consistently rounded in the inspect code mode [Taiga #5886](https://tree.taiga.io/project/penpot/issue/5886)
 - Adding vector path points before first point of existing open path not working [Taiga #6593](https://tree.taiga.io/project/penpot/issue/6593)
-- Some image formats include the extension when importing  [Taiga #5485](https://tree.taiga.io/project/penpot/issue/5485)
+- Some image formats include the extension when importing [Taiga #5485](https://tree.taiga.io/project/penpot/issue/5485)
 - Gradient color tool doesn't work properly with flipped items [Taiga #6485](https://tree.taiga.io/project/penpot/issue/6485)
 - [TEXT] Align options are not shown when several text are selected [Taiga #5948](https://tree.taiga.io/project/penpot/issue/5948)
 - [VIEW MODE] Comments not working properly on multiple pages [Taiga #6281](https://tree.taiga.io/project/penpot/issue/6281)
@@ -829,7 +1204,7 @@ time being.
 
 ### :sparkles: New features
 
-- Improve selected colors [Taiga #5805]( https://tree.taiga.io/project/penpot/us/5805)
+- Improve selected colors [Taiga #5805](https://tree.taiga.io/project/penpot/us/5805)
 
 ### :bug: Bugs fixed
 
@@ -864,7 +1239,6 @@ time being.
 - Fix deleted pages comments shown in right sidebar [Taiga #5648](https://tree.taiga.io/project/penpot/us/5648)
 - Fix tooltip on toggle visibility and toggle lock buttons [Taiga #5141](https://tree.taiga.io/project/penpot/issue/5141)
 
-
 ## 1.19.1
 
 ### :bug: Bugs fixed
@@ -978,7 +1352,6 @@ time being.
 
 - Update google fonts catalog (at 2023/07/06) [Taiga #5592](https://tree.taiga.io/project/penpot/issue/5592)
 
-
 ### :heart: Community contributions by (Thank you!)
 
 - Update Typography palette order (by @akshay-gupta7) [Github #3156](https://github.com/penpot/penpot/pull/3156)
@@ -1132,12 +1505,14 @@ time being.
 - Fix problem with opacity in imported SVG's [Taiga #4923](https://tree.taiga.io/project/penpot/issue/4923)
 
 ### :heart: Community contributions by (Thank you!)
+
 - To @ondrejkonec: for contributing to the code with:
 - Refactor CSS variables [Github #2948](https://github.com/penpot/penpot/pull/2948)
 
 ## 1.17.3
 
 ### :bug: Bugs fixed
+
 - Fix copy and paste very nested inside itself [Taiga #4848](https://tree.taiga.io/project/penpot/issue/4848)
 - Fix custom fonts not rendered correctly [Taiga #4874](https://tree.taiga.io/project/penpot/issue/4874)
 - Fix problem with shadows and blur on multiple selection
@@ -1170,6 +1545,7 @@ time being.
 ## 1.17.1
 
 ### :bug: Bugs fixed
+
 - Fix components groups items show the component name in list mode [Taiga #4770](https://tree.taiga.io/project/penpot/issue/4770)
 - Fix typing CMD+Z on MacOS turns the cursor into a Zoom cursor [Taiga #4778](https://tree.taiga.io/project/penpot/issue/4778)
 - Fix white space on small screens [Taiga #4774](https://tree.taiga.io/project/penpot/issue/4774)
@@ -1284,7 +1660,7 @@ time being.
 
 ### :boom: Breaking changes & Deprecations
 
-- Removed the support for v2 internal file data blob format.  This
+- Removed the support for v2 internal file data blob format. This
   version has never been documented nor set as default value so
   technically this is not a breaking change because we are removing
   a "private API".
@@ -1389,7 +1765,6 @@ time being.
 - Fix when ungrouping, the items previously grouped should ALWAYS remain selected [Taiga #4064](https://tree.taiga.io/project/penpot/issue/4064)
 - Change shortcut for "Clear undo" [#2219](https://github.com/penpot/penpot/issues/2219)
 
-
 ## 1.15.2-beta
 
 ### :bug: Bugs fixed
@@ -1473,6 +1848,7 @@ time being.
 - Fix bringing complete file data when launching the export dialog [Taiga #4006](https://tree.taiga.io/project/penpot/issue/4006)
 
 ### :arrow_up: Deps updates
+
 ### :heart: Community contributions by (Thank you!)
 
 ## 1.14.2-beta
@@ -1513,10 +1889,10 @@ time being.
 - Prototype connection should be under the rules [Taiga #3384](https://tree.taiga.io/project/penpot/issue/3384)
 - Fix problem with empty text boxes events [Taiga #3627](https://tree.taiga.io/project/penpot/issue/3627)
 
-
 ## 1.13.5-beta
 
 ### :bug: Bugs fixed
+
 - Fix orientation artboard preset not working with differently sized artboards [Taiga #3548](https://tree.taiga.io/project/penpot/issue/3548)
 - Fix background on export arboards [Taiga #1991](https://tree.taiga.io/project/penpot/issue/1991)
 
@@ -1660,6 +2036,7 @@ time being.
 - Fix problem when resizing a group with texts with auto-width/height [#3171](https://tree.taiga.io/project/penpot/issue/3171)
 
 ### :arrow_up: Deps updates
+
 ### :heart: Community contributions by (Thank you!)
 
 ## 1.12.4-beta
@@ -1677,7 +2054,7 @@ time being.
 ### :bug: Bugs fixed
 
 - Fix issue with shift+select to deselect shapes [Taiga #3154](https://tree.taiga.io/project/penpot/issue/3154)
-- Fix issue with drag-select shapes  [Taiga #3165](https://tree.taiga.io/project/penpot/issue/3165)
+- Fix issue with drag-select shapes [Taiga #3165](https://tree.taiga.io/project/penpot/issue/3165)
 - Fix issue on password persistence after registration process on private instances
 
 ## 1.12.2-beta
@@ -1695,7 +2072,6 @@ time being.
 - Fix length of names in sidebar [Taiga #2962](https://tree.taiga.io/project/penpot/issue/2962)
 - Fix issues on loki integration
 
-
 ## 1.12.0-beta
 
 ### :boom: Breaking changes

README.md

@@ -77,17 +77,14 @@ Provide your team or organization with a completely owned collaborative design t
 ### Integrations ###
 Penpot offers integration into the development toolchain, thanks to its support for webhooks and an API accessible through access tokens.
 
-### What’s great for design ###
+###  Building Design Systems: design tokens, components and variants  ###
-With Penpot you can design libraries to share and reuse; turn design elements into components and tokens to allow reusability and scalability; and build realistic user flows and interactions.
+Penpot brings design systems to code-minded teams: a single source of truth with native Design Tokens, Components, and Variants for scalable, reusable, and consistent UI across projects and platforms.
-
-### Design Tokens ###
-With Penpot’s standardized [design tokens](https://penpot.dev/collaboration/design-tokens) format, you can easily reuse and sync tokens across different platforms, workflows, and disciplines.
 
 
 <br />
 
 <p align="center">
-  <img src="https://img.plasmic.app/img-optimizer/v1/img?src=https%3A%2F%2Fimg.plasmic.app%2Fimg-optimizer%2Fv1%2Fimg%2F9dd677c36afb477e9666ccd1d3f009ad.png" alt="Open Source" style="width: 65%;">
+ <img src="https://github.com/user-attachments/assets/cce75ad6-f783-473f-8803-da9eb8255fef">
 </p>
 
 <br />

backend/deps.edn

@@ -3,10 +3,10 @@
 
  :deps
  {penpot/common {:local/root "../common"}
-  org.clojure/clojure {:mvn/version "1.12.1"}
+  org.clojure/clojure {:mvn/version "1.12.4"}
   org.clojure/tools.namespace {:mvn/version "1.5.0"}
 
-  com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.7-4"}
 
   io.prometheus/simpleclient {:mvn/version "0.16.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
@@ -17,7 +17,7 @@
 
   io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.7.0.RELEASE"}
+  io.lettuce/lettuce-core {:mvn/version "6.8.1.RELEASE"}
   ;; Minimal dependencies required by lettuce, we need to include them
   ;; explicitly because clojure dependency management does not support
   ;; yet the BOM format.
@@ -28,29 +28,30 @@
   com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.4"
+  {:git/tag "v11.9"
-   :git/sha "ce50d42"
+   :git/sha "5fad7a9"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 
   com.github.seancorfield/next.jdbc
-  {:mvn/version "1.3.1002"}
+  {:mvn/version "1.3.1070"}
+
   metosin/reitit-core {:mvn/version "0.9.1"}
-  nrepl/nrepl {:mvn/version "1.3.1"}
+  nrepl/nrepl {:mvn/version "1.4.0"}
 
-  org.postgresql/postgresql {:mvn/version "42.7.6"}
+  org.postgresql/postgresql {:mvn/version "42.7.9"}
-  org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
+  org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}
 
-  com.zaxxer/HikariCP {:mvn/version "6.3.0"}
+  com.zaxxer/HikariCP {:mvn/version "7.0.2"}
 
   io.whitfin/siphash {:mvn/version "2.0.0"}
 
   buddy/buddy-hashers {:mvn/version "2.0.167"}
   buddy/buddy-sign {:mvn/version "3.6.1-359"}
 
-  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.0"}
+  com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.3"}
 
-  org.jsoup/jsoup {:mvn/version "1.20.1"}
+  org.jsoup/jsoup {:mvn/version "1.21.2"}
   org.im4java/im4java
   {:git/tag "1.4.0-penpot-2"
    :git/sha "e2b3e16"
@@ -60,12 +61,12 @@
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
 
-  dawran6/emoji {:mvn/version "0.1.5"}
+  dawran6/emoji {:mvn/version "0.2.0"}
-  markdown-clj/markdown-clj {:mvn/version "1.12.3"}
+  markdown-clj/markdown-clj {:mvn/version "1.12.4"}
 
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
-  software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
+  software.amazon.awssdk/s3 {:mvn/version "2.41.21"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases
@@ -80,12 +81,14 @@
 
   :build
   {:extra-deps
-   {io.github.clojure/tools.build {:git/tag "v0.10.9" :git/sha "e405aac"}}
+   {io.github.clojure/tools.build {:mvn/version "0.10.10"}}
    :ns-default build}
 
   :test
   {:main-opts ["-m" "kaocha.runner"]
-   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"]
+   :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
+              "--sun-misc-unsafe-memory-access=allow"
+              "--enable-native-access=ALL-UNNAMED"]
    :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
 
   :outdated
@@ -94,8 +97,8 @@
 
   :jmx-remote
   {:jvm-opts ["-Dcom.sun.management.jmxremote"
-              "-Dcom.sun.management.jmxremote.port=9090"
+              "-Dcom.sun.management.jmxremote.port=9000"
-              "-Dcom.sun.management.jmxremote.rmi.port=9090"
+              "-Dcom.sun.management.jmxremote.rmi.port=9000"
               "-Dcom.sun.management.jmxremote.local.only=false"
               "-Dcom.sun.management.jmxremote.authenticate=false"
               "-Dcom.sun.management.jmxremote.ssl=false"

backend/dev/user.clj

@@ -6,12 +6,14 @@
 
 (ns user
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.debug :as debug]
    [app.common.exceptions :as ex]
    [app.common.files.helpers :as cfh]
    [app.common.fressian :as fres]
    [app.common.geom.matrix :as gmt]
+   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.perf :as perf]
    [app.common.pprint :as pp]
@@ -19,20 +21,22 @@
    [app.common.schema.desc-js-like :as smdj]
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.generators :as sg]
+   [app.common.schema.openapi :as oapi]
    [app.common.spec :as us]
-   [app.common.json :as json]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]
-   [app.srepl.helpers :as srepl.helpers]
+   [app.srepl.helpers :as h]
-   [app.srepl.main :as srepl]
+   [app.srepl.main :refer :all]
    [app.util.blob :as blob]
-   [app.util.time :as dt]
    [clj-async-profiler.core :as prof]
    [clojure.contrib.humanize :as hum]
+   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]

backend/resources/app/email/change-email/en.html

@@ -193,7 +193,7 @@
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
                       <div
                         style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
-                        Click to the link below to confirm the change:</div>
+                        Click the link below to confirm the change.</div>
                     </td>
                   </tr>
                   <tr>
@@ -217,8 +217,7 @@
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
                       <div
                         style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">
-                        If you received this email by mistake, please consider changing your password for security
+                        If you did not request this change, consider changing your password for security reasons.</div>
-                        reasons.</div>
                     </td>
                   </tr>
                   <tr>

backend/resources/app/email/change-email/en.txt

@@ -2,12 +2,11 @@ Hello {{name|abbreviate:25}}!
 
 We received a request to change your current email to {{ pending-email }}.
 
-Click to the link below to confirm the change:
+Click the link below to confirm the change.
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
-If you received this email by mistake, please consider changing your password
+If you did not request this change, consider changing your password for security reasons.
-for security reasons.
 
 Enjoy!
 The Penpot team.

backend/resources/app/email/feedback/en.html

@@ -8,38 +8,41 @@
   <body>
     <p>
       <strong>Feedback from:</strong><br />
-      {% if profile %}
+      <span>
-        <span>
+        <span>Name: </span>
-          <span>Name: </span>
+        <span><code>{{profile.fullname|abbreviate:25}}</code></span>
-          <span><code>{{profile.fullname|abbreviate:25}}</code></span>
+      </span>
-        </span>
+      <br />
-        <br />
+      <span>
-
+        <span>Email: </span>
-        <span>
+        <span>{{profile.email}}</span>
-          <span>Email: </span>
+      </span>
-          <span>{{profile.email}}</span>
+      <br />
-        </span>
+      <span>
-        <br />
+        <span>ID: </span>
-
+        <span><code>{{profile.id}}</code></span>
-        <span>
+      </span>
-          <span>ID: </span>
-          <span><code>{{profile.id}}</code></span>
-        </span>
-      {% else %}
-        <span>
-          <span>Email: </span>
-          <span>{{profile.email}}</span>
-        </span>
-      {% endif %}
     </p>
     <p>
       <strong>Subject:</strong><br />
-      <span>{{subject|abbreviate:300}}</span>
+      <span>{{feedback-subject|abbreviate:300}}</span>
     </p>
 
+    <p>
+      <strong>Type:</strong><br />
+      <span>{{feedback-type|abbreviate:300}}</span>
+    </p>
+
+    {% if feedback-error-href %}
+    <p>
+      <strong>Error HREF:</strong><br />
+      <span>{{feedback-error-href|abbreviate:500}}</span>
+    </p>
+    {% endif %}
+
     <p>
       <strong>Message:</strong><br />
-      {{content|linebreaks-br|safe}}
+      {{feedback-content|linebreaks-br}}
     </p>
   </body>
 </html>

backend/resources/app/email/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{subject}}
+[PENPOT FEEDBACK]: {{feedback-subject}}

backend/resources/app/email/feedback/en.txt

@@ -1,9 +1,11 @@
-{% if profile %}
+From: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
-Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+Subject: {{feedback-subject}}
-{% else %}
+Type: {{feedback-type}}
-Feedback from: {{email}}
-{% endif %}
 
-Subject: {{subject}}
+{% if feedback-error-href %}
+HREF: {{feedback-error-href}}
+{% endif -%}
 
-{{content}}
+Message:
+
+{{feedback-content}}

backend/resources/app/email/invite-to-team/en.html

@@ -240,4 +240,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/email/invite-to-team/en.subj

@@ -1 +1 @@
-Invitation to join {{team}}
+{{invited-by|abbreviate:25}} has invited you to join the team “{{ team|abbreviate:25 }}”

backend/resources/app/onboarding.edn

@@ -1,6 +1,9 @@
 [{:id "tokens-starter-kit"
   :name "Design tokens starter kit"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"},
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"}
+ {:id "penpot-design-system"
+  :name "Penpot Design System | Pencil"
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Pencil-Penpot-Design-System.penpot"}
  {:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}
@@ -10,9 +13,6 @@
  {:id "plants-app"
   :name "UI mockup example"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/Plants-app.penpot"}
- {:id "penpot-design-system"
-  :name "Design system example"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Penpot%20-%20Design%20System%20v2.1.penpot"}
  {:id "tutorial-for-beginners"
   :name "Tutorial for beginners"
   :file-uri "https://github.com/penpot/penpot-files/raw/main/tutorial-for-beginners.penpot"}

backend/resources/app/templates/api-doc.tmpl

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="robots" content="noindex,nofollow">
     <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>Builtin API Documentation - Penpot</title>
+    <title>{{label|upper}} API Documentation</title>
 
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
@@ -19,7 +19,7 @@
   <body>
     <main>
       <header>
-        <h1>Penpot API Documentation (v{{version}})</h1>
+        <h1>{{label|upper}}: API Documentation (v{{version}})</h1>
         <small class="menu">
           [
           <nav>
@@ -31,9 +31,10 @@
       </header>
       <section class="doc-content">
         <h2>INTRODUCTION</h2>
-        <p>This documentation is intended to be a general overview of the penpot RPC API.
+        <p>This documentation is intended to be a general overview of
-          If you prefer, you can use <a href="/api/openapi.json">OpenAPI</a>
+          the {{label}} API.  If you prefer, you can
-          and/or <a href="/api/openapi">SwaggerUI</a> as alternative.</p>
+          use <a href="{{openapi}}">Swagger/OpenAPI</a> as
+          alternative.</p>
 
         <h2>GENERAL NOTES</h2>
 
@@ -43,7 +44,7 @@
         that starts with <b>get-</b> in the name, can use GET HTTP
         method which in many cases benefits from the HTTP cache.</p>
 
-
+        {% block auth-section %}
         <h3>Authentication</h3>
         <p>The penpot backend right now offers two way for authenticate the request:
         <b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
@@ -56,9 +57,10 @@
         <p>The access token can be obtained on the appropriate section on profile settings
         and it should be provided using <b>`Authorization`</b> header with <b>`Token
         &lt;token-string&gt;`</b> value.</p>
+        {% endblock %}
 
         <h3>Content Negotiation</h3>
-        <p>The penpot API by default operates indistinctly with: <b>`application/json`</b>
+        <p>This API operates indistinctly with: <b>`application/json`</b>
         and <b>`application/transit+json`</b> content types. You should specify the
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
@@ -75,13 +77,16 @@
         standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
         API</a></p>
 
+        {% block limits-section %}
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share
         the same rate limit). For now the limits are not documented because we are
         studying and analyzing the data. As a general rule, it should not be abused, if an
         abusive use is detected, we will proceed to block the user's access to the
         API.</p>
+        {% endblock %}
 
+        {% block webhooks-section %}
         <h3>Webhooks</h3>
         <p>All methods that emit webhook events are marked with flag <b>WEBHOOK</b>, the
         data structure defined on each method represents the <i>payload</i> of the
@@ -97,9 +102,11 @@
   "profileId": "db601c95-045f-808b-8002-361312e63531"
 }
         </pre>
+        {% endblock %}
+
       </section>
       <section class="rpc-doc-content">
-        <h2>RPC METHODS REFERENCE:</h2>
+        <h2>METHODS REFERENCE:</h2>
         <ul class="rpc-items">
           {% for item in methods %}
             {% include "app/templates/api-doc-entry.tmpl" with item=item %}

backend/resources/app/templates/debug.tmpl

@@ -17,38 +17,6 @@ Debug Main Page
       <desc><a href="/dbg/error">CLICK HERE TO SEE THE ERROR REPORTS</a> </desc>
     </fieldset>
 
-    <fieldset>
-      <legend>Download file data:</legend>
-      <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
-      <form method="get" action="/dbg/file/data">
-        <div class="row">
-          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
-        </div>
-        <div class="row">
-          <input type="submit" name="download" value="Download" />
-          <input type="submit" name="clone" value="Clone" />
-        </div>
-      </form>
-    </fieldset>
-
-    <fieldset>
-      <legend>Upload File Data:</legend>
-      <desc>Create a new file on your draft projects using the file downloaded from the previous section.</desc>
-      <form method="post" enctype="multipart/form-data" action="/dbg/file/data">
-        <div class="row">
-          <input type="file" name="file" value="" />
-        </div>
-        <div class="row">
-          <label>Import with same id?</label>
-          <input type="checkbox" name="reuseid" />
-        </div>
-
-        <div class="row">
-          <input type="submit" value="Upload" />
-        </div>
-      </form>
-    </fieldset>
-
     <fieldset>
       <legend>Profile Management</legend>
       <form method="post" action="/dbg/actions/resend-email-verification">
@@ -77,10 +45,88 @@ Debug Main Page
       </form>
     </fieldset>
 
+    <fieldset>
+      <legend>VIRTUAL CLOCK</legend>
 
+      <desc>
+        <p>
+          CURRENT CLOCK: <b>{{current-clock}}</b>
+          <br />
+          CURRENT OFFSET: <b>{{current-offset}}</b>
+          <br />
+          CURRENT TIME: <b>{{current-time}}</b>
+        </p>
+
+        <p>Examples: 3h, -7h, 24h (allowed suffixes: h, s)</p>
+      </desc>
+
+      <form method="post" action="/dbg/actions/set-virtual-clock">
+        <div class="row">
+          <input type="text" name="offset" placeholder="3h" value="" />
+        </div>
+
+        <div class="row">
+          <label for="force-verify">Are you sure?</label>
+          <input id="force-verify" type="checkbox" name="force" />
+          <br />
+          <small>
+            This is a just a security double check for prevent non intentional submits.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" name="submit" value="Submit" />
+          <input type="submit" name="reset" value="Reset" />
+        </div>
+      </form>
+    </fieldset>
 
   </section>
 
+  <section class="widget">
+
+    <fieldset>
+      <legend>Download RAW file data:</legend>
+      <desc>Given an FILE-ID, downloads the file AS-IS (no validation
+        checks, just exports the file data and related objects in raw)
+
+        <br/>
+        <br/>
+        <b>WARNING: this operation does not performs any checks</b>
+      </desc>
+      <form method="get" action="/dbg/actions/file-raw-export-import">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="submit" name="download" value="Download" />
+          <input type="submit" name="clone" value="Clone" />
+        </div>
+      </form>
+    </fieldset>
+
+    <fieldset>
+      <legend>Upload File Data:</legend>
+      <desc>Create a new file on your draft projects using the file downloaded from the previous section.
+        <br/>
+        <br/>
+        <b>WARNING: this operation does not performs any checks</b>
+      </desc>
+      <form method="post" enctype="multipart/form-data" action="/dbg/actions/file-raw-export-import">
+        <div class="row">
+          <input type="file" name="file" value="" />
+        </div>
+        <div class="row">
+          <label>Import with same id?</label>
+          <input type="checkbox" name="reuseid" />
+        </div>
+
+        <div class="row">
+          <input type="submit" value="Upload" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
   <section class="widget">
     <fieldset>
       <legend>Export binfile:</legend>
@@ -88,7 +134,7 @@ Debug Main Page
       the related libraries in a single custom formatted binary
       file.</desc>
 
-      <form method="get" action="/dbg/file/export">
+      <form method="get" action="/dbg/actions/file-export">
         <div class="row set-of-inputs">
           <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
           <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
@@ -116,7 +162,7 @@ Debug Main Page
       <legend>Import binfile:</legend>
       <desc>Import penpot file in binary format.</desc>
 
-      <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
+      <form method="post" enctype="multipart/form-data" action="/dbg/actions/file-import">
         <div class="row">
           <input type="file" name="file" value="" />
         </div>
@@ -130,79 +176,27 @@ Debug Main Page
 
   <section class="widget">
     <fieldset>
-      <legend>Reset file version</legend>
+      <legend>Feature Flags for Team</legend>
-      <desc>Allows reset file data version to a specific number/</desc>
-
-      <form method="post" action="/dbg/actions/reset-file-version">
-        <div class="row">
-          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
-        </div>
-        <div class="row">
-          <input type="number" style="width:100px" name="version" placeholder="version" value="32" />
-        </div>
-
-        <div class="row">
-          <label for="force-version">Are you sure?</label>
-          <input id="force-version" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-
-        <div class="row">
-          <input type="submit" value="Submit" />
-        </div>
-      </form>
-    </fieldset>
-  </section>
-
-  <section class="widget">
-    <h2>Feature Flags</h2>
-    <fieldset>
-      <legend>Enable</legend>
       <desc>Add a feature flag to a team</desc>
-      <form method="post" action="/dbg/actions/add-team-feature">
+      <form method="post" action="/dbg/actions/handle-team-features">
         <div class="row">
           <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
         </div>
         <div class="row">
-          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
+          <select type="text" style="width:100px" name="feature">
+            {% for feature in supported-features %}
+              <option value="{{feature}}">{{feature}}</option>
+            {% endfor %}
+          </select>
         </div>
 
         <div class="row">
-          <label for="check-feature">Skip feature check</label>
+          <select style="width:100px" name="action">
-          <input id="check-feature" type="checkbox" name="skip-check" />
+            <option value="">Action...</option>
-          <br />
+            <option value="show">Show</option>
-          <small>
+            <option value="enable">Enable</option>
-            Do not check if the feature is supported
+            <option value="disable">Disable</option>
-          </small>
+          </select>
-        </div>
-
-        <div class="row">
-          <label for="force-version">Are you sure?</label>
-          <input id="force-version" type="checkbox" name="force" />
-          <br />
-          <small>
-            This is a just a security double check for prevent non intentional submits.
-          </small>
-        </div>
-
-        <div class="row">
-          <input type="submit" value="Submit" />
-        </div>
-      </form>
-    </fieldset>
-    <fieldset>
-      <legend>Disable</legend>
-      <desc>Remove a feature flag from a team</desc>
-      <form method="post" action="/dbg/actions/remove-team-feature">
-        <div class="row">
-          <input type="text" style="width:300px" name="team-id" placeholder="team-id" />
-        </div>
-        <div class="row">
-          <input type="text" style="width:100px" name="feature" placeholder="feature" value="" />
         </div>
 
         <div class="row">

backend/resources/app/templates/error-list.tmpl

@@ -7,7 +7,9 @@ penpot - error list
 {% block content %}
 <nav>
   <div class="title">
-    <h1>Error reports (last 200)</h1>
+    <h1>Error reports (last 200)
+      <a href="/dbg">[GO BACK]</a>
+    </h1>
   </div>
 </nav>
 <main class="horizontal-list">

backend/resources/app/templates/main-api-doc.tmpl

@@ -0,0 +1 @@
+{% extends "app/templates/api-doc.tmpl" %}

backend/resources/app/templates/management-api-doc.tmpl

@@ -0,0 +1,10 @@
+{% extends "app/templates/api-doc.tmpl" %}
+
+{% block auth-section %}
+{% endblock %}
+
+{% block limits-section %}
+{% endblock %}
+
+{% block webhooks-section %}
+{% endblock %}

backend/resources/app/templates/openapi.tmpl

@@ -7,7 +7,7 @@
     name="description"
     content="SwaggerUI"
   />
-  <title>PENPOT Swagger UI</title>
+  <title>{{label|upper}} API</title>
   <style>{{swagger-css|safe}}</style>
 </head>
 <body>
@@ -16,7 +16,7 @@
 <script>
   window.onload = () => {
     window.ui = SwaggerUIBundle({
-      url: '{{public-uri}}/api/openapi.json',
+      url: '{{uri}}',
       dom_id: '#swagger-ui',      
       presets: [
         SwaggerUIBundle.presets.apis,

backend/resources/log4j2-devenv-repl.xml

@@ -25,8 +25,7 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http" level="info" />
-    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/resources/log4j2-devenv.xml

@@ -25,8 +25,7 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http" level="info" />
-    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/scripts/_env

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key
+export PENPOT_SECRET_KEY=super-secret-devenv-key
+export PENPOT_HOST=devenv
+export PENPOT_PUBLIC_URI=https://localhost:3449
+
+export PENPOT_FLAGS="\
+       $PENPOT_FLAGS \
+       enable-login-with-password
+       disable-login-with-ldap \
+       disable-login-with-oidc \
+       disable-login-with-google \
+       disable-login-with-github \
+       disable-login-with-gitlab \
+       enable-backend-worker \
+       enable-backend-asserts \
+       disable-feature-fdata-pointer-map \
+       enable-feature-fdata-objects-map \
+       enable-audit-log \
+       enable-transit-readable-response \
+       enable-demo-users \
+       enable-user-feedback \
+       disable-secure-session-cookies \
+       enable-smtp \
+       enable-prepl-server \
+       enable-urepl-server \
+       enable-rpc-climit \
+       enable-rpc-rlimit \
+       enable-quotes \
+       enable-soft-rpc-rlimit \
+       enable-auto-file-snapshot \
+       enable-webhooks \
+       enable-access-tokens \
+       disable-tiered-file-data-storage \
+       enable-file-validation \
+       enable-file-schema-validation \
+       enable-redis-cache \
+       enable-subscriptions";
+
+# Default deletion delay for devenv
+export PENPOT_DELETION_DELAY="24h"
+
+# Setup default upload media file size to 100MiB
+export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
+
+# Setup default multipart upload size to 300MiB
+export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
+
+export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com"
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_OBJECTS_STORAGE_BACKEND=s3
+export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
+export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
+
+export JAVA_OPTS="\
+       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -Djdk.attach.allowAttachSelf \
+       -Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -Djdk.tracePinnedThreads=full \
+       -Dim4java.useV7=true \
+       -XX:+UnlockExperimentalVMOptions \
+       -XX:+UseShenandoahGC \
+       -XX:+UseCompactObjectHeaders \
+       -XX:ShenandoahGCMode=generational \
+       -XX:-OmitStackTraceInFastThrow  \
+       --sun-misc-unsafe-memory-access=allow \
+       --enable-preview \
+       --enable-native-access=ALL-UNNAMED";
+
+function setup_minio() {
+    # Initialize MINIO config
+    mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+    mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
+    mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
+    if [ "$?" = "1" ]; then
+        mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
+    fi
+    mc mb penpot-s3/penpot -p -q
+}
+
+

backend/scripts/manage.py

@@ -71,19 +71,27 @@ def run_cmd(params):
         print("EXC:", str(cause))
         sys.exit(-2)
 
-def create_profile(fullname, email, password):
+def create_profile(fullname, email, password, skip_tutorial=False, skip_walkthrough=False):
+    props = {}
+    if skip_tutorial:
+        props["viewed-tutorial?"] = True
+    if skip_walkthrough:
+        props["viewed-walkthrough?"] = True
+
     params = {
         "cmd": "create-profile",
         "params": {
             "fullname": fullname,
             "email": email,
-            "password": password
+            "password": password,
+            **props
         }
     }
 
     res = run_cmd(params)
     print(f"Created: {res['email']} / {res['id']}")
 
+
 def update_profile(email, fullname, password, is_active):
     params = {
         "cmd": "update-profile",
@@ -170,6 +178,8 @@ parser.add_argument("-n", "--fullname", help="fullname", action="store")
 parser.add_argument("-e", "--email", help="email", action="store")
 parser.add_argument("-p", "--password", help="password", action="store")
 parser.add_argument("-c", "--connect", help="connect to PREPL", action="store", default="tcp://localhost:6063")
+parser.add_argument("--skip-tutorial", help="mark tutorial as viewed", action="store_true")
+parser.add_argument("--skip-walkthrough", help="mark walkthrough as viewed", action="store_true")
 
 args = parser.parse_args()
 

backend/scripts/repl

@@ -1,112 +1,13 @@
 #!/usr/bin/env bash
 
-export PENPOT_SECRET_KEY=super-secret-devenv-key
+SCRIPT_DIR=$(dirname $0);
-export PENPOT_HOST=devenv
+source $SCRIPT_DIR/_env;
-export PENPOT_FLAGS="\
-       $PENPOT_FLAGS \
-       enable-login-with-ldap \
-       enable-login-with-password
-       enable-login-with-oidc \
-       enable-login-with-google \
-       enable-login-with-github \
-       enable-login-with-gitlab \
-       enable-backend-worker \
-       enable-backend-asserts \
-       enable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
-       enable-audit-log \
-       enable-transit-readable-response \
-       enable-demo-users \
-       disable-secure-session-cookies \
-       enable-smtp \
-       enable-prepl-server \
-       enable-urepl-server \
-       enable-rpc-climit \
-       enable-rpc-rlimit \
-       enable-quotes \
-       enable-soft-rpc-rlimit \
-       enable-auto-file-snapshot \
-       enable-webhooks \
-       enable-access-tokens \
-       enable-tiered-file-data-storage \
-       enable-file-validation \
-       enable-file-schema-validation \
-       enable-subscriptions \
-       enable-subscriptions-old";
-
-# Default deletion delay for devenv
-export PENPOT_DELETION_DELAY="24h"
-
-# Setup default upload media file size to 100MiB
-export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
-
-# Setup default multipart upload size to 300MiB
-export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
-
-# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
-# export PENPOT_DATABASE_USERNAME="penpot"
-# export PENPOT_DATABASE_PASSWORD="penpot"
-# export PENPOT_DATABASE_READONLY=true
-
-# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
-# export PENPOT_DATABASE_USERNAME="penpot_pre"
-# export PENPOT_DATABASE_PASSWORD="penpot_pre"
-
-# export PENPOT_LOGGERS_LOKI_URI="http://172.17.0.1:3100/loki/api/v1/push"
-# export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
 
 # Initialize MINIO config
-mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+setup_minio;
-mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-if [ "$?" = "1" ]; then
-    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-fi
-mc mb penpot-s3/penpot -p -q
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-
-export PENPOT_OBJECTS_STORAGE_BACKEND=s3
-export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
-export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
-export PENPOT_OBJECTS_STORAGE_FS_DIRECTORY="assets"
-
-export JAVA_OPTS="\
-       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -Djdk.attach.allowAttachSelf \
-       -Dlog4j2.configurationFile=log4j2-devenv-repl.xml \
-       -Djdk.tracePinnedThreads=full \
-       -Dim4java.useV7=true \
-       -XX:+EnableDynamicAgentLoading \
-       -XX:-OmitStackTraceInFastThrow  \
-       -XX:+UnlockDiagnosticVMOptions \
-       -XX:+DebugNonSafepoints \
-       --sun-misc-unsafe-memory-access=allow \
-       --enable-preview \
-       --enable-native-access=ALL-UNNAMED";
 
+export JAVA_OPTS="$JAVA_OPTS -Dlog4j2.configurationFile=log4j2-devenv-repl.xml"
 export OPTIONS="-A:jmx-remote -A:dev"
-
-# Setup HEAP
-# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
-# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
-
-# Increase virtual thread pool size
-# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
-
-# Disable C2 Compiler
-# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
-
-# Disable all compilers
-# export OPTIONS="$OPTIONS -J-Xint"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
-
-# Setup GC
-# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
-
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 

backend/scripts/repl-test

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-source /home/penpot/environ
-export PENPOT_FLAGS="$PENPOT_FLAGS disable-backend-worker"
-
-export OPTIONS="
-       -A:jmx-remote -A:dev \
-       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -J-Djdk.attach.allowAttachSelf \
-       -J-Dlog4j2.configurationFile=log4j2-experiments.xml \
-       -J-XX:-OmitStackTraceInFastThrow \
-       -J-XX:+UnlockDiagnosticVMOptions \
-       -J-XX:+DebugNonSafepoints \
-       -J-Djdk.tracePinnedThreads=full \
-       -J-XX:+UseTransparentHugePages \
-       -J-XX:ReservedCodeCacheSize=1g \
-       -J-Dpolyglot.engine.WarnInterpreterOnly=false \
-       -J--enable-preview";
-
-# Setup HEAP
-export OPTIONS="$OPTIONS -J-Xms320g -J-Xmx320g -J-XX:+AlwaysPreTouch"
-
-export PENPOT_HTTP_SERVER_IO_THREADS=2
-export PENPOT_HTTP_SERVER_WORKER_THREADS=2
-
-# Increase virtual thread pool size
-# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
-
-# Disable C2 Compiler
-# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
-
-# Disable all compilers
-# export OPTIONS="$OPTIONS -J-Xint"
-
-# Setup GC
-export OPTIONS="$OPTIONS -J-XX:+UseG1GC -J-Xlog:gc:logs/gc.log"
-
-# Setup GC
-#export OPTIONS="$OPTIONS -J-XX:+UseZGC -J-XX:+ZGenerational -J-Xlog:gc:logs/gc.log"
-
-# Enable ImageMagick v7.x support
-# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
-
-export OPTIONS_EVAL="nil"
-# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
-
-set -ex
-exec clojure $OPTIONS -M -e "$OPTIONS_EVAL" -m rebel-readline.main

backend/scripts/run

@@ -1,44 +1,13 @@
 #!/usr/bin/env bash
 
-export PENPOT_SECRET_KEY=super-secret-devenv-key
+SCRIPT_DIR=$(dirname $0);
-export PENPOT_HOST=devenv
-export PENPOT_FLAGS="\
-       $PENPOT_FLAGS \
-       enable-backend-asserts \
-       enable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
-       enable-file-snapshot \
-       enable-tiered-file-data-storage";
 
-export JAVA_OPTS="
+source $SCRIPT_DIR/_env;
-       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+export OPTIONS="-A:dev"
-       -Djdk.attach.allowAttachSelf \
-       -Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -XX:+EnableDynamicAgentLoading \
-       -XX:-OmitStackTraceInFastThrow \
-       -XX:+UnlockDiagnosticVMOptions \
-       -XX:+DebugNonSafepoints";
-
-export CLOJURE_OPTIONS="-A:dev"
-
-
-# Default deletion delay for devenv
-export PENPOT_DELETION_DELAY="24h"
-
-# Setup default upload media file size to 100MiB
-export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
-
-# Setup default multipart upload size to 300MiB
-export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-export PENPOT_OBJECTS_STORAGE_BACKEND=s3
-export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
-export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
 
 entrypoint=${1:-app.main};
+
 shift 1;
 set -ex
 
-clojure $CLOJURE_OPTIONS -A:dev -M -m $entrypoint "$@";
+exec clojure $OPTIONS -A:dev -M -m $entrypoint "$@";

backend/scripts/start-dev

@@ -1,70 +1,11 @@
 #!/usr/bin/env bash
 
-export PENPOT_SECRET_KEY=super-secret-devenv-key
+SCRIPT_DIR=$(dirname $0);
-export PENPOT_HOST=devenv
+source $SCRIPT_DIR/_env;
-export PENPOT_FLAGS="\
-       $PENPOT_FLAGS \
-       enable-prepl-server \
-       enable-urepl-server \
-       enable-nrepl-server \
-       enable-webhooks \
-       enable-backend-asserts \
-       enable-audit-log \
-       enable-login-with-ldap \
-       enable-transit-readable-response \
-       enable-demo-users \
-       enable-feature-fdata-pointer-map \
-       enable-feature-fdata-objects-map \
-       disable-secure-session-cookies \
-       enable-rpc-climit \
-       enable-smtp \
-       enable-quotes \
-       enable-file-snapshot \
-       enable-access-tokens \
-       enable-tiered-file-data-storage \
-       enable-file-validation \
-       enable-file-schema-validation \
-       enable-subscriptions \
-       enable-subscriptions-old ";
-
-# Default deletion delay for devenv
-export PENPOT_DELETION_DELAY="24h"
-
-# Setup default upload media file size to 100MiB
-export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
-
-# Setup default multipart upload size to 300MiB
-export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
 # Initialize MINIO config
-mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
+setup_minio;
-mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
-mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
-if [ "$?" = "1" ]; then
-    mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
-fi
-mc mb penpot-s3/penpot -p -q
-
-export AWS_ACCESS_KEY_ID=penpot-devenv
-export AWS_SECRET_ACCESS_KEY=penpot-devenv
-export PENPOT_OBJECTS_STORAGE_BACKEND=s3
-export PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=http://minio:9000
-export PENPOT_OBJECTS_STORAGE_S3_BUCKET=penpot
-
-entrypoint=${1:-app.main};
-
-export JAVA_OPTS="\
-       -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
-       -Djdk.attach.allowAttachSelf \
-       -Dlog4j2.configurationFile=log4j2-devenv.xml \
-       -Djdk.tracePinnedThreads=full \
-       -Dim4java.useV7=true \
-       -XX:-OmitStackTraceInFastThrow  \
-       --sun-misc-unsafe-memory-access=allow \
-       --enable-preview \
-       --enable-native-access=ALL-UNNAMED";
-
-export OPTIONS="-A:jmx-remote -A:dev"
 
+shift 1;
 set -ex
-clojure $OPTIONS -M -m $entrypoint;
+exec clojure -A:jmx-remote -A:dev -M -m app.main "$@";

backend/src/app/binfile/common.clj

@@ -15,19 +15,21 @@
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.common.weak :as weak]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.features.fdata :as feat.fdata]
+   [app.features.fdata :as fdata]
-   [app.features.file-migrations :as feat.fmigr]
+   [app.features.file-migrations :as fmigr]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
-   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
    [cuerdas.core :as str]
@@ -38,6 +40,7 @@
 
 (def ^:dynamic *state* nil)
 (def ^:dynamic *options* nil)
+(def ^:dynamic *reference-file* nil)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
@@ -53,17 +56,12 @@
   (* 1024 1024 100))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (declare get-resolved-file-libraries)
+(declare update-file!)
 
 (def file-attrs
-  #{:id
+  (sm/keys ctf/schema:file))
-    :name
-    :migrations
-    :features
-    :project-id
-    :is-shared
-    :version
-    :data})
 
 (defn parse-file-format
   [template]
@@ -143,32 +141,176 @@
   ([index coll attr]
    (reduce #(index-object %1 %2 attr) index coll)))
 
-(defn decode-row
+(defn- decode-row-features
-  [{:keys [data changes features] :as row}]
+  [{:keys [features] :as row}]
   (when row
     (cond-> row
-      features (assoc :features (db/decode-pgarray features #{}))
+      (db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
-      changes  (assoc :changes (blob/decode changes))
-      data     (assoc :data (blob/decode data)))))
 
+(def sql:get-minimal-file
+  "SELECT f.id,
+          f.revn,
+          f.modified_at,
+          f.deleted_at
+     FROM file AS f
+    WHERE f.id = ?")
 
-(defn decode-file
+(defn get-minimal-file
-  "A general purpose file decoding function that resolves all external
+  [cfg id & {:as opts}]
-  pointers, run migrations and return plain vanilla file map"
+  (db/get-with-sql cfg [sql:get-minimal-file id] opts))
-  [cfg {:keys [id] :as file}]
-  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-    (let [file (->> file
-                    (feat.fmigr/resolve-applied-migrations cfg)
-                    (feat.fdata/resolve-file-data cfg))
-          libs (delay (get-resolved-file-libraries cfg file))]
 
-      (-> file
+(def sql:files-with-data
-          (update :features db/decode-pgarray #{})
+  "SELECT f.id,
-          (update :data blob/decode)
+          f.project_id,
-          (update :data feat.fdata/process-pointers deref)
+          f.created_at,
-          (update :data feat.fdata/process-objects (partial into {}))
+          f.modified_at,
-          (update :data assoc :id id)
+          f.deleted_at,
-          (fmg/migrate-file libs)))))
+          f.name,
+          f.is_shared,
+          f.has_media_trimmed,
+          f.revn,
+          f.data AS legacy_data,
+          f.ignore_sync_until,
+          f.comment_thread_seqn,
+          f.features,
+          f.version,
+          f.vern,
+          p.team_id,
+          coalesce(fd.backend, 'legacy-db') AS backend,
+          fd.metadata AS metadata,
+          fd.data AS data
+     FROM file AS f
+     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
+    INNER JOIN project AS p ON (p.id = f.project_id)")
+
+(def sql:get-file
+  (str sql:files-with-data " WHERE f.id = ?"))
+
+(def sql:get-file-without-data
+  (str "WITH files AS (" sql:files-with-data ")"
+       "SELECT f.id,
+               f.project_id,
+               f.created_at,
+               f.modified_at,
+               f.deleted_at,
+               f.name,
+               f.is_shared,
+               f.has_media_trimmed,
+               f.revn,
+               f.ignore_sync_until,
+               f.comment_thread_seqn,
+               f.features,
+               f.version,
+               f.vern,
+               f.team_id
+          FROM files AS f
+         WHERE f.id = ?"))
+
+(defn- migrate-file
+  [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)
+            pmap/*tracked* (pmap/create-tracked)]
+    (let [libs (delay (get-resolved-file-libraries cfg file))
+          ;; For avoid unnecesary overhead of creating multiple
+          ;; pointers and handly internally with objects map in their
+          ;; worst case (when probably all shapes and all pointers
+          ;; will be readed in any case), we just realize/resolve them
+          ;; before applying the migration to the file.
+          file (-> (fdata/realize cfg file)
+                   (fmg/migrate-file libs))]
+
+      (if (or read-only? (db/read-only? conn))
+        file
+        (do ;; When file is migrated, we break the rule of no
+          ;; perform mutations on get operations and update the
+          ;; file with all migrations applied
+          (update-file! cfg file)
+          (fmigr/resolve-applied-migrations cfg file))))))
+
+(defn- get-file*
+  [{:keys [::db/conn] :as cfg} id
+   {:keys [migrate?
+           realize?
+           decode?
+           skip-locked?
+           include-deleted?
+           load-data?
+           throw-if-not-exists?
+           lock-for-update?
+           lock-for-share?]
+    :or {lock-for-update? false
+         lock-for-share? false
+         load-data? true
+         migrate? true
+         decode? true
+         include-deleted? false
+         throw-if-not-exists? true
+         realize? false}
+    :as options}]
+
+  (assert (db/connection? conn) "expected cfg with valid connection")
+  (when (and (not load-data?)
+             (or lock-for-share? lock-for-share? skip-locked?))
+    (throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false")))
+
+  (let [sql
+        (if load-data?
+          sql:get-file
+          sql:get-file-without-data)
+
+        sql
+        (cond
+          lock-for-update?
+          (str sql " FOR UPDATE of f")
+
+          lock-for-share?
+          (str sql " FOR SHARE of f")
+
+          :else
+          sql)
+
+        sql
+        (if skip-locked?
+          (str sql " SKIP LOCKED")
+          sql)
+
+        file
+        (db/get-with-sql conn [sql id]
+                         {::db/throw-if-not-exists false
+                          ::db/remove-deleted (not include-deleted?)})
+
+        file
+        (-> file
+            (d/update-when :features db/decode-pgarray #{})
+            (d/update-when :metadata fdata/decode-metadata))]
+
+    (if file
+      (if load-data?
+        (let [file
+              (->> file
+                   (fmigr/resolve-applied-migrations cfg)
+                   (fdata/resolve-file-data cfg))
+
+              will-migrate?
+              (and migrate? (fmg/need-migration? file))]
+
+          (if decode?
+            (cond->> (fdata/decode-file-data cfg file)
+              (and realize? (not will-migrate?))
+              (fdata/realize cfg)
+
+              will-migrate?
+              (migrate-file cfg options))
+
+            file))
+        file)
+
+      (when-not (or skip-locked? (not throw-if-not-exists?))
+        (ex/raise :type :not-found
+                  :code :object-not-found
+                  :hint "database object not found"
+                  :table :file
+                  :file-id id)))))
 
 (defn get-file
   "Get file, resolve all features and apply migrations.
@@ -177,10 +319,7 @@
   operations on file, because it removes the ovehead of lazy fetching
   and decoding."
   [cfg file-id & {:as opts}]
-  (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
+  (db/run! cfg get-file* file-id opts))
-                 (some->> (db/get* conn :file {:id file-id}
-                                   (assoc opts ::db/remove-deleted false))
-                          (decode-file cfg)))))
 
 (defn clean-file-features
   [file]
@@ -192,6 +331,81 @@
                                  (set/difference cfeat/backend-only-features))
                              #{}))))
 
+(defn check-file-exists
+  [cfg id & {:keys [include-deleted?]
+             :or {include-deleted? false}
+             :as options}]
+  (db/get-with-sql cfg [sql:get-minimal-file id]
+                   {:db/remove-deleted (not include-deleted?)}))
+
+(def ^:private sql:file-permissions
+  "select fpr.is_owner,
+          fpr.is_admin,
+          fpr.can_edit
+     from file_profile_rel as fpr
+    inner join file as f on (f.id = fpr.file_id)
+    where fpr.file_id = ?
+      and fpr.profile_id = ?
+   union all
+   select tpr.is_owner,
+          tpr.is_admin,
+          tpr.can_edit
+     from team_profile_rel as tpr
+    inner join project as p on (p.team_id = tpr.team_id)
+    inner join file as f on (p.id = f.project_id)
+    where f.id = ?
+      and tpr.profile_id = ?
+   union all
+   select ppr.is_owner,
+          ppr.is_admin,
+          ppr.can_edit
+     from project_profile_rel as ppr
+    inner join file as f on (f.project_id = ppr.project_id)
+    where f.id = ?
+      and ppr.profile_id = ?")
+
+(defn- get-file-permissions*
+  [conn profile-id file-id]
+  (when (and profile-id file-id)
+    (db/exec! conn [sql:file-permissions
+                    file-id profile-id
+                    file-id profile-id
+                    file-id profile-id])))
+
+(defn get-file-permissions
+  ([conn profile-id file-id]
+   (let [rows     (get-file-permissions* conn profile-id file-id)
+         is-owner (boolean (some :is-owner rows))
+         is-admin (boolean (some :is-admin rows))
+         can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:type :membership
+        :is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true
+        :is-logged (some? profile-id)})))
+
+  ([conn profile-id file-id share-id]
+   (let [perms  (get-file-permissions conn profile-id file-id)
+         ldata  (some-> (db/get* conn :share-link {:id share-id :file-id file-id})
+                        (dissoc :flags)
+                        (update :pages db/decode-pgarray #{}))]
+
+     ;; NOTE: in a future when share-link becomes more powerful and
+     ;; will allow us specify which parts of the app is available, we
+     ;; will probably need to tweak this function in order to expose
+     ;; this flags to the frontend.
+     (cond
+       (some? perms) perms
+       (some? ldata) {:type :share-link
+                      :can-read true
+                      :pages (:pages ldata)
+                      :is-logged (some? profile-id)
+                      :who-comment (:who-comment ldata)
+                      :who-inspect (:who-inspect ldata)}))))
+
+
 (defn get-project
   [cfg project-id]
   (db/get cfg :project {:id project-id}))
@@ -204,12 +418,12 @@
   (let [conn (db/get-connection cfg)
         ids  (db/create-array conn "uuid" ids)]
     (->> (db/exec! conn [sql:get-teams ids])
-         (map decode-row))))
+         (map decode-row-features))))
 
 (defn get-team
   [cfg team-id]
   (-> (db/get cfg :team {:id team-id})
-      (decode-row)))
+      (decode-row-features)))
 
 (defn get-fonts
   [cfg team-id]
@@ -301,7 +515,6 @@
             (do
               (l/trc :hint "lookup index"
                      :file-id (str file-id)
-                     :snap-id (str (:snapshot-id file))
                      :id (str id)
                      :result (str (get mobj :id)))
               (get mobj :id))
@@ -318,7 +531,6 @@
     (doseq [[old-id item] missing-index]
       (l/dbg :hint "create missing references"
              :file-id (str file-id)
-             :snap-id (str (:snapshot-id file))
              :old-id (str old-id)
              :id (str (:id item)))
       (db/insert! conn :file-media-object item
@@ -329,12 +541,16 @@
 (def sql:get-file-media
   "SELECT * FROM file_media_object WHERE id = ANY(?)")
 
+(defn get-file-media*
+  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
+  (let [used (cfh/collect-used-media data)
+        used (db/create-array conn "uuid" used)]
+    (->> (db/exec! conn [sql:get-file-media used])
+         (mapv (fn [row] (assoc row :file-id id))))))
+
 (defn get-file-media
-  [cfg {:keys [data] :as file}]
+  [cfg file]
-  (db/run! cfg (fn [{:keys [::db/conn]}]
+  (db/run! cfg get-file-media* file))
-                 (let [used (cfh/collect-used-media data)
-                       used (db/create-array conn "uuid" used)]
-                   (db/exec! conn [sql:get-file-media used])))))
 
 (def ^:private sql:get-team-files-ids
   "SELECT f.id FROM file AS f
@@ -409,7 +625,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id)]
+              (if-let [library (get-file cfg library-id :include-deleted? true)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data
@@ -421,6 +637,27 @@
     (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
     (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
 
+(defn invalidate-thumbnails
+  [cfg file-id]
+  (let [storage (sto/resolve cfg)
+
+        sql-1
+        (str "update file_tagged_object_thumbnail "
+             "   set deleted_at = now() "
+             " where file_id=? returning media_id")
+
+        sql-2
+        (str "update file_thumbnail "
+             "   set deleted_at = now() "
+             " where file_id=? returning media_id")]
+
+    (run! #(sto/touch-object! storage %)
+          (sequence
+           (keep :media-id)
+           (concat
+            (db/exec! cfg [sql-1 file-id])
+            (db/exec! cfg [sql-2 file-id]))))))
+
 (defn process-file
   [cfg {:keys [id] :as file}]
   (let [libs (delay (get-resolved-file-libraries cfg file))]
@@ -444,78 +681,102 @@
         ;; all of them, not only the applied
         (vary-meta dissoc ::fmg/migrated))))
 
-(defn encode-file
+(defn- encode-file
-  [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
+  [cfg {:keys [id features] :as file}]
-  (let [file (if (contains? features "fdata/objects-map")
+  (let [file (if (and (contains? features "fdata/objects-map")
-               (feat.fdata/enable-objects-map file)
+                      (:data file))
+               (fdata/enable-objects-map file)
                file)
 
-        file (if (contains? features "fdata/pointer-map")
+        file (if (and (contains? features "fdata/pointer-map")
-               (binding [pmap/*tracked* (pmap/create-tracked)]
+                      (:data file))
-                 (let [file (feat.fdata/enable-pointer-map file)]
+
-                   (feat.fdata/persist-pointers! cfg id)
+               (binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
+                 (let [file (fdata/enable-pointer-map file)]
+                   (fdata/persist-pointers! cfg id)
                    file))
                file)]
 
     (-> file
-        (update :features db/encode-pgarray conn "text")
+        (d/update-when :features into-array)
-        (update :data blob/encode))))
+        (d/update-when :data blob/encode))))
 
-(defn get-params-from-file
+(defn- file->params
   [file]
-  (let [params {:has-media-trimmed (:has-media-trimmed file)
+  (-> (select-keys file file-attrs)
-                :ignore-sync-until (:ignore-sync-until file)
+      (assoc :data nil)
-                :project-id (:project-id file)
+      (dissoc :team-id)
-                :features (:features file)
+      (dissoc :migrations)))
-                :name (:name file)
-                :is-shared (:is-shared file)
-                :version (:version file)
-                :data (:data file)
-                :id (:id file)
-                :deleted-at (:deleted-at file)
-                :created-at (:created-at file)
-                :modified-at (:modified-at file)
-                :revn (:revn file)
-                :vern (:vern file)}]
 
-    (-> (d/without-nils params)
+(defn- file->file-data-params
-        (assoc :data-backend nil)
+  [{:keys [id] :as file} & {:as opts}]
-        (assoc :data-ref-id nil))))
+  (let [created-at  (or (:created-at file) (ct/now))
+        modified-at (or (:modified-at file) created-at)]
+    (d/without-nils
+     {:id id
+      :type "main"
+      :file-id id
+      :data (:data file)
+      :metadata (:metadata file)
+      :created-at created-at
+      :modified-at modified-at})))
 
 (defn insert-file!
-  "Insert a new file into the database table"
+  "Insert a new file into the database table. Expectes a not-encoded file.
+  Returns nil."
   [{:keys [::db/conn] :as cfg} file & {:as opts}]
-  (feat.fmigr/upsert-migrations! conn file)
+  (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
-  (let [params (-> (encode-file cfg file)
+
-                   (get-params-from-file))]
+  (when (:migrations file)
-    (db/insert! conn :file params opts)))
+    (fmigr/upsert-migrations! conn file))
+
+  (let [file (encode-file cfg file)]
+    (db/insert! conn :file
+                (file->params file)
+                (assoc opts ::db/return-keys false))
+
+    (->> (file->file-data-params file)
+         (fdata/upsert! cfg))
+
+    nil))
 
 (defn update-file!
-  "Update an existing file on the database."
+  "Update an existing file on the database. Expects not encoded file."
-  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
-  (let [file   (encode-file cfg file)
-        params (-> (get-params-from-file file)
-                   (dissoc :id))]
 
-    ;; If file was already offloaded, we touch the underlying storage
+  (if (::reset-migrations? opts false)
-    ;; object for properly trigger storage-gc-touched task
+    (fmigr/reset-migrations! conn file)
-    (when (feat.fdata/offloaded? file)
+    (fmigr/upsert-migrations! conn file))
-      (some->> (:data-ref-id file) (sto/touch-object! storage)))
 
-    (feat.fmigr/upsert-migrations! conn file)
+  (let [file
-    (db/update! conn :file params {:id id} opts)))
+        (encode-file cfg file)
+
+        file-params
+        (file->params (dissoc file :id))
+
+        file-data-params
+        (file->file-data-params file)]
+
+    (db/update! conn :file file-params
+                {:id id}
+                {::db/return-keys false})
+
+    (fdata/upsert! cfg file-data-params)
+    nil))
 
 (defn save-file!
   "Applies all the final validations and perist the file, binfile
-  specific, should not be used outside of binfile domain"
+  specific, should not be used outside of binfile domain.
+  Returns nil"
   [{:keys [::timestamp] :as cfg} file & {:as opts}]
 
-  (assert (dt/instant? timestamp) "expected valid timestamp")
+  (assert (ct/inst? timestamp) "expected valid timestamp")
 
   (let [file (-> file
                  (assoc :created-at timestamp)
                  (assoc :modified-at timestamp)
-                 (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
+                 (cond-> (not (::overwrite cfg))
+                   (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))))
                  (update :features
                          (fn [features]
                            (-> (::features cfg #{})
@@ -532,8 +793,9 @@
         (when (ex/exception? result)
           (l/error :hint "file schema validation error" :cause result))))
 
-    (insert-file! cfg file opts)))
+    (if (::overwrite cfg)
-
+      (update-file! cfg file (assoc opts ::reset-migrations? true))
+      (insert-file! cfg file opts))))
 
 (def ^:private sql:get-file-libraries
   "WITH RECURSIVE libs AS (
@@ -558,10 +820,11 @@
           l.revn,
           l.vern,
           l.synced_at,
-          l.is_shared
+          l.is_shared,
+          l.version
      FROM libs AS l
     INNER JOIN project AS p ON (p.id = l.project_id)
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+    WHERE l.deleted_at IS NULL;")
 
 (defn get-file-libraries
   [conn file-id]
@@ -570,12 +833,22 @@
          ;; FIXME: :is-indirect set to false to all rows looks
          ;; completly useless
          (map #(assoc % :is-indirect false))
-         (map decode-row))
+         (map decode-row-features))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
 (defn get-resolved-file-libraries
-  "A helper for preload file libraries"
+  "Get all file libraries including itself. Returns an instance of
-  [{:keys [::db/conn] :as cfg} file]
+  LoadableWeakValueMap that allows do not have strong references to
-  (->> (get-file-libraries conn (:id file))
+  the loaded libraries and reduce possible memory pressure on having
-       (into [file] (map #(get-file cfg (:id %))))
+  all this libraries loaded at same time on processing file validation
-       (d/index-by :id)))
+  or file migration.
+
+  This still requires at least one library at time to be loaded while
+  access to it is performed, but it improves considerable not having
+  the need of loading all the libraries at the same time."
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+  (let [library-ids (->> (get-file-libraries conn (:id file))
+                         (map :id)
+                         (cons (:id file)))
+        load-fn     #(get-file cfg % :migrate? false)]
+    (weak/loadable-weak-value-map library-ids load-fn {id file})))

backend/src/app/binfile/migrations.clj

@@ -36,11 +36,6 @@
       "fdata/shape-data-type"
       nil
 
-      ;; There is no migration needed, but we don't want to allow
-      ;; copy paste nor import of variant files into no-variant teams
-      "variants/v1"
-      nil
-
       (ex/raise :type :internal
                 :code :no-migration-defined
                 :hint (str/ffmt "no migation for feature '%' on file importation" feature)

backend/src/app/binfile/v1.clj

@@ -17,6 +17,7 @@
    [app.common.fressian :as fres]
    [app.common.logging :as l]
    [app.common.spec :as us]
+   [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -30,7 +31,6 @@
    [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.events :as events]
-   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.java.io :as jio]
    [clojure.set :as set]
@@ -346,7 +346,7 @@
           thumbnails (->> (bfc/get-file-object-thumbnails cfg file-id)
                           (mapv #(dissoc % :file-id)))
 
-          file       (cond-> (bfc/get-file cfg file-id)
+          file       (cond-> (bfc/get-file cfg file-id :realize? true)
                        detach?
                        (-> (ctf/detach-external-references file-id)
                            (dissoc :libraries))
@@ -434,7 +434,7 @@
 (defn read-import!
   "Do the importation of the specified resource in penpot custom binary
   format."
-  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
+  [{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (ct/now)} :as options}]
 
   (dm/assert!
    "expected input stream"
@@ -442,7 +442,7 @@
 
   (dm/assert!
    "expected valid instant"
-   (dt/instant? timestamp))
+   (ct/inst? timestamp))
 
   (let [version (read-header! input)]
     (read-import (assoc options ::version version ::bfc/timestamp timestamp))))
@@ -682,7 +682,7 @@
    (io/coercible? output))
 
   (let [id (uuid/next)
-        tp (dt/tpoint)
+        tp (ct/tpoint)
         ab (volatile! false)
         cs (volatile! nil)]
     (try
@@ -720,7 +720,7 @@
    (satisfies? jio/IOFactory input))
 
   (let [id (uuid/next)
-        tp (dt/tpoint)
+        tp (ct/tpoint)
         cs (volatile! nil)]
 
     (l/info :hint "import: started" :id (str id))
@@ -742,6 +742,6 @@
       (finally
         (l/info :hint "import: terminated"
                 :id (str id)
-                :elapsed (dt/format-duration (tp))
+                :elapsed (ct/format-duration (tp))
                 :error? (some? @cs))))))
 

backend/src/app/binfile/v2.clj

@@ -13,6 +13,7 @@
    [app.common.data :as d]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -23,7 +24,6 @@
    [app.storage :as sto]
    [app.storage.tmp :as tmp]
    [app.util.events :as events]
-   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.set :as set]
    [cuerdas.core :as str]
@@ -153,7 +153,7 @@
 
 (defn- write-file!
   [cfg file-id]
-  (let [file   (bfc/get-file cfg file-id)
+  (let [file   (bfc/get-file cfg file-id :realize? true)
         thumbs (bfc/get-file-object-thumbnails cfg file-id)
         media  (bfc/get-file-media cfg file)
         rels   (bfc/get-files-rels cfg #{file-id})]
@@ -344,7 +344,7 @@
 (defn export-team!
   [cfg team-id]
   (let [id  (uuid/next)
-        tp  (dt/tpoint)
+        tp  (ct/tpoint)
         cfg (create-database cfg)]
 
     (l/inf :hint "start"
@@ -378,15 +378,15 @@
           (l/inf :hint "end"
                  :operation "export"
                  :id (str id)
-                 :elapsed (dt/format-duration elapsed)))))))
+                 :elapsed (ct/format-duration elapsed)))))))
 
 (defn import-team!
   [cfg path]
   (let [id  (uuid/next)
-        tp  (dt/tpoint)
+        tp  (ct/tpoint)
 
         cfg (-> (create-database cfg path)
-                (assoc ::bfc/timestamp (dt/now)))]
+                (assoc ::bfc/timestamp (ct/now)))]
 
     (l/inf :hint "start"
            :operation "import"
@@ -434,4 +434,4 @@
           (l/inf :hint "end"
                  :operation "import"
                  :id (str id)
-                 :elapsed (dt/format-duration elapsed)))))))
+                 :elapsed (ct/format-duration elapsed)))))))

backend/src/app/binfile/v3.clj

@@ -12,7 +12,6 @@
    [app.binfile.common :as bfc]
    [app.binfile.migrations :as bfm]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.files.migrations :as-alias fmg]
@@ -21,13 +20,14 @@
    [app.common.media :as cmedia]
    [app.common.schema :as sm]
    [app.common.thumbnails :as cth]
+   [app.common.time :as ct]
    [app.common.types.color :as ctcl]
    [app.common.types.component :as ctc]
    [app.common.types.file :as ctf]
    [app.common.types.page :as ctp]
    [app.common.types.plugins :as ctpg]
    [app.common.types.shape :as cts]
-   [app.common.types.tokens-lib :as cto]
+   [app.common.types.tokens-lib :as ctob]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -36,14 +36,15 @@
    [app.storage :as sto]
    [app.storage.impl :as sto.impl]
    [app.util.events :as events]
-   [app.util.time :as dt]
    [clojure.java.io :as jio]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
    [datoteka.io :as io])
   (:import
+   java.io.File
    java.io.InputStream
    java.io.OutputStreamWriter
+   java.lang.AutoCloseable
    java.util.zip.ZipEntry
    java.util.zip.ZipFile
    java.util.zip.ZipOutputStream))
@@ -54,7 +55,7 @@
   [:map {:title "Manifest"}
    [:version ::sm/int]
    [:type :string]
-
+   [:referer {:optional true} :string]
    [:generated-by {:optional true} :string]
 
    [:files
@@ -93,7 +94,7 @@
 
 (defn- default-now
   [o]
-  (or o (dt/now)))
+  (or o (ct/now)))
 
 ;; --- ENCODERS
 
@@ -104,25 +105,25 @@
   (sm/encoder ctp/schema:page sm/json-transformer))
 
 (def encode-shape
-  (sm/encoder ::cts/shape sm/json-transformer))
+  (sm/encoder cts/schema:shape sm/json-transformer))
 
 (def encode-media
-  (sm/encoder ::ctf/media sm/json-transformer))
+  (sm/encoder ctf/schema:media sm/json-transformer))
 
 (def encode-component
-  (sm/encoder ::ctc/component sm/json-transformer))
+  (sm/encoder ctc/schema:component sm/json-transformer))
 
 (def encode-color
   (sm/encoder ctcl/schema:library-color sm/json-transformer))
 
 (def encode-typography
-  (sm/encoder ::cty/typography sm/json-transformer))
+  (sm/encoder cty/schema:typography sm/json-transformer))
 
 (def encode-tokens-lib
-  (sm/encoder ::cto/tokens-lib sm/json-transformer))
+  (sm/encoder ctob/schema:tokens-lib sm/json-transformer))
 
 (def encode-plugin-data
-  (sm/encoder ::ctpg/plugin-data sm/json-transformer))
+  (sm/encoder ctpg/schema:plugin-data sm/json-transformer))
 
 (def encode-storage-object
   (sm/encoder schema:storage-object sm/json-transformer))
@@ -139,7 +140,7 @@
   (sm/decoder ctf/schema:media sm/json-transformer))
 
 (def decode-component
-  (sm/decoder ::ctc/component sm/json-transformer))
+  (sm/decoder ctc/schema:component sm/json-transformer))
 
 (def decode-color
   (sm/decoder ctcl/schema:library-color sm/json-transformer))
@@ -148,19 +149,19 @@
   (sm/decoder schema:file sm/json-transformer))
 
 (def decode-page
-  (sm/decoder ::ctp/page sm/json-transformer))
+  (sm/decoder ctp/schema:page sm/json-transformer))
 
 (def decode-shape
-  (sm/decoder ::cts/shape sm/json-transformer))
+  (sm/decoder cts/schema:shape sm/json-transformer))
 
 (def decode-typography
-  (sm/decoder ::cty/typography sm/json-transformer))
+  (sm/decoder cty/schema:typography sm/json-transformer))
 
 (def decode-tokens-lib
-  (sm/decoder cto/schema:tokens-lib sm/json-transformer))
+  (sm/decoder ctob/schema:tokens-lib sm/json-transformer))
 
 (def decode-plugin-data
-  (sm/decoder ::ctpg/plugin-data sm/json-transformer))
+  (sm/decoder ctpg/schema:plugin-data sm/json-transformer))
 
 (def decode-storage-object
   (sm/decoder schema:storage-object sm/json-transformer))
@@ -174,31 +175,31 @@
   (sm/check-fn schema:manifest))
 
 (def validate-file
-  (sm/check-fn ::ctf/file))
+  (sm/check-fn ctf/schema:file))
 
 (def validate-page
-  (sm/check-fn ::ctp/page))
+  (sm/check-fn ctp/schema:page))
 
 (def validate-shape
-  (sm/check-fn ::cts/shape))
+  (sm/check-fn cts/schema:shape))
 
 (def validate-media
-  (sm/check-fn ::ctf/media))
+  (sm/check-fn ctf/schema:media))
 
 (def validate-color
   (sm/check-fn ctcl/schema:library-color))
 
 (def validate-component
-  (sm/check-fn ::ctc/component))
+  (sm/check-fn ctc/schema:component))
 
 (def validate-typography
-  (sm/check-fn ::cty/typography))
+  (sm/check-fn cty/schema:typography))
 
 (def validate-tokens-lib
-  (sm/check-fn ::cto/tokens-lib))
+  (sm/check-fn ctob/schema:tokens-lib))
 
 (def validate-plugin-data
-  (sm/check-fn ::ctpg/plugin-data))
+  (sm/check-fn ctpg/schema:plugin-data))
 
 (def validate-storage-object
   (sm/check-fn schema:storage-object))
@@ -223,9 +224,12 @@
     (throw (IllegalArgumentException.
             "the `include-libraries` and `embed-assets` are mutally excluding options")))
 
-  (let [detach?  (and (not embed-assets) (not include-libraries))]
+  (let [detach? (and (not embed-assets) (not include-libraries))]
     (db/tx-run! cfg (fn [cfg]
-                      (cond-> (bfc/get-file cfg file-id {::sql/for-update true})
+                      (cond-> (bfc/get-file cfg file-id
+                                            {:realize? true
+                                             :include-deleted? true
+                                             :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
                             (dissoc :libraries))
@@ -251,10 +255,12 @@
 
         (write-entry! output path params)
 
+        (events/tap :progress {:section :storage-object :id id})
+
         (with-open [input (sto/get-object-data storage sobject)]
-          (.putNextEntry output (ZipEntry. (str "objects/" id ext)))
+          (.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
           (io/copy input output :size (:size sobject))
-          (.closeEntry output))))))
+          (.closeEntry ^ZipOutputStream output))))))
 
 (defn- export-file
   [{:keys [::file-id ::output] :as cfg}]
@@ -275,6 +281,8 @@
 
         thumbnails   (bfc/get-file-object-thumbnails cfg file-id)]
 
+    (events/tap :progress {:section :file :id file-id})
+
     (vswap! bfc/*state* update :files assoc file-id
             {:id file-id
              :name (:name file)
@@ -282,13 +290,13 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data))
+                 (assoc :options (:options data)))
 
-                 :always
+          file (-> file
-                 (dissoc :data)
+                   (dissoc :data)
+                   (dissoc :deleted-at)
+                   (encode-file))
 
-                 :always
-                 (encode-file))
           path (str "files/" file-id ".json")]
       (write-entry! output path file))
 
@@ -346,7 +354,8 @@
             typography (encode-typography object)]
         (write-entry! output path typography)))
 
-    (when tokens-lib
+    (when (and tokens-lib
+               (not (ctob/empty-lib? tokens-lib)))
       (let [path           (str "files/" file-id "/tokens.json")
             encoded-tokens (encode-tokens-lib tokens-lib)]
         (write-entry! output path encoded-tokens)))))
@@ -373,6 +382,7 @@
           params {:type "penpot/export-files"
                   :version 1
                   :generated-by (str "penpot/" (:full cf/version))
+                  :refer "penpot"
                   :files (vec (vals files))
                   :relations rels}]
       (write-entry! output "manifest.json" params))))
@@ -445,7 +455,7 @@
 (defn- read-manifest
   [^ZipFile input]
   (let [entry (get-zip-entry input "manifest.json")]
-    (with-open [reader (zip-entry-reader input entry)]
+    (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
       (let [manifest (json/read reader :key-fn json/read-kebab-key)]
         (decode-manifest manifest)))))
 
@@ -535,24 +545,27 @@
 
 (defn- read-entry
   [^ZipFile input entry]
-  (with-open [reader (zip-entry-reader input entry)]
+  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
     (json/read reader :key-fn json/read-kebab-key)))
 
 (defn- read-plain-entry
   [^ZipFile input entry]
-  (with-open [reader (zip-entry-reader input entry)]
+  (with-open [^AutoCloseable reader (zip-entry-reader input entry)]
     (json/read reader)))
 
 (defn- read-file
-  [{:keys [::bfc/input ::file-id]}]
+  [{:keys [::bfc/input ::bfc/timestamp]} file-id]
   (let [path  (str "files/" file-id ".json")
         entry (get-zip-entry input path)]
     (-> (read-entry input entry)
         (decode-file)
+        (update :revn d/nilv 1)
+        (update :created-at d/nilv timestamp)
+        (update :modified-at d/nilv timestamp)
         (validate-file))))
 
 (defn- read-file-plugin-data
-  [{:keys [::bfc/input ::file-id]}]
+  [{:keys [::bfc/input]} file-id]
   (let [path  (str "files/" file-id "/plugin-data.json")
         entry (get-zip-entry* input path)]
     (some->> entry
@@ -561,7 +574,7 @@
              (validate-plugin-data))))
 
 (defn- read-file-media
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-media-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -581,7 +594,7 @@
        (not-empty)))
 
 (defn- read-file-colors
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-color-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -594,7 +607,7 @@
        (not-empty)))
 
 (defn- read-file-components
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (let [clean-component-post-decode
         (fn [component]
           (d/update-when component :objects
@@ -625,7 +638,7 @@
          (not-empty))))
 
 (defn- read-file-typographies
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (->> (keep (match-typography-entry-fn file-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -638,14 +651,14 @@
        (not-empty)))
 
 (defn- read-file-tokens-lib
-  [{:keys [::bfc/input ::file-id ::entries]}]
+  [{:keys [::bfc/input ::entries]} file-id]
   (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
     (->> (read-plain-entry input entry)
          (decode-tokens-lib)
          (validate-tokens-lib))))
 
 (defn- read-file-shapes
-  [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
   (->> (keep (match-shape-entry-fn file-id page-id) entries)
        (reduce (fn [result {:keys [id entry]}]
                  (let [object (->> (read-entry input entry)
@@ -659,15 +672,14 @@
        (not-empty)))
 
 (defn- read-file-pages
-  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id]
   (->> (keep (match-page-entry-fn file-id) entries)
        (keep (fn [{:keys [id entry]}]
                (let [page (->> (read-entry input entry)
                                (decode-page))
                      page (dissoc page :options)]
                  (when (= id (:id page))
-                   (let [objects (-> (assoc cfg ::page-id id)
+                   (let [objects (read-file-shapes cfg file-id id)]
-                                     (read-file-shapes))]
                      (assoc page :objects objects))))))
        (sort-by :index)
        (reduce (fn [result {:keys [id] :as page}]
@@ -675,7 +687,7 @@
                (d/ordered-map))))
 
 (defn- read-file-thumbnails
-  [{:keys [::bfc/input ::file-id ::entries] :as cfg}]
+  [{:keys [::bfc/input ::entries] :as cfg} file-id]
   (->> (keep (match-thumbnail-entry-fn file-id) entries)
        (reduce (fn [result {:keys [page-id frame-id tag entry]}]
                  (let [object (->> (read-entry input entry)
@@ -690,13 +702,13 @@
        (not-empty)))
 
 (defn- read-file-data
-  [cfg]
+  [cfg file-id]
-  (let [colors       (read-file-colors cfg)
+  (let [colors       (read-file-colors cfg file-id)
-        typographies (read-file-typographies cfg)
+        typographies (read-file-typographies cfg file-id)
-        tokens-lib   (read-file-tokens-lib cfg)
+        tokens-lib   (read-file-tokens-lib cfg file-id)
-        components   (read-file-components cfg)
+        components   (read-file-components cfg file-id)
-        plugin-data  (read-file-plugin-data cfg)
+        plugin-data  (read-file-plugin-data cfg file-id)
-        pages        (read-file-pages cfg)]
+        pages        (read-file-pages cfg file-id)]
     {:pages (-> pages keys vec)
      :pages-index (into {} pages)
      :colors colors
@@ -706,11 +718,11 @@
      :plugin-data plugin-data}))
 
 (defn- import-file
-  [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
+  [{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
   (let [file-id'   (bfc/lookup-index file-id)
-        file       (read-file cfg)
+        file       (read-file cfg file-id)
-        media      (read-file-media cfg)
+        media      (read-file-media cfg file-id)
-        thumbnails (read-file-thumbnails cfg)]
+        thumbnails (read-file-thumbnails cfg file-id)]
 
     (l/dbg :hint "processing file"
            :id (str file-id')
@@ -719,28 +731,50 @@
            :version (:version file)
            ::l/sync? true)
 
-    (events/tap :progress {:section :file :name file-name})
+    (vswap! bfc/*state* update :index bfc/update-index media :id)
 
-    (when media
+    (events/tap :progress {:section :media :file-id file-id})
-      ;; Update index with media
-      (l/dbg :hint "update media index"
-             :file-id (str file-id')
-             :total (count media)
-             ::l/sync? true)
 
-      (vswap! bfc/*state* update :index bfc/update-index (map :id media))
+    (doseq [item media]
-      (vswap! bfc/*state* update :media into media))
+      (let [params (-> item
+                       (update :id bfc/lookup-index)
+                       (assoc :file-id file-id')
+                       (d/update-when :media-id bfc/lookup-index)
+                       (d/update-when :thumbnail-id bfc/lookup-index))]
 
-    (when thumbnails
+        (l/dbg :hint "inserting media object"
-      (l/dbg :hint "update thumbnails index"
+               :file-id (str file-id')
-             :file-id (str file-id')
+               :id (str (:id params))
-             :total (count thumbnails)
+               :media-id (str (:media-id params))
-             ::l/sync? true)
+               :thumbnail-id (str (:thumbnail-id params))
+               :old-id (str (:id item))
+               ::l/sync? true)
 
-      (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
+        (db/insert! conn :file-media-object params
-      (vswap! bfc/*state* update :thumbnails into thumbnails))
+                    ::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
 
-    (let [data (-> (read-file-data cfg)
+    (events/tap :progress {:section :thumbnails :file-id file-id})
+
+    (doseq [item thumbnails]
+      (let [media-id  (bfc/lookup-index (:media-id item))
+            object-id (-> (assoc item :file-id file-id')
+                          (cth/fmt-object-id))
+            params    {:file-id file-id'
+                       :object-id object-id
+                       :tag (:tag item)
+                       :media-id media-id}]
+
+        (l/dbg :hint "inserting object thumbnail"
+               :file-id (str file-id')
+               :media-id (str media-id)
+               ::l/sync? true)
+
+        (db/insert! conn :file-tagged-object-thumbnail params
+                    ::db/on-conflict-do-nothing? true)))
+
+    (events/tap :progress {:section :file :file-id file-id})
+
+    (let [data (-> (read-file-data cfg file-id)
                    (d/without-nils)
                    (assoc :id file-id')
                    (cond-> (:options file)
@@ -757,7 +791,7 @@
           file  (ctf/check-file file)]
 
       (bfm/register-pending-migrations! cfg file)
-      (bfc/save-file! cfg file ::db/return-keys false)
+      (bfc/save-file! cfg file)
 
       file-id')))
 
@@ -787,108 +821,111 @@
         entries (keep (match-storage-entry-fn) entries)]
 
     (doseq [{:keys [id entry]} entries]
-      (let [object (->> (read-entry input entry)
+      (let [object  (-> (read-entry input entry)
                         (decode-storage-object)
-                        (validate-storage-object))]
+                        (update :bucket d/nilv sto/default-bucket)
+                        (validate-storage-object))
 
-        (when (not= id (:id object))
+            ext     (cmedia/mtype->extension (:content-type object))
+            path    (str "objects/" id ext)
+            content (->> path
+                         (get-zip-entry input)
+                         (zip-entry-storage-content input))]
+
+        (when (not= (:size object) (sto/get-size content))
           (ex/raise :type :validation
                     :code :inconsistent-penpot-file
-                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"
+                    :hint "found corrupted storage object: size does not match"
-                    :expected-id (str id)
+                    :path path
-                    :found-id (str (:id object))))
+                    :expected-size (:size object)
+                    :found-size (sto/get-size content)))
 
-        (let [ext     (cmedia/mtype->extension (:content-type object))
+        (when-let [hash (get object :hash)]
-              path    (str "objects/" id ext)
+          (when (not= hash (sto/get-hash content))
-              content (->> path
-                           (get-zip-entry input)
-                           (zip-entry-storage-content input))]
-
-          (when (not= (:size object) (sto/get-size content))
             (ex/raise :type :validation
                       :code :inconsistent-penpot-file
-                      :hint "found corrupted storage object: size does not match"
+                      :hint "found corrupted storage object: hash does not match"
                       :path path
-                      :expected-size (:size object)
+                      :expected-hash (:hash object)
-                      :found-size (sto/get-size content)))
+                      :found-hash (sto/get-hash content))))
 
-          (when-let [hash (get object :hash)]
+        (let [params  (-> object
-            (when (not= hash (sto/get-hash content))
+                          (dissoc :id :size)
-              (ex/raise :type :validation
+                          (assoc ::sto/content content)
-                        :code :inconsistent-penpot-file
+                          (assoc ::sto/deduplicate? true)
-                        :hint "found corrupted storage object: hash does not match"
+                          (assoc ::sto/touched-at timestamp))
-                        :path path
+              sobject (sto/put-object! storage params)]
-                        :expected-hash (:hash object)
-                        :found-hash (sto/get-hash content))))
 
-          (let [params  (-> object
+          (l/dbg :hint "persisted storage object"
-                            (dissoc :id :size)
+                 :id (str (:id sobject))
-                            (assoc ::sto/content content)
+                 :prev-id (str id)
-                            (assoc ::sto/deduplicate? true)
+                 :bucket (:bucket params)
-                            (assoc ::sto/touched-at timestamp))
+                 ::l/sync? true)
-                sobject (sto/put-object! storage params)]
 
-            (l/dbg :hint "persisted storage object"
+          (vswap! bfc/*state* update :index assoc id (:id sobject)))))))
-                   :id (str (:id sobject))
-                   :prev-id (str id)
-                   :bucket (:bucket params)
-                   ::l/sync? true)
 
-            (vswap! bfc/*state* update :index assoc id (:id sobject))))))))
+(defn- import-files*
+  [{:keys [::manifest] :as cfg}]
+  (bfc/disable-database-timeouts! cfg)
 
-(defn- import-file-media
+  (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
-  [{:keys [::db/conn] :as cfg}]
-  (events/tap :progress {:section :media})
 
-  (doseq [item (:media @bfc/*state*)]
+  (import-storage-objects cfg)
-    (let [params (-> item
-                     (update :id bfc/lookup-index)
-                     (update :file-id bfc/lookup-index)
-                     (d/update-when :media-id bfc/lookup-index)
-                     (d/update-when :thumbnail-id bfc/lookup-index))]
 
-      (l/dbg :hint "inserting file media object"
+  (let [files  (get manifest :files)
-             :old-id (str (:id item))
+        result (reduce (fn [result file]
-             :id (str (:id params))
+                         (let [name' (get file :name)
-             :file-id (str (:file-id params))
+                               file (assoc file :name name')]
-             ::l/sync? true)
+                           (conj result (import-file cfg file))))
+                       []
+                       files)]
 
-      (db/insert! conn :file-media-object params))))
+    (import-file-relations cfg)
+    (bfm/apply-pending-migrations! cfg)
 
-(defn- import-file-thumbnails
+    result))
-  [{:keys [::db/conn] :as cfg}]
-  (events/tap :progress {:section :thumbnails})
-  (doseq [item (:thumbnails @bfc/*state*)]
-    (let [file-id   (bfc/lookup-index (:file-id item))
-          media-id  (bfc/lookup-index (:media-id item))
-          object-id (-> (assoc item :file-id file-id)
-                        (cth/fmt-object-id))
-          params    {:file-id file-id
-                     :object-id object-id
-                     :tag (:tag item)
-                     :media-id media-id}]
 
-      (l/dbg :hint "inserting file object thumbnail"
+(defn- import-file-and-overwrite*
-             :file-id (str file-id)
+  [{:keys [::manifest ::bfc/file-id] :as cfg}]
-             :media-id (str media-id)
-             ::l/sync? true)
 
-      (db/insert! conn :file-tagged-object-thumbnail params))))
+  (when (not= 1 (count (:files manifest)))
+    (ex/raise :type :validation
+              :code :invalid-condition
+              :hint "unable to perform in-place update with binfile containing more than 1 file"
+              :manifest manifest))
+
+  (bfc/disable-database-timeouts! cfg)
+
+  (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
+        file     (first (get manifest :files))
+        cfg      (assoc cfg ::bfc/overwrite true)]
+
+    (vswap! bfc/*state* update :index assoc (:id file) file-id)
+
+    (binding [bfc/*options* cfg
+              bfc/*reference-file* ref-file]
+
+      (import-storage-objects cfg)
+      (import-file cfg file)
+
+      (bfc/invalidate-thumbnails cfg file-id)
+      (bfm/apply-pending-migrations! cfg)
+
+      [file-id])))
 
 (defn- import-files
-  [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
+  [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
 
-  (dm/assert!
+  (assert (instance? ZipFile input) "expected zip file")
-   "expected zip file"
+  (assert (ct/inst? timestamp) "expected valid instant")
-   (instance? ZipFile input))
-
-  (dm/assert!
-   "expected valid instant"
-   (dt/instant? timestamp))
 
   (let [manifest (-> (read-manifest input)
                      (validate-manifest))
-        entries  (read-zip-entries input)]
+        entries  (read-zip-entries input)
+        cfg      (-> cfg
+                     (assoc ::entries entries)
+                     (assoc ::manifest manifest)
+                     (assoc ::bfc/timestamp timestamp))]
 
     (when-not (= "penpot/export-files" (:type manifest))
       (ex/raise :type :validation
@@ -911,35 +948,10 @@
 
     (events/tap :progress {:section :manifest})
 
-    (let [index (bfc/update-index (map :id (:files manifest)))
+    (binding [bfc/*state* (volatile! {:media [] :index {}})]
-          state {:media [] :index index}
+      (if (::bfc/file-id cfg)
-          cfg   (-> cfg
+        (db/tx-run! cfg import-file-and-overwrite*)
-                    (assoc ::entries entries)
+        (db/tx-run! cfg import-files*)))))
-                    (assoc ::manifest manifest)
-                    (assoc ::bfc/timestamp timestamp))]
-
-      (binding [bfc/*state* (volatile! state)]
-        (db/tx-run! cfg (fn [cfg]
-                          (bfc/disable-database-timeouts! cfg)
-                          (let [ids (->> (:files manifest)
-                                         (reduce (fn [result {:keys [id] :as file}]
-                                                   (let [name' (get file :name)
-                                                         name' (if (map? name)
-                                                                 (get name id)
-                                                                 name')]
-                                                     (conj result (-> cfg
-                                                                      (assoc ::file-id id)
-                                                                      (assoc ::file-name name')
-                                                                      (import-file)))))
-                                                 []))]
-                            (import-file-relations cfg)
-                            (import-storage-objects cfg)
-                            (import-file-media cfg)
-                            (import-file-thumbnails cfg)
-
-                            (bfm/apply-pending-migrations! cfg)
-
-                            ids)))))))
 
 ;; --- PUBLIC API
 
@@ -956,24 +968,23 @@
 
   [{:keys [::bfc/ids] :as cfg} output]
 
-  (dm/assert!
+  (assert
-   "expected a set of uuid's for `::bfc/ids` parameter"
+   (and (set? ids) (every? uuid? ids))
-   (and (set? ids)
+   "expected a set of uuid's for `::bfc/ids` parameter")
-        (every? uuid? ids)))
 
-  (dm/assert!
+  (assert
-   "expected instance of jio/IOFactory for `input`"
+   (satisfies? jio/IOFactory output)
-   (satisfies? jio/IOFactory output))
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
-        tp (dt/tpoint)
+        tp (ct/tpoint)
         ab (volatile! false)
         cs (volatile! nil)]
     (try
       (l/info :hint "start exportation" :export-id (str id))
       (binding [bfc/*state* (volatile! (bfc/initial-state))]
-        (with-open [output (io/output-stream output)]
+        (with-open [^AutoCloseable output (io/output-stream output)]
-          (with-open [output (ZipOutputStream. output)]
+          (with-open [^AutoCloseable output (ZipOutputStream. output)]
             (let [cfg (assoc cfg ::output output)]
               (export-files cfg)
               (export-storage-objects cfg)))))
@@ -1002,22 +1013,22 @@
 (defn import-files!
   [{:keys [::bfc/input] :as cfg}]
 
-  (dm/assert!
+  (assert
-   "expected valid profile-id and project-id on `cfg`"
    (and (uuid? (::bfc/profile-id cfg))
-        (uuid? (::bfc/project-id cfg))))
+        (uuid? (::bfc/project-id cfg)))
+   "expected valid profile-id and project-id on `cfg`")
 
-  (dm/assert!
+  (assert
-   "expected instance of jio/IOFactory for `input`"
+   (io/coercible? input)
-   (io/coercible? input))
+   "expected instance of jio/IOFactory for `input`")
 
   (let [id (uuid/next)
-        tp (dt/tpoint)
+        tp (ct/tpoint)
         cs (volatile! nil)]
 
     (l/info :hint "import: started" :id (str id))
     (try
-      (with-open [input (ZipFile. (fs/file input))]
+      (with-open [input (ZipFile. ^File (fs/file input))]
         (import-files (assoc cfg ::bfc/input input)))
 
       (catch Throwable cause
@@ -1027,5 +1038,11 @@
       (finally
         (l/info :hint "import: terminated"
                 :id (str id)
-                :elapsed (dt/format-duration (tp))
+                :elapsed (ct/format-duration (tp))
                 :error? (some? @cs))))))
+
+(defn get-manifest
+  [path]
+  (with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
+    (-> (read-manifest input)
+        (validate-manifest))))

backend/src/app/config.clj

@@ -5,17 +5,16 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.config
-  "A configuration management."
   (:refer-clojure :exclude [get])
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.flags :as flags]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.common.version :as v]
    [app.util.overrides]
-   [app.util.time :as dt]
    [clojure.core :as c]
    [clojure.java.io :as io]
    [cuerdas.core :as str]
@@ -47,22 +46,27 @@
    :auto-file-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
+
    :host "localhost"
    :tenant "default"
 
    :redis-uri "redis://redis/0"
 
+   :file-data-backend "legacy-db"
+
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
+   :auth-token-cookie-name "auth-token"
+
    :assets-path "/internal/assets/"
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
-   :profile-complaint-max-age (dt/duration {:days 7})
+   :profile-complaint-max-age (ct/duration {:days 7})
    :profile-complaint-threshold 2
 
-   :profile-bounce-max-age (dt/duration {:days 7})
+   :profile-bounce-max-age (ct/duration {:days 7})
    :profile-bounce-threshold 10
 
    :telemetry-uri "https://telemetry.penpot.app/"
@@ -88,7 +92,7 @@
     [:secret-key {:optional true} :string]
 
     [:tenant {:optional false} :string]
-    [:public-uri {:optional false} :string]
+    [:public-uri {:optional false} ::sm/uri]
     [:host {:optional false} :string]
 
     [:http-server-port {:optional true} ::sm/int]
@@ -96,16 +100,19 @@
     [:http-server-max-body-size {:optional true} ::sm/int]
     [:http-server-max-multipart-body-size {:optional true} ::sm/int]
     [:http-server-io-threads {:optional true} ::sm/int]
-    [:http-server-worker-threads {:optional true} ::sm/int]
+    [:http-server-max-worker-threads {:optional true} ::sm/int]
+
+    [:management-api-key {:optional true} :string]
 
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
 
     [:auto-file-snapshot-every {:optional true} ::sm/int]
-    [:auto-file-snapshot-timeout {:optional true} ::dt/duration]
+    [:auto-file-snapshot-timeout {:optional true} ::ct/duration]
 
     [:media-max-file-size {:optional true} ::sm/int]
-    [:deletion-delay {:optional true} ::dt/duration] ;; REVIEW
+    [:deletion-delay {:optional true} ::ct/duration]
+    [:file-clean-delay {:optional true} ::ct/duration]
     [:telemetry-enabled {:optional true} ::sm/boolean]
     [:default-blob-version {:optional true} ::sm/int]
     [:allow-demo-users {:optional true} ::sm/boolean]
@@ -146,12 +153,11 @@
     [:quotes-team-access-requests-per-team {:optional true} ::sm/int]
     [:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
 
-    [:auth-data-cookie-domain {:optional true} :string]
     [:auth-token-cookie-name {:optional true} :string]
-    [:auth-token-cookie-max-age {:optional true} ::dt/duration]
+    [:auth-token-cookie-max-age {:optional true} ::ct/duration]
 
     [:registration-domain-whitelist {:optional true} [::sm/set :string]]
-    [:email-verify-threshold {:optional true} ::dt/duration]
+    [:email-verify-threshold {:optional true} ::ct/duration]
 
     [:github-client-id {:optional true} :string]
     [:github-client-secret {:optional true} :string]
@@ -161,7 +167,7 @@
     [:google-client-id {:optional true} :string]
     [:google-client-secret {:optional true} :string]
     [:oidc-client-id {:optional true} :string]
-    [:oidc-user-info-source {:optional true} :keyword]
+    [:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
     [:oidc-client-secret {:optional true} :string]
     [:oidc-base-uri {:optional true} :string]
     [:oidc-token-uri {:optional true} :string]
@@ -186,9 +192,9 @@
     [:ldap-starttls {:optional true} ::sm/boolean]
     [:ldap-user-query {:optional true} :string]
 
-    [:profile-bounce-max-age {:optional true} ::dt/duration]
+    [:profile-bounce-max-age {:optional true} ::ct/duration]
     [:profile-bounce-threshold {:optional true} ::sm/int]
-    [:profile-complaint-max-age {:optional true} ::dt/duration]
+    [:profile-complaint-max-age {:optional true} ::ct/duration]
     [:profile-complaint-threshold {:optional true} ::sm/int]
 
     [:redis-uri {:optional true} ::sm/uri]
@@ -210,24 +216,27 @@
     [:prepl-host {:optional true} :string]
     [:prepl-port {:optional true} ::sm/int]
 
+    [:file-data-backend {:optional true} [:enum "db" "legacy-db" "storage"]]
+
     [:media-directory {:optional true} :string] ;; REVIEW
     [:media-uri {:optional true} :string]
     [:assets-path {:optional true} :string]
 
-    ;; Legacy, will be removed in 2.5
+    [:netty-io-threads {:optional true} ::sm/int]
+    [:executor-threads {:optional true} ::sm/int]
+
+    ;; DEPRECATED
     [:assets-storage-backend {:optional true} :keyword]
     [:storage-assets-fs-directory {:optional true} :string]
     [:storage-assets-s3-bucket {:optional true} :string]
     [:storage-assets-s3-region {:optional true} :keyword]
     [:storage-assets-s3-endpoint {:optional true} ::sm/uri]
-    [:storage-assets-s3-io-threads {:optional true} ::sm/int]
 
     [:objects-storage-backend {:optional true} :keyword]
     [:objects-storage-fs-directory {:optional true} :string]
     [:objects-storage-s3-bucket {:optional true} :string]
     [:objects-storage-s3-region {:optional true} :keyword]
-    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]
+    [:objects-storage-s3-endpoint {:optional true} ::sm/uri]]))
-    [:objects-storage-s3-io-threads {:optional true} ::sm/int]]))
 
 (defn- parse-flags
   [config]
@@ -298,7 +307,12 @@
 (defn get-deletion-delay
   []
   (or (c/get config :deletion-delay)
-      (dt/duration {:days 7})))
+      (ct/duration {:days 7})))
+
+(defn get-file-clean-delay
+  []
+  (or (c/get config :file-clean-delay)
+      (ct/duration {:days 2})))
 
 (defn get
   "A configuration getter. Helps code be more testable."
@@ -307,5 +321,9 @@
   ([key default]
    (c/get config key default)))
 
+(defn logging-context
+  []
+  {:version/backend (:full version)})
+
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -10,19 +10,20 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
+   [app.common.json :as json]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.db.sql :as sql]
    [app.metrics :as mtx]
-   [app.util.json :as json]
-   [app.util.time :as dt]
    [clojure.java.io :as io]
    [clojure.set :as set]
    [integrant.core :as ig]
    [next.jdbc :as jdbc]
    [next.jdbc.date-time :as jdbc-dt]
+   [next.jdbc.prepare :as jdbc.prepare]
    [next.jdbc.transaction])
   (:import
    com.zaxxer.hikari.HikariConfig
@@ -33,6 +34,7 @@
    java.io.InputStream
    java.io.OutputStream
    java.sql.Connection
+   java.sql.PreparedStatement
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
@@ -296,7 +298,7 @@
 (defn insert!
   "A helper that builds an insert sql statement and executes it. By
   default returns the inserted row with all the field; you can delimit
-  the returned columns with the `::columns` option."
+  the returned columns with the `::sql/columns` option."
   [ds table params & {:as opts}]
   (let [conn (get-connectable ds)
         sql  (sql/insert table params opts)
@@ -377,9 +379,7 @@
 
 (defn is-row-deleted?
   [{:keys [deleted-at]}]
-  (and (dt/instant? deleted-at)
+  (some? deleted-at))
-       (< (inst-ms deleted-at)
-          (inst-ms (dt/now)))))
 
 (defn get*
   "Retrieve a single row from database that matches a simple filters. Do
@@ -404,6 +404,23 @@
                 :hint "database object not found"))
     row))
 
+(defn get-with-sql
+  [ds sql & {:as opts}]
+  (let [rows
+        (cond->> (exec! ds sql opts)
+          (::remove-deleted opts true)
+          (remove is-row-deleted?)
+
+          :always
+          (not-empty))]
+
+    (when (and (not rows) (::throw-if-not-exists opts true))
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "database object not found"))
+
+    (first rows)))
+
 (def ^:private default-plan-opts
   (-> default-opts
       (assoc :fetch-size 1000)
@@ -558,10 +575,10 @@
   [system f & params]
   (cond
     (connection? system)
-    (run! {::conn system} f)
+    (apply run! {::conn system} f params)
 
     (pool? system)
-    (run! {::pool system} f)
+    (apply run! {::pool system} f params)
 
     (::conn system)
     (apply f system params)
@@ -585,7 +602,7 @@
     (string? o)
     (pginterval o)
 
-    (dt/duration? o)
+    (ct/duration? o)
     (interval (inst-ms o))
 
     :else
@@ -599,7 +616,7 @@
           val (.getValue o)]
       (if (or (= typ "json")
               (= typ "jsonb"))
-        (json/decode val)
+        (json/decode val :key-fn keyword)
         val))))
 
 (defn decode-transit-pgobject
@@ -640,7 +657,7 @@
   (when data
     (doto (org.postgresql.util.PGobject.)
       (.setType "jsonb")
-      (.setValue (json/encode-str data)))))
+      (.setValue (json/encode data)))))
 
 ;; --- Locks
 
@@ -686,3 +703,14 @@
   [cause]
   (and (sql-exception? cause)
        (= "40001" (.getSQLState ^java.sql.SQLException cause))))
+
+(defn duplicate-key-error?
+  [cause]
+  (and (sql-exception? cause)
+       (= "23505" (.getSQLState ^java.sql.SQLException cause))))
+
+
+(extend-protocol jdbc.prepare/SettableParameter
+  clojure.lang.Keyword
+  (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]
+    (.setObject s i ^String (d/name v))))

backend/src/app/db/sql.clj

@@ -53,8 +53,15 @@
          opts (cond-> opts
                 (::order-by opts)   (assoc :order-by (::order-by opts))
                 (::columns opts)    (assoc :columns (::columns opts))
-                (::for-update opts) (assoc :suffix "FOR UPDATE")
+
-                (::for-share opts)  (assoc :suffix "FOR SHARE"))]
+                (or (::db/for-update opts)
+                    (::for-update opts))
+                (assoc :suffix "FOR UPDATE")
+
+                (or (::db/for-share opts)
+                    (::for-share opts))
+                (assoc :suffix "FOR SHARE"))]
+
      (sql/for-query table where-params opts))))
 
 (defn update

backend/src/app/email.clj

@@ -7,6 +7,7 @@
 (ns app.email
   "Main api for send emails."
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -93,36 +94,42 @@
                headers)))
 
 (defn- assign-body
-  [^MimeMessage mmsg {:keys [body charset] :or {charset "utf-8"}}]
+  [^MimeMessage mmsg {:keys [body charset attachments] :or {charset "utf-8"}}]
-  (let [mpart (MimeMultipart. "mixed")]
+  (let [mixed-mpart  (MimeMultipart. "mixed")]
     (cond
       (string? body)
-      (let [bpart (MimeBodyPart.)]
+      (let [text-part (MimeBodyPart.)]
-        (.setContent bpart ^String body (str "text/plain; charset=" charset))
+        (.setText text-part ^String body ^String charset)
-        (.addBodyPart mpart bpart))
+        (.addBodyPart mixed-mpart text-part))
-
-      (vector? body)
-      (let [mmp (MimeMultipart. "alternative")
-            mbp (MimeBodyPart.)]
-        (.addBodyPart mpart mbp)
-        (.setContent mbp mmp)
-        (doseq [item body]
-          (let [mbp (MimeBodyPart.)]
-            (.setContent mbp
-                         ^String (:content item)
-                         ^String (str (:type item "text/plain") "; charset=" charset))
-            (.addBodyPart mmp mbp))))
 
       (map? body)
-      (let [bpart (MimeBodyPart.)]
+      (let [content-part      (MimeBodyPart.)
-        (.setContent bpart
+            alternative-mpart (MimeMultipart. "alternative")]
-                     ^String (:content body)
+
-                     ^String (str (:type body "text/plain") "; charset=" charset))
+        (when-let [content (get body "text/plain")]
-        (.addBodyPart mpart bpart))
+          (let [text-part (MimeBodyPart.)]
+            (.setText text-part ^String content ^String charset)
+            (.addBodyPart alternative-mpart text-part)))
+
+        (when-let [content (get body "text/html")]
+          (let [html-part (MimeBodyPart.)]
+            (.setContent html-part ^String content
+                         (str "text/html; charset=" charset))
+            (.addBodyPart alternative-mpart html-part)))
+
+        (.setContent content-part alternative-mpart)
+        (.addBodyPart mixed-mpart content-part))
 
       :else
-      (throw (ex-info "Unsupported type" {:body body})))
+      (throw (IllegalArgumentException. "invalid email body provided")))
-    (.setContent mmsg mpart)
+
+    (doseq [[name content] attachments]
+      (let [attachment-part (MimeBodyPart.)]
+        (.setFileName attachment-part ^String name)
+        (.setContent attachment-part ^String content (str "text/plain; charset=" charset))
+        (.addBodyPart mixed-mpart attachment-part)))
+
+    (.setContent mmsg mixed-mpart)
     mmsg))
 
 (defn- opts->props
@@ -210,24 +217,26 @@
       (ex/raise :type :internal
                 :code :missing-email-templates))
     {:subject subj
-     :body (into
+     :body (d/without-nils
-            [{:type "text/plain"
+            {"text/plain" text
-              :content text}]
+             "text/html" html})}))
-            (when html
-              [{:type "text/html"
-                :content html}]))}))
 
-(def ^:private schema:context
+(def ^:private schema:params
-  [:map
+  [:map {:title "Email Params"}
    [:to [:or ::sm/email [::sm/vec ::sm/email]]]
    [:reply-to {:optional true} ::sm/email]
    [:from {:optional true} ::sm/email]
    [:lang {:optional true} ::sm/text]
+   [:subject {:optional true} ::sm/text]
    [:priority {:optional true} [:enum :high :low]]
-   [:extra-data {:optional true} ::sm/text]])
+   [:extra-data {:optional true} ::sm/text]
+   [:body {:optional true}
+    [:or :string [:map-of :string :string]]]
+   [:attachments {:optional true}
+    [:map-of :string :string]]])
 
-(def ^:private check-context
+(def ^:private check-params
-  (sm/check-fn schema:context))
+  (sm/check-fn schema:params))
 
 (defn template-factory
   [& {:keys [id schema]}]
@@ -235,9 +244,9 @@
   (let [check-fn (if schema
                    (sm/check-fn schema)
                    (constantly nil))]
-    (fn [context]
+    (fn [params]
-      (let [context (-> context check-context check-fn)
+      (let [params (-> params check-params check-fn)
-            email   (build-email-template id context)]
+            email  (build-email-template id params)]
         (when-not email
           (ex/raise :type :internal
                     :code :email-template-does-not-exists
@@ -245,35 +254,40 @@
                     :template-id id))
 
         (cond-> (assoc email :id (name id))
-          (:extra-data context)
+          (:extra-data params)
-          (assoc :extra-data (:extra-data context))
+          (assoc :extra-data (:extra-data params))
 
-          (:from context)
+          (seq (:attachments params))
-          (assoc :from (:from context))
+          (assoc :attachments (:attachments params))
 
-          (:reply-to context)
+          (:from params)
-          (assoc :reply-to (:reply-to context))
+          (assoc :from (:from params))
 
-          (:to context)
+          (:reply-to params)
-          (assoc :to (:to context)))))))
+          (assoc :reply-to (:reply-to params))
+
+          (:to params)
+          (assoc :to (:to params)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PUBLIC HIGH-LEVEL API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn render
-  [email-factory context]
+  [email-factory params]
-  (email-factory context))
+  (email-factory params))
 
 (defn send!
   "Schedule an already defined email to be sent using asynchronously
   using worker task."
-  [{:keys [::conn ::factory] :as context}]
+  [{:keys [::conn ::factory] :as params}]
   (assert (db/connectable? conn) "expected a valid database connection or pool")
 
   (let [email (if factory
-                (factory context)
+                (factory params)
-                (dissoc context ::conn))]
+                (-> params
+                    (dissoc params)
+                    (check-params)))]
     (wrk/submit! {::wrk/task :sendmail
                   ::wrk/delay 0
                   ::wrk/max-retries 4
@@ -343,8 +357,10 @@
 
 (def ^:private schema:feedback
   [:map
-   [:subject ::sm/text]
+   [:feedback-subject ::sm/text]
-   [:content ::sm/text]])
+   [:feedback-type ::sm/text]
+   [:feedback-content ::sm/text]
+   [:profile :map]])
 
 (def user-feedback
   "A profile feedback email."

backend/src/app/features/fdata.clj

@@ -9,46 +9,22 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
-   [app.common.files.helpers :as cfh]
-   [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
-   [app.common.types.path :as path]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.types.objects-map :as omap]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap]
+   [app.util.objects-map :as omap.legacy]
    [app.util.pointer-map :as pmap]))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; OFFLOAD
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn offloaded?
-  [file]
-  (= "objects-storage" (:data-backend file)))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OBJECTS-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn enable-objects-map
-  [file & _opts]
-  (let [update-page
-        (fn [page]
-          (if (and (pmap/pointer-map? page)
-                   (not (pmap/loaded? page)))
-            page
-            (update page :objects omap/wrap)))
-
-        update-data
-        (fn [fdata]
-          (update fdata :pages-index d/update-vals update-page))]
-
-    (-> file
-        (update :data update-data)
-        (update :features conj "fdata/objects-map"))))
-
 (defn process-objects
   "Apply a function to all objects-map on the file. Usualy used for convert
   the objects-map instances to plain maps"
@@ -58,35 +34,237 @@
             (fn [page]
               (update page :objects
                       (fn [objects]
-                        (if (omap/objects-map? objects)
+                        (if (or (omap/objects-map? objects)
+                                (omap.legacy/objects-map? objects))
                           (update-fn objects)
                           objects)))))
     fdata))
 
+
+(defn realize-objects
+  "Process a file and remove all instances of objects map realizing them
+  to a plain data. Used in operation where is more efficient have the
+  whole file loaded in memory or we going to persist it in an
+  alterantive storage."
+  [_cfg file]
+  (update file :data process-objects (partial into {})))
+
+(defn enable-objects-map
+  [file & _opts]
+  (let [update-page
+        (fn [page]
+          (update page :objects omap/wrap))
+
+        update-data
+        (fn [fdata]
+          (update fdata :pages-index d/update-vals update-page))]
+
+    (-> file
+        (update :data update-data)
+        (update :features conj "fdata/objects-map"))))
+
+(defn disable-objects-map
+  [file & _opts]
+  (let [update-page
+        (fn [page]
+          (update page :objects #(into {} %)))
+
+        update-data
+        (fn [fdata]
+          (update fdata :pages-index d/update-vals update-page))]
+
+    (-> file
+        (update :data update-data)
+        (update :features disj "fdata/objects-map"))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; STORAGE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmulti resolve-file-data
+  (fn [_cfg file] (get file :backend "legacy-db")))
+
+(defmethod resolve-file-data "legacy-db"
+  [_cfg {:keys [legacy-data] :as file}]
+  (-> file
+      (assoc :data legacy-data)
+      (dissoc :legacy-data)))
+
+(defmethod resolve-file-data "db"
+  [_cfg file]
+  (dissoc file :legacy-data))
+
+(defmethod resolve-file-data "storage"
+  [cfg {:keys [metadata] :as file}]
+  (let [storage (sto/resolve cfg ::db/reuse-conn true)
+        ref-id  (:storage-ref-id metadata)
+        data    (->> (sto/get-object storage ref-id)
+                     (sto/get-object-bytes storage))]
+    (-> file
+        (assoc :data data)
+        (dissoc :legacy-data))))
+
+(defn decode-file-data
+  [_cfg {:keys [data] :as file}]
+  (cond-> file
+    (bytes? data)
+    (assoc :data (blob/decode data))))
+
+(def ^:private sql:insert-file-data
+  "INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at,
+                          type, backend, metadata, data)
+   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
+
+(def ^:private sql:upsert-file-data
+  (str sql:insert-file-data
+       " ON CONFLICT (file_id, id)
+         DO UPDATE SET modified_at=?,
+                       deleted_at=?,
+                       backend=?,
+                       metadata=?,
+                       data=?"))
+
+(defn- upsert-in-database
+  [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}]
+  (let [created-at  (or created-at (ct/now))
+        metadata    (some-> metadata db/json)
+        modified-at (or modified-at created-at)]
+
+    (db/exec-one! cfg [sql:upsert-file-data
+                       file-id id
+                       created-at
+                       modified-at
+                       deleted-at
+                       type
+                       backend
+                       metadata
+                       data
+                       modified-at
+                       deleted-at
+                       backend
+                       metadata
+                       data])))
+
+(defn- handle-persistence
+  [cfg {:keys [type backend id file-id data] :as params}]
+
+  (cond
+    (= backend "storage")
+    (let [storage  (sto/resolve cfg)
+          content  (sto/content data)
+          sobject  (sto/put-object! storage
+                                    {::sto/content content
+                                     ::sto/touch true
+                                     :bucket "file-data"
+                                     :content-type "application/octet-stream"
+                                     :file-id file-id
+                                     :id id})
+          metadata {:storage-ref-id (:id sobject)}
+          params   (-> params
+                       (assoc :metadata metadata)
+                       (assoc :data nil))]
+      (upsert-in-database cfg params))
+
+    (= backend "db")
+    (->> (dissoc params :metadata)
+         (upsert-in-database cfg))
+
+    (= backend "legacy-db")
+    (cond
+      (= type "main")
+      (do
+        (db/delete! cfg :file-data
+                    {:id id :file-id file-id :type "main"}
+                    {::db/return-keys false})
+        (db/update! cfg :file
+                    {:data data}
+                    {:id file-id}
+                    {::db/return-keys false}))
+
+      (= type "snapshot")
+      (do
+        (db/delete! cfg :file-data
+                    {:id id :file-id file-id :type "snapshot"}
+                    {::db/return-keys false})
+        (db/update! cfg :file-change
+                    {:data data}
+                    {:file-id file-id :id id}
+                    {::db/return-keys false}))
+
+      (= type "fragment")
+      (upsert-in-database cfg
+                          (-> (dissoc params :metadata)
+                              (assoc :backend "db")))
+
+      :else
+      (throw (RuntimeException. "not implemented")))
+
+    :else
+    (throw (IllegalArgumentException.
+            (str "backend '" backend "' not supported")))))
+
+(defn process-metadata
+  [cfg metadata]
+  (when-let [storage-id (:storage-ref-id metadata)]
+    (let [storage (sto/resolve cfg ::db/reuse-conn true)]
+      (sto/touch-object! storage storage-id))))
+
+(defn- default-backend
+  [backend]
+  (or backend (cf/get :file-data-backend)))
+
+(def ^:private schema:metadata
+  [:map {:title "Metadata"}
+   [:storage-ref-id {:optional true} ::sm/uuid]])
+
+(def decode-metadata-with-schema
+  (sm/decoder schema:metadata sm/json-transformer))
+
+(defn decode-metadata
+  [metadata]
+  (some-> metadata
+          (db/decode-json-pgobject)
+          (decode-metadata-with-schema)))
+
+(def ^:private schema:update-params
+  [:map {:closed true}
+   [:id ::sm/uuid]
+   [:type [:enum "main" "snapshot" "fragment"]]
+   [:file-id ::sm/uuid]
+   [:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
+   [:metadata {:optional true} [:maybe schema:metadata]]
+   [:data {:optional true} bytes?]
+   [:created-at {:optional true} ::ct/inst]
+   [:modified-at {:optional true} [:maybe ::ct/inst]]
+   [:deleted-at {:optional true} [:maybe ::ct/inst]]])
+
+(def ^:private check-update-params
+  (sm/check-fn schema:update-params :hint "invalid params received for update"))
+
+(defn upsert!
+  "Create or update file data"
+  [cfg params & {:as opts}]
+  (let [params (-> (check-update-params params)
+                   (update :backend default-backend))]
+
+    (some->> (:metadata params)
+             (process-metadata cfg))
+
+    (-> (handle-persistence cfg params)
+        (db/get-update-count)
+        (pos?))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; POINTER-MAP
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn get-file-data
-  "Get file data given a file instance."
-  [system file]
-  (if (offloaded? file)
-    (let [storage (sto/resolve system ::db/reuse-conn true)]
-      (->> (sto/get-object storage (:data-ref-id file))
-           (sto/get-object-bytes storage)))
-    (:data file)))
-
-(defn resolve-file-data
-  [system file]
-  (let [data (get-file-data system file)]
-    (assoc file :data data)))
-
 (defn load-pointer
   "A database loader pointer helper"
-  [system file-id id]
+  [cfg file-id id]
-  (let [fragment (db/get* system :file-data-fragment
+  (let [fragment (some-> (db/get* cfg :file-data
-                          {:id id :file-id file-id}
+                                  {:id id :file-id file-id :type "fragment"}
-                          {::sql/columns [:data :data-backend :data-ref-id :id]})]
+                                  {::sql/columns [:data :backend :id :metadata]})
+                         (update :metadata decode-metadata))]
 
     (l/trc :hint "load pointer"
            :file-id (str file-id)
@@ -100,22 +278,21 @@
                 :file-id file-id
                 :fragment-id id))
 
-    (let [data (get-file-data system fragment)]
+    (-> (resolve-file-data cfg fragment)
-      ;; FIXME: conditional thread scheduling for decoding big objects
+        (get :data)
-      (blob/decode data))))
+        (blob/decode))))
 
 (defn persist-pointers!
   "Persist all currently tracked pointer objects"
-  [system file-id]
+  [cfg file-id]
-  (let [conn (db/get-connection system)]
+  (doseq [[id item] @pmap/*tracked*]
-    (doseq [[id item] @pmap/*tracked*]
+    (when (pmap/modified? item)
-      (when (pmap/modified? item)
+      (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
-        (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
+      (let [content (-> item deref blob/encode)]
-        (let [content (-> item deref blob/encode)]
+        (upsert! cfg {:id id
-          (db/insert! conn :file-data-fragment
+                      :file-id file-id
-                      {:id id
+                      :type "fragment"
-                       :file-id file-id
+                      :data content})))))
-                       :data content}))))))
 
 (defn process-pointers
   "Apply a function to all pointers on the file. Usuly used for
@@ -129,6 +306,14 @@
         (d/update-vals update-fn')
         (update :pages-index d/update-vals update-fn'))))
 
+(defn realize-pointers
+  "Process a file and remove all instances of pointers realizing them to
+  a plain data. Used in operation where is more efficient have the
+  whole file loaded in memory."
+  [cfg {:keys [id] :as file}]
+  (binding [pmap/*load-fn* (partial load-pointer cfg id)]
+    (update file :data process-pointers deref)))
+
 (defn get-used-pointer-ids
   "Given a file, return all pointer ids used in the data."
   [fdata]
@@ -148,47 +333,12 @@
       (update :features conj "fdata/pointer-map")))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; PATH-DATA
+;; GENERAL PURPOSE HELPERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn enable-path-data
+(defn realize
-  "Enable the fdata/path-data feature on the file."
+  "A helper that combines realize-pointers and realize-objects"
-  [file & _opts]
+  [cfg file]
-  (letfn [(update-object [object]
+  (->> file
-            (if (or (cfh/path-shape? object)
+       (realize-pointers cfg)
-                    (cfh/bool-shape? object))
+       (realize-objects cfg)))
-              (update object :content path/content)
-              object))
-
-          (update-container [container]
-            (d/update-when container :objects d/update-vals update-object))]
-
-    (-> file
-        (update :data (fn [data]
-                        (-> data
-                            (update :pages-index d/update-vals update-container)
-                            (d/update-when :components d/update-vals update-container))))
-        (update :features conj "fdata/path-data"))))
-
-(defn disable-path-data
-  [file & _opts]
-  (letfn [(update-object [object]
-            (if (or (cfh/path-shape? object)
-                    (cfh/bool-shape? object))
-              (update object :content vec)
-              object))
-
-          (update-container [container]
-            (d/update-when container :objects d/update-vals update-object))]
-
-    (when-let [conn db/*conn*]
-      (db/delete! conn :file-migration {:file-id (:id file)
-                                        :name "0003-convert-path-content"}))
-    (-> file
-        (update :data (fn [data]
-                        (-> data
-                            (update :pages-index d/update-vals update-container)
-                            (d/update-when :components d/update-vals update-container))))
-        (update :features disj "fdata/path-data")
-        (update :migrations disj "0003-convert-path-content")
-        (vary-meta update ::fmg/migrated disj "0003-convert-path-content"))))

backend/src/app/features/file_migrations.clj

@@ -8,6 +8,7 @@
   "Backend specific code for file migrations. Implemented as permanent feature of files."
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
    [app.common.files.migrations :as fmg :refer [xf:map-name]]
    [app.db :as db]
    [app.db.sql :as-alias sql]))
@@ -26,14 +27,27 @@
 (defn upsert-migrations!
   "Persist or update file migrations. Return the updated/inserted number
   of rows"
-  [conn {:keys [id] :as file}]
+  [cfg {:keys [id] :as file}]
-  (let [migrations (or (-> file meta ::fmg/migrated)
+  (let [conn       (db/get-connection cfg)
-                       (-> file :migrations not-empty)
+        migrations (or (-> file meta ::fmg/migrated)
-                       fmg/available-migrations)
+                       (-> file :migrations))
         columns    [:file-id :name]
-        rows       (mapv (fn [name] [id name]) migrations)]
+        rows       (->> migrations
+                        (mapv (fn [name] [id name]))
+                        (not-empty))]
+
+    (when-not rows
+      (ex/raise :type :internal
+                :code :missing-migrations
+                :hint "no migrations available on file"))
 
     (-> (db/insert-many! conn :file-migration columns rows
                          {::db/return-keys false
                           ::sql/on-conflict-do-nothing true})
         (db/get-update-count))))
+
+(defn reset-migrations!
+  "Replace file migrations"
+  [cfg {:keys [id] :as file}]
+  (db/delete! cfg :file-migration {:file-id id})
+  (upsert-migrations! cfg file))

backend/src/app/features/file_snapshots.clj

@@ -0,0 +1,446 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.features.file-snapshots
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.features :as-alias cfeat]
+   [app.common.files.migrations :as fmg]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as-alias sql]
+   [app.features.fdata :as fdata]
+   [app.storage :as sto]
+   [app.util.blob :as blob]
+   [app.worker :as wrk]
+   [cuerdas.core :as str]))
+
+(def sql:snapshots
+  "SELECT c.id,
+          c.label,
+          c.created_at,
+          c.updated_at AS modified_at,
+          c.deleted_at,
+          c.profile_id,
+          c.created_by,
+          c.locked_by,
+          c.revn,
+          c.features,
+          c.migrations,
+          c.version,
+          c.file_id,
+          c.data AS legacy_data,
+          fd.data AS data,
+          coalesce(fd.backend, 'legacy-db') AS backend,
+          fd.metadata AS metadata
+     FROM file_change AS c
+     LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id
+                                   AND fd.id = c.id
+                                   AND fd.type = 'snapshot')
+    WHERE c.label IS NOT NULL")
+
+(defn- decode-snapshot
+  [snapshot]
+  (some-> snapshot
+          (-> (d/update-when :metadata fdata/decode-metadata)
+              (d/update-when :migrations db/decode-pgarray [])
+              (d/update-when :features db/decode-pgarray #{}))))
+
+(def ^:private sql:get-minimal-file
+  "SELECT f.id,
+          f.revn,
+          f.modified_at,
+          f.deleted_at,
+          fd.backend AS backend,
+          fd.metadata AS metadata
+     FROM file AS f
+     LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
+    WHERE f.id = ?")
+
+(defn- get-minimal-file
+  [cfg id & {:as opts}]
+  (-> (db/get-with-sql cfg [sql:get-minimal-file id] opts)
+      (d/update-when :metadata fdata/decode-metadata)))
+
+(def ^:private sql:get-snapshot-without-data
+  (str "WITH snapshots AS (" sql:snapshots ")"
+       "SELECT c.id,
+               c.label,
+               c.revn,
+               c.created_at,
+               c.modified_at,
+               c.deleted_at,
+               c.profile_id,
+               c.created_by,
+               c.locked_by,
+               c.features,
+               c.metadata,
+               c.migrations,
+               c.version,
+               c.file_id
+         FROM snapshots AS c
+        WHERE c.id = ?
+          AND CASE WHEN c.created_by = 'user'
+                   THEN c.deleted_at IS NULL
+                   WHEN c.created_by = 'system'
+                   THEN c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz
+               END"))
+
+(defn get-minimal-snapshot
+  [cfg snapshot-id]
+  (let [now (ct/now)]
+    (-> (db/get-with-sql cfg [sql:get-snapshot-without-data snapshot-id now]
+                         {::db/remove-deleted false})
+        (decode-snapshot))))
+
+(def ^:private sql:get-snapshot
+  (str sql:snapshots
+       " AND c.file_id = ?
+         AND c.id = ?
+         AND CASE WHEN c.created_by = 'user'
+                  THEN (c.deleted_at IS NULL)
+                  WHEN c.created_by = 'system'
+                  THEN (c.deleted_at IS NULL OR c.deleted_at >= ?::timestamptz)
+              END"))
+
+(defn- get-snapshot
+  "Get snapshot with decoded data"
+  [cfg file-id snapshot-id]
+  (let [now (ct/now)]
+    (->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now]
+                          {::db/remove-deleted false})
+         (decode-snapshot)
+         (fdata/resolve-file-data cfg)
+         (fdata/decode-file-data cfg))))
+
+(def ^:private sql:get-visible-snapshots
+  (str "WITH "
+       "snapshots1 AS ( " sql:snapshots "),"
+       "snapshots2 AS (
+          SELECT c.id,
+                 c.label,
+                 c.revn,
+                 c.version,
+                 c.created_at,
+                 c.modified_at,
+                 c.created_by,
+                 c.locked_by,
+                 c.profile_id,
+                 c.deleted_at
+            FROM snapshots1 AS c
+           WHERE c.file_id = ?
+       ), snapshots3 AS (
+          (SELECT * FROM snapshots2
+            WHERE created_by = 'system'
+              AND (deleted_at IS NULL OR
+                   deleted_at >= ?::timestamptz)
+            LIMIT 500)
+          UNION ALL
+          (SELECT * FROM snapshots2
+            WHERE created_by = 'user'
+              AND deleted_at IS NULL
+            LIMIT 500)
+       )
+       SELECT * FROM snapshots3
+        ORDER BY created_at DESC"))
+
+(defn get-visible-snapshots
+  "Return a list of snapshots fecheable from the API, it has a limited
+  set of fields and applies big but safe limits over all available
+  snapshots. It return a ordered vector by the snapshot date of
+  creation."
+  [cfg file-id]
+  (let [now (ct/now)]
+    (->> (db/exec! cfg [sql:get-visible-snapshots file-id now])
+         (mapv decode-snapshot))))
+
+(def ^:private schema:decoded-file
+  [:map {:title "DecodedFile"}
+   [:id ::sm/uuid]
+   [:revn :int]
+   [:vern :int]
+   [:data :map]
+   [:version :int]
+   [:features ::cfeat/features]
+   [:migrations [::sm/set :string]]])
+
+(def ^:private schema:snapshot
+  [:map {:title "Snapshot"}
+   [:id ::sm/uuid]
+   [:revn [::sm/int {:min 0}]]
+   [:version [::sm/int {:min 0}]]
+   [:features ::cfeat/features]
+   [:migrations [::sm/set ::sm/text]]
+   [:profile-id {:optional true} ::sm/uuid]
+   [:label ::sm/text]
+   [:file-id ::sm/uuid]
+   [:created-by [:enum "system" "user" "admin"]]
+   [:deleted-at {:optional true} ::ct/inst]
+   [:modified-at ::ct/inst]
+   [:created-at ::ct/inst]])
+
+(def ^:private check-snapshot
+  (sm/check-fn schema:snapshot))
+
+(def ^:private check-decoded-file
+  (sm/check-fn schema:decoded-file))
+
+(defn- generate-snapshot-label
+  []
+  (let [ts (-> (ct/now)
+               (ct/format-inst)
+               (str/replace #"[T:\.]" "-")
+               (str/rtrim "Z"))]
+    (str "snapshot-" ts)))
+
+(def ^:private schema:create-params
+  [:map {:title "SnapshotCreateParams"}
+   [:profile-id ::sm/uuid]
+   [:created-by {:optional true} [:enum "user" "system"]]
+   [:label {:optional true} ::sm/text]
+   [:session-id {:optional true} ::sm/uuid]
+   [:modified-at {:optional true} ::ct/inst]
+   [:deleted-at {:optional true} ::ct/inst]])
+
+(def ^:private check-create-params
+  (sm/check-fn schema:create-params))
+
+(defn create!
+  "Create a file snapshot; expects a non-encoded file"
+  [cfg file & {:as params}]
+  (let [{:keys [label created-by deleted-at profile-id session-id]}
+        (check-create-params params)
+
+        file
+        (check-decoded-file file)
+
+        created-by
+        (or created-by "system")
+
+        snapshot-id
+        (uuid/next)
+
+        created-at
+        (ct/now)
+
+        deleted-at
+        (or deleted-at
+            (if (= created-by "system")
+              (ct/in-future (cf/get-deletion-delay))
+              nil))
+
+        label
+        (or label (generate-snapshot-label))
+
+        snapshot
+        (cond-> {:id snapshot-id
+                 :revn (:revn file)
+                 :version (:version file)
+                 :file-id (:id file)
+                 :features (:features file)
+                 :migrations (:migrations file)
+                 :label label
+                 :created-at created-at
+                 :modified-at created-at
+                 :created-by created-by}
+
+          deleted-at
+          (assoc :deleted-at deleted-at)
+
+          :always
+          (check-snapshot))]
+
+    (db/insert! cfg :file-change
+                (-> snapshot
+                    (update :features into-array)
+                    (update :migrations into-array)
+                    (assoc :updated-at created-at)
+                    (assoc :profile-id profile-id)
+                    (assoc :session-id session-id)
+                    (dissoc :modified-at))
+                {::db/return-keys false})
+
+    (fdata/upsert! cfg
+                   {:id snapshot-id
+                    :file-id (:id file)
+                    :type "snapshot"
+                    :data (blob/encode (:data file))
+                    :created-at created-at
+                    :deleted-at deleted-at})
+
+    snapshot))
+
+(def ^:private schema:update-params
+  [:map {:title "SnapshotUpdateParams"}
+   [:id ::sm/uuid]
+   [:file-id ::sm/uuid]
+   [:label ::sm/text]
+   [:modified-at {:optional true} ::ct/inst]])
+
+(def ^:private check-update-params
+  (sm/check-fn schema:update-params))
+
+(defn update!
+  [cfg params]
+
+  (let [{:keys [id file-id label modified-at]}
+        (check-update-params params)
+
+        modified-at
+        (or modified-at (ct/now))]
+
+    (db/update! cfg :file-data
+                {:deleted-at nil
+                 :modified-at modified-at}
+                {:file-id file-id
+                 :id id
+                 :type "snapshot"}
+                {::db/return-keys false})
+
+    (-> (db/update! cfg :file-change
+                    {:label label
+                     :created-by "user"
+                     :updated-at modified-at
+                     :deleted-at nil}
+                    {:file-id file-id
+                     :id id}
+                    {::db/return-keys false})
+        (db/get-update-count)
+        (pos?))))
+
+(defn restore!
+  [{:keys [::db/conn] :as cfg} file-id snapshot-id]
+  (let [file (get-minimal-file conn file-id {::db/for-update true})
+        vern (rand-int Integer/MAX_VALUE)
+
+        storage
+        (sto/resolve cfg {::db/reuse-conn true})
+
+        snapshot
+        (get-snapshot cfg file-id snapshot-id)]
+
+    (when-not snapshot
+      (ex/raise :type :not-found
+                :code :snapshot-not-found
+                :hint "unable to find snapshot with the provided label"
+                :snapshot-id snapshot-id
+                :file-id file-id))
+
+    (when-not (:data snapshot)
+      (ex/raise :type :internal
+                :code :snapshot-without-data
+                :hint "snapshot has no data"
+                :label (:label snapshot)
+                :file-id file-id))
+
+    (let [;; If the snapshot has applied migrations stored, we reuse
+          ;; them, if not, we take a safest set of migrations as
+          ;; starting point. This is because, at the time of
+          ;; implementing snapshots, migrations were not taken into
+          ;; account so we need to make this backward compatible in
+          ;; some way.
+          migrations
+          (or (:migrations snapshot)
+              (fmg/generate-migrations-from-version 67))
+
+          file
+          (-> file
+              (update :revn inc)
+              (assoc :migrations migrations)
+              (assoc :data (:data snapshot))
+              (assoc :vern vern)
+              (assoc :version (:version snapshot))
+              (assoc :has-media-trimmed false)
+              (assoc :modified-at (:modified-at snapshot))
+              (assoc :features (:features snapshot)))]
+
+      (l/dbg :hint "restoring snapshot"
+             :file-id (str file-id)
+             :label (:label snapshot)
+             :snapshot-id (str (:id snapshot)))
+
+      ;; In the same way, on reseting the file data, we need to restore
+      ;; the applied migrations on the moment of taking the snapshot
+      (bfc/update-file! cfg file ::bfc/reset-migrations? true)
+
+      ;; FIXME: this should be separated functions, we should not have
+      ;; inline sql here.
+
+      ;; clean object thumbnails
+      (let [sql (str "update file_tagged_object_thumbnail "
+                     "   set deleted_at = now() "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/touch-object! storage media-id)))
+
+      ;; clean file thumbnails
+      (let [sql (str "update file_thumbnail "
+                     "   set deleted_at = now() "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/touch-object! storage media-id)))
+
+      vern)))
+
+(defn delete!
+  [cfg & {:keys [id file-id deleted-at]}]
+  (assert (uuid? id) "missing id")
+  (assert (uuid? file-id) "missing file-id")
+  (assert (ct/inst? deleted-at) "missing deleted-at")
+
+  (wrk/submit! {::db/conn (db/get-connection cfg)
+                ::wrk/task :delete-object
+                ::wrk/params {:object :snapshot
+                              :deleted-at deleted-at
+                              :file-id file-id
+                              :id id}})
+  (db/update! cfg :file-change
+              {:deleted-at deleted-at}
+              {:id id :file-id file-id}
+              {::db/return-keys false})
+  true)
+
+(def ^:private sql:get-snapshots
+  (str sql:snapshots " AND c.file_id = ?"))
+
+(defn lock-by!
+  [conn id profile-id]
+  (-> (db/update! conn :file-change
+                  {:locked-by profile-id}
+                  {:id id}
+                  {::db/return-keys false})
+      (db/get-update-count)
+      (pos?)))
+
+(defn unlock!
+  [conn id]
+  (-> (db/update! conn :file-change
+                  {:locked-by nil}
+                  {:id id}
+                  {::db/return-keys false})
+      (db/get-update-count)
+      (pos?)))
+
+(defn reduce-snapshots
+  "Process the file snapshots using efficient reduction; the file
+  reduction comes with all snapshots, including maked as deleted"
+  [cfg file-id xform f init]
+  (let [conn  (db/get-connection cfg)
+        xform (comp
+               (map (partial fdata/resolve-file-data cfg))
+               (map (partial fdata/decode-file-data cfg))
+               xform)]
+
+    (->> (db/plan conn [sql:get-snapshots file-id] {:fetch-size 1})
+         (transduce xform f init))))

backend/src/app/features/logical_deletion.clj

@@ -7,22 +7,23 @@
 (ns app.features.logical-deletion
   "A code related to handle logical deletion mechanism"
   (:require
-   [app.config :as cf]
+   [app.common.time :as ct]
-   [app.util.time :as dt]))
+   [app.config :as cf]))
+
+(def ^:private canceled-status
+  #{"canceled" "unpaid"})
 
 (defn get-deletion-delay
   "Calculate the next deleted-at for a resource (file, team, etc) in function
   of team settings"
   [team]
-  (if-let [subscription (get team :subscription)]
+  (if-let [{:keys [type status]} (get team :subscription)]
     (cond
-      (and (= (:type subscription) "unlimited")
+      (and (= "unlimited" type) (not (contains? canceled-status status)))
-           (= (:status subscription) "active"))
+      (ct/duration {:days 30})
-      (dt/duration {:days 30})
 
-      (and (= (:type subscription) "enterprise")
+      (and (= "enterprise" type) (not (contains? canceled-status status)))
-           (= (:status subscription) "active"))
+      (ct/duration {:days 90})
-      (dt/duration {:days 90})
 
       :else
       (cf/get-deletion-delay))

backend/src/app/http.clj

@@ -17,16 +17,16 @@
    [app.http.awsns :as-alias awsns]
    [app.http.debug :as-alias debug]
    [app.http.errors :as errors]
+   [app.http.management :as mgmt]
    [app.http.middleware :as mw]
+   [app.http.security :as sec]
    [app.http.session :as session]
    [app.http.websocket :as-alias ws]
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [integrant.core :as ig]
-   [promesa.exec :as px]
    [reitit.core :as r]
    [reitit.middleware :as rr]
    [yetti.adapter :as yt]
@@ -53,6 +53,8 @@
   [:map
    [::port ::sm/int]
    [::host ::sm/text]
+   [::io-threads {:optional true} ::sm/int]
+   [::max-worker-threads {:optional true} ::sm/int]
    [::max-body-size {:optional true} ::sm/int]
    [::max-multipart-body-size {:optional true} ::sm/int]
    [::router {:optional true} [:fn r/router?]]
@@ -63,30 +65,41 @@
   (assert (sm/check schema:server-params params)))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [::handler ::router ::host ::port] :as cfg}]
+  [_ {:keys [::handler ::router ::host ::port ::mtx/metrics] :as cfg}]
   (l/info :hint "starting http server" :port port :host host)
-  (let [options {:http/port port
+  (let [on-dispatch
-                 :http/host host
+        (fn [_ start-at-ns]
-                 :http/max-body-size (::max-body-size cfg)
+          (let [timing (- (System/nanoTime) start-at-ns)
-                 :http/max-multipart-body-size (::max-multipart-body-size cfg)
+                timing (int (/ timing 1000000))]
-                 :xnio/io-threads (or (::io-threads cfg)
+            (mtx/run! metrics
-                                      (max 3 (px/get-available-processors)))
+                      :id :http-server-dispatch-timing
-                 :xnio/dispatch :virtual
+                      :val timing)))
-                 :ring/compat :ring2
-                 :socket/backlog 4069}
 
-        handler (cond
+        options
-                  (some? router)
+        {:http/port port
-                  (router-handler router)
+         :http/host host
+         :http/max-body-size (::max-body-size cfg)
+         :http/max-multipart-body-size (::max-multipart-body-size cfg)
+         :xnio/direct-buffers false
+         :xnio/io-threads (::io-threads cfg)
+         :xnio/max-worker-threads (::max-worker-threads cfg)
+         :ring/compat :ring2
+         :events/on-dispatch on-dispatch
+         :socket/backlog 4069}
 
-                  (some? handler)
+        handler
-                  handler
+        (cond
+          (some? router)
+          (router-handler router)
 
-                  :else
+          (some? handler)
-                  (throw (UnsupportedOperationException. "handler or router are required")))
+          handler
 
-        options (d/without-nils options)
+          :else
-        server  (yt/server handler options)]
+          (throw (UnsupportedOperationException. "handler or router are required")))
+
+        server
+        (yt/server handler (d/without-nils options))]
 
     (assoc cfg ::server (yt/start! server))))
 
@@ -135,12 +148,12 @@
   [:map
    [::ws/routes schema:routes]
    [::rpc/routes schema:routes]
-   [::rpc.doc/routes schema:routes]
    [::oidc/routes schema:routes]
    [::assets/routes schema:routes]
    [::debug/routes schema:routes]
    [::mtx/routes schema:routes]
    [::awsns/routes schema:routes]
+   [::mgmt/routes schema:routes]
    ::session/manager
    ::setup/props
    ::db/pool])
@@ -153,10 +166,12 @@
   [_ cfg]
   (rr/router
    [["" {:middleware [[mw/server-timing]
+                      [sec/sec-fetch-metadata]
                       [mw/params]
                       [mw/format-response]
-                      [session/soft-auth cfg]
+                      [mw/auth {:bearer (partial session/decode-token cfg)
-                      [actoken/soft-auth cfg]
+                                :cookie (partial session/decode-token cfg)
+                                :token  (partial actoken/decode-token cfg)}]
                       [mw/parse-request]
                       [mw/errors errors/handle]
                       [mw/restrict-methods]]}
@@ -168,9 +183,9 @@
      ["/webhooks"
       (::awsns/routes cfg)]
 
-     (::ws/routes cfg)
+     ["/management"
+      (::mgmt/routes cfg)]
 
-     ["/api" {:middleware [[mw/cors]]}
+     (::ws/routes cfg)
-      (::oidc/routes cfg)
+     (::oidc/routes cfg)
-      (::rpc.doc/routes cfg)
+     (::rpc/routes cfg)]]))
-      (::rpc/routes cfg)]]]))

backend/src/app/http/access_token.clj

@@ -9,23 +9,19 @@
    [app.common.logging :as l]
    [app.config :as cf]
    [app.db :as db]
+   [app.http :as-alias http]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
-   [app.tokens :as tokens]
+   [app.tokens :as tokens]))
-   [yetti.request :as yreq]))
 
-(def header-re #"^Token\s+(.*)")
+(defn decode-token
-
+  [cfg token]
-(defn- get-token
+  (try
-  [request]
+    (tokens/verify cfg {:token token :iss "access-token"})
-  (some->> (yreq/get-header request "authorization")
+    (catch Throwable cause
-           (re-matches header-re)
+      (l/trc :hint "exception on decoding token"
-           (second)))
+             :token token
-
+             :cause cause))))
-(defn- decode-token
-  [props token]
-  (when token
-    (tokens/verify props {:token token :iss "access-token"})))
 
 (def sql:get-token-data
   "SELECT perms, profile_id, expires_at
@@ -35,47 +31,28 @@
            OR (expires_at > now()));")
 
 (defn- get-token-data
-  [pool token-id]
+  [pool claims]
   (when-not (db/read-only? pool)
-    (some-> (db/exec-one! pool [sql:get-token-data token-id])
+    (when-let [token-id (get claims :tid)]
-            (update :perms db/decode-pgarray #{}))))
+      (some-> (db/exec-one! pool [sql:get-token-data token-id])
-
+              (update :perms db/decode-pgarray #{})))))
-(defn- wrap-soft-auth
-  "Soft Authentication, will be executed synchronously on the undertow
-  worker thread."
-  [handler {:keys [::setup/props]}]
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token props token)]
-                (cond-> request
-                  (map? claims)
-                  (assoc ::id (:tid claims))))
-              (catch Throwable cause
-                (l/trace :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
 
 (defn- wrap-authz
-  "Authorization middleware, will be executed synchronously on vthread."
   [handler {:keys [::db/pool]}]
   (fn [request]
-    (let [{:keys [perms profile-id expires-at]} (some->> (::id request) (get-token-data pool))]
+    (let [{:keys [type claims]} (get request ::http/auth-data)]
-      (handler (cond-> request
+      (if (= :token type)
-                 (some? perms)
+        (let [{:keys [perms profile-id expires-at]} (some->> claims (get-token-data pool))]
-                 (assoc ::perms perms)
+          ;; FIXME: revisit this, this data looks unused
-                 (some? profile-id)
+          (handler (cond-> request
-                 (assoc ::profile-id profile-id)
+                     (some? perms)
-                 (some? expires-at)
+                     (assoc ::perms perms)
-                 (assoc ::expires-at expires-at))))))
+                     (some? profile-id)
+                     (assoc ::profile-id profile-id)
+                     (some? expires-at)
+                     (assoc ::expires-at expires-at))))
 
-(def soft-auth
+        (handler request)))))
-  {:name ::soft-auth
-   :compile (fn [& _]
-              (when (contains? cf/flags :access-tokens)
-                wrap-soft-auth))})
 
 (def authz
   {:name ::authz

backend/src/app/http/assets.clj

@@ -9,18 +9,18 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.time :as ct]
    [app.common.uri :as u]
    [app.db :as db]
    [app.storage :as sto]
-   [app.util.time :as dt]
    [integrant.core :as ig]
    [yetti.response :as-alias yres]))
 
 (def ^:private cache-max-age
-  (dt/duration {:hours 24}))
+  (ct/duration {:hours 24}))
 
 (def ^:private signature-max-age
-  (dt/duration {:hours 24 :minutes 15}))
+  (ct/duration {:hours 24 :minutes 15}))
 
 (defn get-id
   [{:keys [path-params]}]
@@ -30,7 +30,7 @@
 
 (defn- get-file-media-object
   [pool id]
-  (db/get pool :file-media-object {:id id}))
+  (db/get pool :file-media-object {:id id} {::db/remove-deleted false}))
 
 (defn- serve-object-from-s3
   [{:keys [::sto/storage] :as cfg} obj]

backend/src/app/http/awsns.clj

@@ -17,11 +17,9 @@
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [app.worker :as-alias wrk]
    [clojure.data.json :as j]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [promesa.exec :as px]
    [yetti.request :as yreq]
    [yetti.response :as-alias yres]))
 
@@ -40,8 +38,8 @@
   [_ cfg]
   (letfn [(handler [request]
             (let [data (-> request yreq/body slurp)]
-              (px/run! :vthread (partial handle-request cfg data)))
+              (handle-request cfg data)
-            {::yres/status 200})]
+              {::yres/status 200}))]
     ["/sns" {:handler handler
              :allowed-methods #{:post}}]))
 
@@ -109,7 +107,7 @@
   [cfg headers]
   (let [tdata (get headers "x-penpot-data")]
     (when-not (str/empty? tdata)
-      (let [result (tokens/verify (::setup/props cfg) {:token tdata :iss :profile-identity})]
+      (let [result (tokens/verify cfg {:token tdata :iss :profile-identity})]
         (:profile-id result)))))
 
 (defn- parse-notification

backend/src/app/http/client.clj

@@ -9,8 +9,7 @@
   (:require
    [app.common.schema :as sm]
    [integrant.core :as ig]
-   [java-http-clj.core :as http]
+   [java-http-clj.core :as http])
-   [promesa.core :as p])
   (:import
    java.net.http.HttpClient))
 
@@ -29,14 +28,9 @@
 
 (defn send!
   ([client req] (send! client req {}))
-  ([client req {:keys [response-type sync?] :or {response-type :string sync? false}}]
+  ([client req {:keys [response-type] :or {response-type :string}}]
    (assert (client? client) "expected valid http client")
-   (if sync?
+   (http/send req {:client client :as response-type})))
-     (http/send req {:client client :as response-type})
-     (try
-       (http/send-async req {:client client :as response-type})
-       (catch Throwable cause
-         (p/rejected cause))))))
 
 (defn- resolve-client
   [params]
@@ -56,8 +50,8 @@
   ([cfg-or-client request]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request {:sync? true})))
+     (send! client request {})))
   ([cfg-or-client request options]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request (merge {:sync? true} options)))))
+     (send! client request options))))

backend/src/app/http/debug.clj

@@ -15,21 +15,24 @@
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.pprint :as pp]
+   [app.common.time :as ct]
+   [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.features.file-migrations :as feat.fmig]
    [app.http.session :as session]
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.files-create :refer [create-file]]
    [app.rpc.commands.profile :as profile]
    [app.rpc.commands.teams :as teams]
    [app.setup :as-alias setup]
+   [app.setup.clock :as clock]
    [app.srepl.main :as srepl]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [app.util.blob :as blob]
    [app.util.template :as tmpl]
-   [app.util.time :as dt]
    [cuerdas.core :as str]
    [datoteka.io :as io]
    [emoji.core :as emj]
@@ -47,29 +50,35 @@
 
 (defn index-handler
   [_cfg _request]
-  {::yres/status  200
+  (let [{:keys [clock offset]} @clock/current]
-   ::yres/headers {"content-type" "text/html"}
+    {::yres/status  200
-   ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
+     ::yres/headers {"content-type" "text/html"}
-                      (tmpl/render {:version (:full cf/version)}))})
+     ::yres/body    (-> (io/resource "app/templates/debug.tmpl")
+                        (tmpl/render {:version (:full cf/version)
+                                      :current-clock  (str clock)
+                                      :current-offset (if offset
+                                                        (ct/format-duration offset)
+                                                        "NO OFFSET")
+                                      :current-time  (ct/format-inst (ct/now) :http)
+                                      :supported-features cfeat/supported-features}))}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FILE CHANGES
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn prepare-response
+(defn- get-resolved-file
-  [body]
+  [cfg file-id]
-  (let [headers {"content-type" "application/transit+json"}]
+  (some-> (bfc/get-file cfg file-id :migrate? false)
-    {::yres/status 200
+          (update :data blob/encode)))
-     ::yres/body body
-     ::yres/headers headers}))
 
-(defn prepare-download-response
+(defn prepare-download
-  [body filename]
+  [file filename]
-  (let [headers {"content-disposition" (str "attachment; filename=" filename)
+  {::yres/status 200
-                 "content-type" "application/octet-stream"}]
+   ::yres/headers
-    {::yres/status 200
+   {"content-disposition" (str "attachment; filename=" filename ".json")
-     ::yres/body body
+    "content-type" "application/octet-stream"}
-     ::yres/headers headers}))
+   ::yres/body
+   (t/encode file {:type :json-verbose})})
 
 (def sql:retrieve-range-of-changes
   "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
@@ -77,45 +86,51 @@
 (def sql:retrieve-single-change
   "select revn, changes, data from file_change where file_id=? and revn = ?")
 
-(defn- retrieve-file-data
+(defn- download-file-data
-  [{:keys [::db/pool]} {:keys [params ::session/profile-id] :as request}]
+  [cfg {:keys [params ::session/profile-id] :as request}]
   (let [file-id  (some-> params :file-id parse-uuid)
-        revn     (some-> params :revn parse-long)
         filename (str file-id)]
 
     (when-not file-id
       (ex/raise :type :validation
                 :code :missing-arguments))
 
-    (let [data (if (integer? revn)
+    (if-let [file (get-resolved-file cfg file-id)]
-                 (some-> (db/exec-one! pool [sql:retrieve-single-change file-id revn]) :data)
-                 (some-> (db/get-by-id pool :file file-id) :data))]
-
-      (when-not data
-        (ex/raise :type :not-found
-                  :code :enpty-data
-                  :hint "empty response"))
       (cond
         (contains? params :download)
-        (prepare-download-response data filename)
+        (prepare-download file filename)
 
         (contains? params :clone)
-        (let [profile    (profile/get-profile pool profile-id)
+        (db/tx-run! cfg
-              project-id (:default-project-id profile)]
+                    (fn [{:keys [::db/conn] :as cfg}]
+                      (let [profile    (profile/get-profile conn profile-id)
+                            project-id (:default-project-id profile)
+                            file       (-> (create-file cfg {:id (uuid/next)
+                                                             :name (str "Cloned: " (:name file))
+                                                             :features (:features file)
+                                                             :project-id project-id
+                                                             :profile-id profile-id})
+                                           (assoc :data (:data file))
+                                           (assoc :migrations (:migrations file)))]
 
-          (db/run! pool (fn [{:keys [::db/conn] :as cfg}]
+                        (feat.fmig/reset-migrations! conn file)
-                          (create-file cfg {:id file-id
+                        (db/update! conn :file
-                                            :name (str "Cloned file: " filename)
+                                    {:data (:data file)}
-                                            :project-id project-id
+                                    {:id (:id file)}
-                                            :profile-id profile-id})
+                                    {::db/return-keys false})
-                          (db/update! conn :file
+
-                                      {:data data}
+
-                                      {:id file-id})
+                        {::yres/status 201
-                          {::yres/status 201
+                         ::yres/body "OK CLONED"})))
-                           ::yres/body "OK CREATED"})))
 
         :else
-        (prepare-response (blob/decode data))))))
+        (ex/raise :type :validation
+                  :code :invalid-params
+                  :hint "invalid button"))
+
+      (ex/raise :type :not-found
+                :code :enpty-data
+                :hint "empty response"))))
 
 (defn- is-file-exists?
   [pool id]
@@ -123,81 +138,61 @@
     (-> (db/exec-one! pool [sql id]) :exists)))
 
 (defn- upload-file-data
-  [{:keys [::db/pool]} {:keys [::session/profile-id params] :as request}]
+  [{:keys [::db/pool] :as cfg} {:keys [::session/profile-id params] :as request}]
   (let [profile    (profile/get-profile pool profile-id)
         project-id (:default-project-id profile)
-        data       (some-> params :file :path io/read*)]
+        file       (some-> params :file :path io/read* t/decode)]
 
-    (if (and data project-id)
+    (if (and file project-id)
-      (let [fname     (str "Imported file *: " (dt/now))
+      (let [fname     (str "Imported: " (:name file) "(" (ct/now) ")")
             reuse-id? (contains? params :reuseid)
             file-id   (or (and reuse-id? (ex/ignoring (-> params :file :filename parse-uuid)))
                           (uuid/next))]
 
         (if (and reuse-id? file-id
                  (is-file-exists? pool file-id))
-          (do
+          (db/tx-run! cfg
-            (db/update! pool :file
+                      (fn [{:keys [::db/conn] :as cfg}]
-                        {:data data
+                        (db/update! conn :file
-                         :deleted-at nil}
+                                    {:data (:data file)
-                        {:id file-id})
+                                     :features (into-array (:features file))
-            {::yres/status 200
+                                     :deleted-at nil}
-             ::yres/body "OK UPDATED"})
+                                    {:id file-id}
+                                    {::db/return-keys false})
+                        (feat.fmig/reset-migrations! conn file)
+                        {::yres/status 200
+                         ::yres/body "OK UPDATED"}))
+
+          (db/tx-run! cfg
+                      (fn [{:keys [::db/conn] :as cfg}]
+                        (let [file (-> (create-file cfg {:id file-id
+                                                         :name fname
+                                                         :features (:features file)
+                                                         :project-id project-id
+                                                         :profile-id profile-id})
+                                       (assoc :data (:data file))
+                                       (assoc :migrations (:migrations file)))]
 
-          (db/run! pool (fn [{:keys [::db/conn] :as cfg}]
-                          (create-file cfg {:id file-id
-                                            :name fname
-                                            :project-id project-id
-                                            :profile-id profile-id})
                           (db/update! conn :file
-                                      {:data data}
+                                      {:data (:data file)}
-                                      {:id file-id})
+                                      {:id file-id}
+                                      {::db/return-keys false})
+                          (feat.fmig/reset-migrations! conn file)
                           {::yres/status 201
-                           ::yres/body "OK CREATED"}))))
+                           ::yres/body "OK CREATED"})))))
 
-      {::yres/status 500
+      (ex/raise :type :validation
-       ::yres/body "ERROR"})))
+                :code :invalid-params
+                :hint "invalid file uploaded"))))
 
-(defn file-data-handler
+(defn raw-export-import-handler
   [cfg request]
   (case (yreq/method request)
-    :get (retrieve-file-data cfg request)
+    :get (download-file-data cfg request)
     :post (upload-file-data cfg request)
     (ex/raise :type :http
               :code :method-not-found)))
 
-(defn file-changes-handler
-  [{:keys [::db/pool]} {:keys [params] :as request}]
-  (letfn [(retrieve-changes [file-id revn]
-            (if (str/includes? revn ":")
-              (let [[start end] (->> (str/split revn #":")
-                                     (map str/trim)
-                                     (map parse-long))]
-                (some->> (db/exec! pool [sql:retrieve-range-of-changes file-id start end])
-                         (map :changes)
-                         (map blob/decode)
-                         (mapcat identity)
-                         (vec)))
-
-              (if-let [revn (parse-long revn)]
-                (let [item (db/exec-one! pool [sql:retrieve-single-change file-id revn])]
-                  (some-> item :changes blob/decode vec))
-                (ex/raise :type :validation :code :invalid-arguments))))]
-
-    (let [file-id  (some-> params :id parse-uuid)
-          revn     (or (some-> params :revn parse-long) "latest")
-          filename (str file-id)]
-
-      (when (or (not file-id) (not revn))
-        (ex/raise :type :validation
-                  :code :invalid-arguments
-                  :hint "missing arguments"))
-
-      (let [data (retrieve-changes file-id revn)]
-        (if (contains? params :download)
-          (prepare-download-response data filename)
-          (prepare-response data))))))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; ERROR BROWSER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -234,7 +229,7 @@
             (-> (io/resource "app/templates/error-report.v3.tmpl")
                 (tmpl/render (-> content
                                  (assoc :id id)
-                                 (assoc :created-at (dt/format-instant created-at :rfc1123))))))]
+                                 (assoc :created-at (ct/format-inst created-at :rfc1123))))))]
 
     (if-let [report (get-report request)]
       (let [result (case (:version report)
@@ -258,7 +253,7 @@
 (defn error-list-handler
   [{:keys [::db/pool]} _request]
   (let [items (->> (db/exec! pool [sql:error-reports])
-                   (map #(update % :created-at dt/format-instant :rfc1123)))]
+                   (map #(update % :created-at ct/format-inst :rfc1123)))]
     {::yres/status 200
      ::yres/body (-> (io/resource "app/templates/error-list.tmpl")
                      (tmpl/render {:items items}))
@@ -402,77 +397,67 @@
                            ::yres/headers {"content-type" "text/plain"}
                            ::yres/body    (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))))
 
-
+(defn- handle-team-features
-(defn- reset-file-version
   [cfg {:keys [params] :as request}]
-  (let [file-id (some-> params :file-id d/parse-uuid)
+  (let [team-id    (some-> params :team-id d/parse-uuid)
-        version (some-> params :version d/parse-integer)]
+        feature    (some-> params :feature str)
-
+        action     (some-> params :action)
-    (when-not (contains? params :force)
-      (ex/raise :type :validation
-                :code :missing-force
-                :hint "missing force checkbox"))
-
-    (when (nil? file-id)
-      (ex/raise :type :validation
-                :code :invalid-file-id
-                :hint "provided invalid file id"))
-
-    (when (nil? version)
-      (ex/raise :type :validation
-                :code :invalid-version
-                :hint "provided invalid version"))
-
-    (db/tx-run! cfg srepl/process-file! file-id #(assoc % :version version))
-
-    {::yres/status  200
-     ::yres/headers {"content-type" "text/plain"}
-     ::yres/body    "OK"}))
-
-
-(defn- add-team-feature
-  [{:keys [params] :as request}]
-  (let [team-id (some-> params :team-id d/parse-uuid)
-        feature (some-> params :feature str)
         skip-check (contains? params :skip-check)]
 
-    (when-not (contains? params :force)
-      (ex/raise :type :validation
-                :code :missing-force
-                :hint "missing force checkbox"))
-
     (when (nil? team-id)
       (ex/raise :type :validation
                 :code :invalid-team-id
                 :hint "provided invalid team id"))
 
-    (srepl/enable-team-feature! team-id feature :skip-check skip-check)
+    (if (= action "show")
+      (let [team (db/run! cfg teams/get-team-info {:id team-id})]
+        {::yres/status 200
+         ::yres/headers {"content-type" "text/plain"}
+         ::yres/body (apply str "Team features:\n"
+                            (->> (:features team)
+                                 (map (fn [feature]
+                                        (str "- " feature "\n")))))})
 
-    {::yres/status  200
+      (do
-     ::yres/headers {"content-type" "text/plain"}
+        (when-not (contains? params :force)
-     ::yres/body    "OK"}))
+          (ex/raise :type :validation
+                    :code :missing-force
+                    :hint "missing force checkbox"))
 
-(defn- remove-team-feature
+        (cond
-  [{:keys [params] :as request}]
+          (= action "enable")
-  (let [team-id   (some-> params :team-id d/parse-uuid)
+          (srepl/enable-team-feature! team-id feature :skip-check skip-check)
-        feature   (some-> params :feature str)
-        skip-check (contains? params :skip-check)]
 
-    (when-not (contains? params :force)
+          (= action "disable")
-      (ex/raise :type :validation
+          (srepl/disable-team-feature! team-id feature :skip-check skip-check)
-                :code :missing-force
-                :hint "missing force checkbox"))
 
-    (when (nil? team-id)
+          :else
-      (ex/raise :type :validation
+          (ex/raise :type :validation
-                :code :invalid-team-id
+                    :code :invalid-action
-                :hint "provided invalid team id"))
+                    :hint (str "invalid action: " action)))
 
-    (srepl/disable-team-feature! team-id feature :skip-check skip-check)
 
-    {::yres/status  200
+        {::yres/status  200
-     ::yres/headers {"content-type" "text/plain"}
+         ::yres/headers {"content-type" "text/plain"}
-     ::yres/body    "OK"}))
+         ::yres/body    "OK"}))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; VIRTUAL CLOCK
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- set-virtual-clock
+  [_ {:keys [params] :as request}]
+  (let [offset (some-> params :offset str/trim not-empty ct/duration)
+        reset? (contains? params :reset)]
+    (if (= "production" (cf/get :tenant))
+      {::yres/status 501
+       ::yres/body "OPERATION NOT ALLOWED"}
+      (do
+        (if (or reset? (zero? (inst-ms offset)))
+          (clock/set-offset! nil)
+          (clock/set-offset! offset))
+        {::yres/status 302
+         ::yres/headers {"location" "/dbg"}}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; OTHER SMALL VIEWS/HANDLERS
@@ -525,6 +510,25 @@
            (ex/raise :type :authentication
                      :code :only-admins-allowed)))))})
 
+(def errors
+  (letfn [(handle-error [cause]
+            (when-let [data (ex-data cause)]
+              (when (= :validation (:type data))
+                (str "Error: " (or (:hint data) (ex-message cause)) "\n"))))]
+    {:name ::errors
+     :compile
+     (fn [& _params]
+       (fn [handler]
+         (fn [request]
+           (try
+             (handler request)
+             (catch Throwable cause
+               (let [body (or (handle-error cause)
+                              (ex/format-throwable cause))]
+                 {::yres/status 400
+                  ::yres/headers {"content-type" "text/plain"}
+                  ::yres/body body}))))))}))
+
 (defmethod ig/assert-key ::routes
   [_ params]
   (assert (db/pool? (::db/pool params)) "expected a valid database pool")
@@ -540,15 +544,14 @@
     ["/changelog" {:handler (partial changelog-handler cfg)}]
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
-    ["/actions/resend-email-verification"
+    ["/actions" {:middleware [[errors]]}
-     {:handler (partial resend-email-notification cfg)}]
+     ["/set-virtual-clock"
-    ["/actions/reset-file-version"
+      {:handler (partial set-virtual-clock cfg)}]
-     {:handler (partial reset-file-version cfg)}]
+     ["/resend-email-verification"
-    ["/actions/add-team-feature"
+      {:handler (partial resend-email-notification cfg)}]
-     {:handler (partial add-team-feature)}]
+     ["/handle-team-features"
-    ["/actions/remove-team-feature"
+      {:handler (partial handle-team-features cfg)}]
-     {:handler (partial remove-team-feature)}]
+     ["/file-export" {:handler (partial export-handler cfg)}]
-    ["/file/export" {:handler (partial export-handler cfg)}]
+     ["/file-import" {:handler (partial import-handler cfg)}]
-    ["/file/import" {:handler (partial import-handler cfg)}]
+     ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])
-    ["/file/data" {:handler (partial file-data-handler cfg)}]
+
-    ["/file/changes" {:handler (partial file-changes-handler cfg)}]]])

backend/src/app/http/errors.clj

@@ -13,6 +13,7 @@
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
+   [app.http.auth :as-alias auth]
    [app.http.session :as-alias session]
    [app.util.inet :as inet]
    [clojure.spec.alpha :as s]
@@ -22,18 +23,16 @@
 (defn request->context
   "Extracts error report relevant context data from request."
   [request]
-  (let [claims (-> {}
+  (let [{:keys [claims] :as auth} (get request ::http/auth-data)]
-                   (into (::session/token-claims request))
+    (-> (cf/logging-context)
-                   (into (::actoken/token-claims request)))]
+        (assoc :request/path (:path request))
-    {:request/path       (:path request)
+        (assoc :request/method (:method request))
-     :request/method     (:method request)
+        (assoc :request/params (:params request))
-     :request/params     (:params request)
+        (assoc :request/user-agent (yreq/get-header request "user-agent"))
-     :request/user-agent (yreq/get-header request "user-agent")
+        (assoc :request/ip-addr (inet/parse-request request))
-     :request/ip-addr    (inet/parse-request request)
+        (assoc :request/profile-id (get claims :uid))
-     :request/profile-id (:uid claims)
+        (assoc :request/auth-data auth)
-     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
+        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
-     :version/backend    (:full cf/version)}))
-
 
 (defmulti handle-error
   (fn [cause _ _]
@@ -61,8 +60,6 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
-        (l/err :hint "restriction error"
-               :cause err)
         {::yres/status 400
          ::yres/body data}))))
 

backend/src/app/http/management.clj

@@ -0,0 +1,240 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.http.management
+  "Internal mangement HTTP API"
+  (:require
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.schema.generators :as sg]
+   [app.common.time :as ct]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.http.middleware :as mw]
+   [app.main :as-alias main]
+   [app.rpc.commands.profile :as cmd.profile]
+   [app.setup :as-alias setup]
+   [app.tokens :as tokens]
+   [app.worker :as-alias wrk]
+   [integrant.core :as ig]
+   [yetti.response :as-alias yres]))
+
+;; ---- ROUTES
+
+(declare ^:private authenticate)
+(declare ^:private get-customer)
+(declare ^:private update-customer)
+
+(defmethod ig/assert-key ::routes
+  [_ params]
+  (assert (db/pool? (::db/pool params)) "expect valid database pool"))
+
+(def ^:private default-system
+  {:name ::default-system
+   :compile
+   (fn [_ _]
+     (fn [handler cfg]
+       (fn [request]
+         (handler cfg request))))})
+
+(def ^:private transaction
+  {:name ::transaction
+   :compile
+   (fn [data _]
+     (when (:transaction data)
+       (fn [handler]
+         (fn [cfg request]
+           (db/tx-run! cfg handler request)))))})
+
+(defmethod ig/init-key ::routes
+  [_ {:keys [::setup/props] :as cfg}]
+
+  (let [management-key (or (cf/get :management-api-key)
+                           (get props :management-key))]
+
+    ["" {:middleware [[mw/shared-key-auth management-key]
+                      [default-system cfg]
+                      [transaction]]}
+     ["/authenticate"
+      {:handler authenticate
+       :allowed-methods #{:post}}]
+
+     ["/get-customer"
+      {:handler get-customer
+       :transaction true
+       :allowed-methods #{:post}}]
+
+     ["/update-customer"
+      {:handler update-customer
+       :allowed-methods #{:post}
+       :transaction true}]]))
+
+;; ---- HELPERS
+
+(defn- coercer
+  [schema & {:as opts}]
+  (let [decode-fn (sm/decoder schema sm/json-transformer)
+        check-fn  (sm/check-fn schema opts)]
+    (fn [data]
+      (-> data decode-fn check-fn))))
+
+;; ---- API: AUTHENTICATE
+
+(defn- authenticate
+  [cfg request]
+  (let [token  (-> request :params :token)
+        result (tokens/verify cfg {:token token :iss "authentication"})]
+    {::yres/status 200
+     ::yres/body result}))
+
+;; ---- API: GET-CUSTOMER
+
+(def ^:private schema:get-customer
+  [:map [:id ::sm/uuid]])
+
+(def ^:private coerce-get-customer-params
+  (coercer schema:get-customer
+           :type :validation
+           :hint "invalid data provided for `get-customer` rpc call"))
+
+(def ^:private sql:get-customer-slots
+  "WITH teams AS (
+    SELECT tpr.team_id AS id,
+           tpr.profile_id AS profile_id
+      FROM team_profile_rel AS tpr
+     WHERE tpr.is_owner IS true
+       AND tpr.profile_id = ?
+  ), teams_with_slots AS (
+    SELECT tpr.team_id AS id,
+           count(*) AS total
+      FROM team_profile_rel AS tpr
+     WHERE tpr.team_id IN (SELECT id FROM teams)
+       AND tpr.can_edit IS true
+     GROUP BY 1
+     ORDER BY 2
+  )
+  SELECT max(total) AS total FROM teams_with_slots;")
+
+(defn- get-customer-slots
+  [cfg profile-id]
+  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
+    (:total result)))
+
+(defn- get-customer
+  [cfg request]
+  (let [profile-id (-> request :params coerce-get-customer-params :id)
+        profile    (cmd.profile/get-profile cfg profile-id)
+        result     {:id (get profile :id)
+                    :name (get profile :fullname)
+                    :email (get profile :email)
+                    :num-editors (get-customer-slots cfg profile-id)
+                    :subscription (-> profile :props :subscription)}]
+    {::yres/status 200
+     ::yres/body result}))
+
+
+;; ---- API: UPDATE-CUSTOMER
+
+(def ^:private schema:timestamp
+  (sm/type-schema
+   {:type ::timestamp
+    :pred ct/inst?
+    :type-properties
+    {:title "inst"
+     :description "The same as :app.common.time/inst but encodes to epoch"
+     :error/message "should be an instant"
+     :gen/gen (->> (sg/small-int)
+                   (sg/fmap (fn [v] (ct/inst v))))
+     :decode/string ct/inst
+     :encode/string inst-ms
+     :decode/json ct/inst
+     :encode/json inst-ms}}))
+
+(def ^:private schema:subscription
+  [:map {:title "Subscription"}
+   [:id ::sm/text]
+   [:customer-id ::sm/text]
+   [:type [:enum
+           "unlimited"
+           "professional"
+           "enterprise"]]
+   [:status [:enum
+             "active"
+             "canceled"
+             "incomplete"
+             "incomplete_expired"
+             "past_due"
+             "paused"
+             "trialing"
+             "unpaid"]]
+
+   [:billing-period [:enum
+                     "month"
+                     "day"
+                     "week"
+                     "year"]]
+   [:quantity :int]
+   [:description [:maybe ::sm/text]]
+   [:created-at schema:timestamp]
+   [:start-date [:maybe schema:timestamp]]
+   [:ended-at [:maybe schema:timestamp]]
+   [:trial-end [:maybe schema:timestamp]]
+   [:trial-start [:maybe schema:timestamp]]
+   [:cancel-at [:maybe schema:timestamp]]
+   [:canceled-at [:maybe schema:timestamp]]
+   [:current-period-end [:maybe schema:timestamp]]
+   [:current-period-start [:maybe schema:timestamp]]
+   [:cancel-at-period-end :boolean]
+
+   [:cancellation-details
+    [:map {:title "CancellationDetails"}
+     [:comment [:maybe ::sm/text]]
+     [:reason [:maybe ::sm/text]]
+     [:feedback [:maybe
+                 [:enum
+                  "customer_service"
+                  "low_quality"
+                  "missing_feature"
+                  "other"
+                  "switched_service"
+                  "too_complex"
+                  "too_expensive"
+                  "unused"]]]]]])
+
+(def ^:private schema:update-customer
+  [:map
+   [:id ::sm/uuid]
+   [:subscription [:maybe schema:subscription]]])
+
+(def ^:private coerce-update-customer-params
+  (coercer schema:update-customer
+           :type :validation
+           :hint "invalid data provided for `update-customer` rpc call"))
+
+(defn- update-customer
+  [cfg request]
+  (let [{:keys [id subscription]}
+        (-> request :params coerce-update-customer-params)
+
+        {:keys [props] :as profile}
+        (cmd.profile/get-profile cfg id ::db/for-update true)
+
+        props
+        (assoc props :subscription subscription)]
+
+    (l/dbg :hint "update customer"
+           :profile-id (str id)
+           :subscription-type (get subscription :type)
+           :subscription-status (get subscription :status)
+           :subscription-quantity (get subscription :quantity))
+
+    (db/update! cfg :profile
+                {:props (db/tjson props)}
+                {:id id}
+                {::db/return-keys false})
+
+    {::yres/status 201
+     ::yres/body nil}))

backend/src/app/http/middleware.clj

@@ -12,8 +12,11 @@
    [app.common.schema :as-alias sm]
    [app.common.transit :as t]
    [app.config :as cf]
+   [app.http :as-alias http]
    [app.http.errors :as errors]
+   [app.tokens :as tokens]
    [app.util.pointer-map :as pmap]
+   [buddy.core.codecs :as bc]
    [cuerdas.core :as str]
    [yetti.adapter :as yt]
    [yetti.middleware :as ymw]
@@ -240,3 +243,77 @@
              (if (contains? allowed method)
                (handler request)
                {::yres/status 405}))))))})
+
+(defn- wrap-auth
+  [handler decoders]
+  (let [token-re
+        #"(?i)^(Token|Bearer)\s+(.*)"
+
+        get-token-from-authorization
+        (fn [request]
+          (when-let [[_ token-type token] (some->> (yreq/get-header request "authorization")
+                                                   (re-matches token-re))]
+            (if (= "token" (str/lower token-type))
+              {:type :token
+               :token token}
+              {:type :bearer
+               :token token})))
+
+        get-token-from-cookie
+        (fn [request]
+          (let [cname (cf/get :auth-token-cookie-name)
+                token (some-> (yreq/get-cookie request cname) :value)]
+            (when-not (str/empty? token)
+              {:type :cookie
+               :token token})))
+
+        get-token
+        (some-fn get-token-from-cookie get-token-from-authorization)
+
+        process-request
+        (fn [request]
+          (if-let [{:keys [type token] :as auth} (get-token request)]
+            (let [decode-fn (get decoders type)]
+              (if (or (= type :cookie) (= type :bearer))
+                (let [metadata (tokens/decode-header token)]
+                  ;; NOTE: we only proceed to decode claims on new
+                  ;; cookie tokens. The old cookies dont need to be
+                  ;; decoded because they use the token string as ID
+                  (if (and (= (:kid metadata) 1)
+                           (= (:ver metadata) 1)
+                           (some? decode-fn))
+                    (assoc request ::http/auth-data (assoc auth
+                                                           :claims (decode-fn token)
+                                                           :metadata metadata))
+                    (assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
+
+                (if decode-fn
+                  (assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
+                  (assoc request ::http/auth-data auth))))
+
+            request))]
+
+    (fn [request]
+      (-> request process-request handler))))
+
+(def auth
+  {:name ::auth
+   :compile (constantly wrap-auth)})
+
+(defn- wrap-shared-key-auth
+  [handler shared-key]
+  (if shared-key
+    (let [shared-key (if (string? shared-key)
+                       shared-key
+                       (bc/bytes->b64-str shared-key true))]
+      (fn [request]
+        (let [key (yreq/get-header request "x-shared-key")]
+          (if (= key shared-key)
+            (handler (assoc request ::http/auth-with-shared-key true))
+            {::yres/status 403}))))
+    (fn [_ _]
+      {::yres/status 403})))
+
+(def shared-key-auth
+  {:name ::shared-key-auth
+   :compile (constantly wrap-shared-key-auth)})

backend/src/app/http/security.clj

@@ -0,0 +1,55 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.http.security
+  "Additional security layer middlewares"
+  (:require
+   [app.config :as cf]
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
+
+(def ^:private safe-methods
+  #{:get :head :options})
+
+(defn- wrap-sec-fetch-metadata
+  "Sec-Fetch metadata security layer middleware"
+  [handler]
+  (fn [request]
+    (let [site (yreq/get-header request "sec-fetch-site")]
+      (cond
+        (= site "same-origin")
+        (handler request)
+
+        (or (= site "same-site")
+            (= site "cross-site"))
+        (if (contains? safe-methods (yreq/method request))
+          (handler request)
+          {::yres/status 403})
+
+        :else
+        (handler request)))))
+
+(def sec-fetch-metadata
+  {:name ::sec-fetch-metadata
+   :compile (fn [_ _]
+              (when (contains? cf/flags :sec-fetch-metadata-middleware)
+                wrap-sec-fetch-metadata))})
+
+(defn- wrap-client-header-check
+  "Check for a penpot custom header to be present as additional CSRF
+  protection"
+  [handler]
+  (fn [request]
+    (let [client (yreq/get-header request "x-client")]
+      (if (some? client)
+        (handler request)
+        {::yres/status 403}))))
+
+(def client-header-check
+  {:name ::client-header-check
+   :compile (fn [_ _]
+              (when (contains? cf/flags :client-header-check-middleware)
+                wrap-client-header-check))})

backend/src/app/http/session.clj

@@ -10,45 +10,40 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
-   [app.common.uri :as u]
+   [app.common.time :as ct]
+   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.http :as-alias http]
+   [app.http.auth :as-alias http.auth]
    [app.http.session.tasks :as-alias tasks]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [app.util.time :as dt]
-   [cuerdas.core :as str]
    [integrant.core :as ig]
-   [yetti.request :as yreq]))
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; A default cookie name for storing the session.
-(def default-auth-token-cookie-name "auth-token")
-
-;; A cookie that we can use to check from other sites of the same
-;; domain if a user is authenticated.
-(def default-auth-data-cookie-name "auth-data")
-
 ;; Default value for cookie max-age
-(def default-cookie-max-age (dt/duration {:days 7}))
+(def default-cookie-max-age (ct/duration {:days 7}))
 
 ;; Default age for automatic session renewal
-(def default-renewal-max-age (dt/duration {:hours 6}))
+(def default-renewal-max-age (ct/duration {:hours 6}))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PROTOCOLS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol ISessionManager
-  (read [_ key])
+  (read-session [_ id])
-  (write! [_ key data])
+  (create-session [_ params])
-  (update! [_ data])
+  (update-session [_ session])
-  (delete! [_ key]))
+  (delete-session [_ id]))
 
 (defn manager?
   [o]
@@ -63,67 +58,82 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private schema:params
-  [:map {:title "session-params"}
+  [:map {:title "SessionParams" :closed true}
-   [:user-agent ::sm/text]
    [:profile-id ::sm/uuid]
-   [:created-at ::sm/inst]])
+   [:user-agent {:optional true} ::sm/text]
+   [:sso-provider-id {:optional true} ::sm/uuid]
+   [:sso-session-id {:optional true} :string]])
 
 (def ^:private valid-params?
   (sm/validator schema:params))
 
-(defn- prepare-session-params
-  [key params]
-  (assert (string? key) "expected key to be a string")
-  (assert (not (str/blank? key)) "expected key to be not empty")
-  (assert (valid-params? params) "expected valid params")
-
-  {:user-agent (:user-agent params)
-   :profile-id (:profile-id params)
-   :created-at (:created-at params)
-   :updated-at (:created-at params)
-   :id key})
-
 (defn- database-manager
   [pool]
   (reify ISessionManager
-    (read [_ token]
+    (read-session [_ id]
-      (db/exec-one! pool (sql/select :http-session {:id token})))
+      (if (string? id)
+        ;; Backward compatibility
+        (let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
+          (-> session
+              (assoc :modified-at (:updated-at session))
+              (dissoc :updated-at)))
+        (db/exec-one! pool (sql/select :http-session-v2 {:id id}))))
 
-    (write! [_ key params]
+    (create-session [_ params]
-      (let [params (prepare-session-params key params)]
+      (assert (valid-params? params) "expect valid session params")
-        (db/insert! pool :http-session params)
-        params))
 
-    (update! [_ params]
+      (let [now    (ct/now)
-      (let [updated-at (dt/now)]
+            params (-> params
-        (db/update! pool :http-session
+                       (assoc :id (uuid/next))
-                    {:updated-at updated-at}
+                       (assoc :created-at now)
-                    {:id (:id params)})
+                       (assoc :modified-at now))]
-        (assoc params :updated-at updated-at)))
+        (db/insert! pool :http-session-v2 params
+                    {::db/return-keys true})))
 
-    (delete! [_ token]
+    (update-session [_ session]
-      (db/delete! pool :http-session {:id token})
+      (let [modified-at (ct/now)]
+        (if (string? (:id session))
+          (db/insert! pool :http-session-v2
+                      (-> session
+                          (assoc :id (uuid/next))
+                          (assoc :created-at modified-at)
+                          (assoc :modified-at modified-at)))
+          (db/update! pool :http-session-v2
+                      {:modified-at modified-at}
+                      {:id (:id session)}
+                      {::db/return-keys true}))))
+
+    (delete-session [_ id]
+      (if (string? id)
+        (db/delete! pool :http-session {:id id} {::db/return-keys false})
+        (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
       nil)))
 
 (defn inmemory-manager
   []
   (let [cache (atom {})]
     (reify ISessionManager
-      (read [_ token]
+      (read-session [_ id]
-        (get @cache token))
+        (get @cache id))
 
-      (write! [_ key params]
+      (create-session [_ params]
-        (let [params (prepare-session-params key params)]
+        (assert (valid-params? params) "expect valid session params")
-          (swap! cache assoc key params)
-          params))
 
-      (update! [_ params]
+        (let [now     (ct/now)
-        (let [updated-at (dt/now)]
+              session (-> params
-          (swap! cache update (:id params) assoc :updated-at updated-at)
+                          (assoc :id (uuid/next))
-          (assoc params :updated-at updated-at)))
+                          (assoc :created-at now)
+                          (assoc :modified-at now))]
+          (swap! cache assoc (:id session) session)
+          session))
 
-      (delete! [_ token]
+      (update-session [_ session]
-        (swap! cache dissoc token)
+        (let [modified-at (ct/now)]
+          (swap! cache update (:id session) assoc :modified-at modified-at)
+          (assoc session :modified-at modified-at)))
+
+      (delete-session [_ id]
+        (swap! cache dissoc id)
         nil))))
 
 (defmethod ig/assert-key ::manager
@@ -143,109 +153,116 @@
 ;; MANAGER IMPL
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare ^:private assign-auth-token-cookie)
+(declare ^:private assign-session-cookie)
-(declare ^:private assign-auth-data-cookie)
+(declare ^:private clear-session-cookie)
-(declare ^:private clear-auth-token-cookie)
+
-(declare ^:private clear-auth-data-cookie)
+(defn- assign-token
-(declare ^:private gen-token)
+  [cfg session]
+  (let [claims {:iss "authentication"
+                :aud "penpot"
+                :sid (:id session)
+                :iat (:modified-at session)
+                :uid (:profile-id session)
+                :sso-provider-id (:sso-provider-id session)
+                :sso-session-id (:sso-session-id session)}
+        header {:kid 1 :ver 1}
+        token  (tokens/generate cfg claims header)]
+    (assoc session :token token)))
 
 (defn create-fn
-  [{:keys [::manager ::setup/props]} profile-id]
+  [{:keys [::manager] :as cfg} {profile-id :id :as profile}
+   & {:keys [sso-provider-id sso-session-id]}]
+
   (assert (manager? manager) "expected valid session manager")
   (assert (uuid? profile-id) "expected valid uuid for profile-id")
 
   (fn [request response]
     (let [uagent  (yreq/get-header request "user-agent")
-          params  {:profile-id profile-id
+          session (->> {:user-agent uagent
-                   :user-agent uagent
+                        :profile-id profile-id
-                   :created-at (dt/now)}
+                        :sso-provider-id sso-provider-id
-          token   (gen-token props params)
+                        :sso-session-id sso-session-id}
-          session (write! manager token params)]
+                       (d/without-nils)
-      (l/trace :hint "create" :profile-id (str profile-id))
+                       (create-session manager)
-      (-> response
+                       (assign-token cfg))]
-          (assign-auth-token-cookie session)
+
-          (assign-auth-data-cookie session)))))
+      (l/trc :hint "create" :id (str (:id session)) :profile-id (str profile-id))
+      (assign-session-cookie response session))))
 
 (defn delete-fn
   [{:keys [::manager]}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request response]
-    (let [cname   (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+    (some->> (get request ::id) (delete-session manager))
-          cookie  (yreq/get-cookie request cname)]
+    (clear-session-cookie response)))
-      (l/trace :hint "delete" :profile-id (:profile-id request))
-      (some->> (:value cookie) (delete! manager))
-      (-> response
-          (assoc :status 204)
-          (assoc :body nil)
-          (clear-auth-token-cookie)
-          (clear-auth-data-cookie)))))
 
-(defn- gen-token
+(defn decode-token
-  [props {:keys [profile-id created-at]}]
+  [cfg token]
-  (tokens/generate props {:iss "authentication"
+  (try
-                          :iat created-at
+    (tokens/verify cfg {:token token :iss "authentication"})
-                          :uid profile-id}))
+    (catch Throwable cause
-(defn- decode-token
+      (l/trc :hint "exception on decoding token"
-  [props token]
+             :token token
-  (when token
+             :cause cause))))
-    (tokens/verify props {:token token :iss "authentication"})))
 
-(defn- get-token
+(defn get-session
   [request]
-  (let [cname  (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+  (get request ::session))
-        cookie (some-> (yreq/get-cookie request cname) :value)]
-    (when-not (str/empty? cookie)
-      cookie)))
 
-(defn- get-session
+(defn invalidate-others
-  [manager token]
+  [cfg session]
-  (some->> token (read manager)))
+  (let [sql "delete from http_session_v2 where profile_id = ? and id != ?"]
+    (-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
+        (db/get-update-count))))
 
 (defn- renew-session?
-  [{:keys [updated-at] :as session}]
+  [{:keys [id modified-at] :as session}]
-  (and (dt/instant? updated-at)
+  (or (string? id)
-       (let [elapsed (dt/diff updated-at (dt/now))]
+      (and (ct/inst? modified-at)
-         (neg? (compare default-renewal-max-age elapsed)))))
+           (let [elapsed (ct/diff modified-at (ct/now))]
-
+             (neg? (compare default-renewal-max-age elapsed))))))
-(defn- wrap-soft-auth
-  [handler {:keys [::manager ::setup/props]}]
-  (assert (manager? manager) "expected valid session manager")
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token props token)]
-                (cond-> request
-                  (map? claims)
-                  (-> (assoc ::token-claims claims)
-                      (assoc ::token token))))
-              (catch Throwable cause
-                (l/trace :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
 
 (defn- wrap-authz
-  [handler {:keys [::manager]}]
+  [handler {:keys [::manager] :as cfg}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request]
-    (let [session  (get-session manager (::token request))
+    (let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
-          request  (cond-> request
+      (cond
-                     (some? session)
+        (= type :cookie)
-                     (assoc ::profile-id (:profile-id session)
+        (let [session (case (:ver metadata)
-                            ::id (:id session)))
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
-          response (handler request)]
+                        0 (read-session manager token)
+                        1 (some->> (:sid claims) (read-session manager))
+                        nil)
 
-      (if (renew-session? session)
+              request (cond-> request
-        (let [session (update! manager session)]
+                        (some? session)
-          (-> response
+                        (-> (assoc ::profile-id (:profile-id session))
-              (assign-auth-token-cookie session)
+                            (assoc ::session session)))
-              (assign-auth-data-cookie session)))
-        response))))
 
-(def soft-auth
+              response (handler request)]
-  {:name ::soft-auth
+
-   :compile (constantly wrap-soft-auth)})
+          (if (and session (renew-session? session))
+            (let [session (->> session
+                               (update-session manager)
+                               (assign-token cfg))]
+              (assign-session-cookie response session))
+            response))
+
+        (= type :bearer)
+        (let [session (case (:ver metadata)
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
+                        0 (read-session manager token)
+                        1 (some->> (:sid claims) (read-session manager))
+                        nil)
+              request (cond-> request
+                        (some? session)
+                        (-> (assoc ::profile-id (:profile-id session))
+                            (assoc ::session session)))]
+          (handler request))
+
+        :else
+        (handler request)))))
 
 (def authz
   {:name ::authz
@@ -253,17 +270,17 @@
 
 ;; --- IMPL
 
-(defn- assign-auth-token-cookie
+(defn- assign-session-cookie
-  [response {token :id updated-at :updated-at}]
+  [response {token :token modified-at :modified-at}]
   (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        created-at (or updated-at (dt/now))
+        created-at modified-at
-        renewal    (dt/plus created-at default-renewal-max-age)
+        renewal    (ct/plus created-at default-renewal-max-age)
-        expires    (dt/plus created-at max-age)
+        expires    (ct/plus created-at max-age)
         secure?    (contains? cf/flags :secure-session-cookies)
         strict?    (contains? cf/flags :strict-session-cookies)
         cors?      (contains? cf/flags :cors)
-        name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+        name       (cf/get :auth-token-cookie-name)
-        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
+        comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
         cookie     {:path "/"
                     :http-only true
                     :expires expires
@@ -271,59 +288,21 @@
                     :comment comment
                     :same-site (if cors? :none (if strict? :strict :lax))
                     :secure secure?}]
-    (update response :cookies assoc name cookie)))
+    (update response ::yres/cookies assoc name cookie)))
 
-(defn- assign-auth-data-cookie
+(defn- clear-session-cookie
-  [response {profile-id :profile-id updated-at :updated-at}]
-  (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        domain     (cf/get :auth-data-cookie-domain)
-        cname      default-auth-data-cookie-name
-
-        created-at (or updated-at (dt/now))
-        renewal    (dt/plus created-at default-renewal-max-age)
-        expires    (dt/plus created-at max-age)
-
-        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
-        secure?    (contains? cf/flags :secure-session-cookies)
-        strict?    (contains? cf/flags :strict-session-cookies)
-        cors?      (contains? cf/flags :cors)
-
-        cookie     {:domain domain
-                    :expires expires
-                    :path "/"
-                    :comment comment
-                    :value (u/map->query-string {:profile-id profile-id})
-                    :same-site (if cors? :none (if strict? :strict :lax))
-                    :secure secure?}]
-
-    (cond-> response
-      (string? domain)
-      (update :cookies assoc cname cookie))))
-
-(defn- clear-auth-token-cookie
   [response]
-  (let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
+  (let [cname (cf/get :auth-token-cookie-name)]
-    (update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
+    (update response ::yres/cookies assoc cname {:path "/" :value "" :max-age 0})))
-
-(defn- clear-auth-data-cookie
-  [response]
-  (let [cname  default-auth-data-cookie-name
-        domain (cf/get :auth-data-cookie-domain)]
-    (cond-> response
-      (string? domain)
-      (update :cookies assoc cname {:domain domain :path "/" :value "" :max-age 0}))))
-
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASK: SESSION GC
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; FIXME: MOVE
-
 (defmethod ig/assert-key ::tasks/gc
   [_ params]
   (assert (db/pool? (::db/pool params)) "expected valid database pool")
-  (assert (dt/duration? (::tasks/max-age params))))
+  (assert (ct/duration? (::tasks/max-age params))))
 
 (defmethod ig/expand-key ::tasks/gc
   [k v]
@@ -332,22 +311,23 @@
 
 (def ^:private
   sql:delete-expired
-  "delete from http_session
+  "DELETE FROM http_session
-    where updated_at < now() - ?::interval
+    WHERE updated_at < ?::timestamptz
        or (updated_at is null and
-           created_at < now() - ?::interval)")
+           created_at < ?::timestamptz)")
 
 (defn- collect-expired-tasks
   [{:keys [::db/conn ::tasks/max-age]}]
-  (let [interval (db/interval max-age)
+  (let [threshold (ct/minus (ct/now) max-age)
-        result   (db/exec-one! conn [sql:delete-expired interval interval])
+        result    (-> (db/exec-one! conn [sql:delete-expired threshold threshold])
-        result   (:next.jdbc/update-count result)]
+                      (db/get-update-count))]
-    (l/debug :task "gc"
+    (l/dbg :task "gc"
-             :hint "clean http sessions"
+           :hint "clean http sessions"
-             :deleted result)
+           :deleted result)
     result))
 
 (defmethod ig/init-key ::tasks/gc
   [_ {:keys [::tasks/max-age] :as cfg}]
-  (l/debug :hint "initializing session gc task" :max-age max-age)
+  (l/dbg :hint "initializing session gc task" :max-age max-age)
-  (fn [_] (db/tx-run! cfg collect-expired-tasks)))
+  (fn [_]
+    (db/tx-run! cfg collect-expired-tasks)))

backend/src/app/http/sse.clj

@@ -33,7 +33,7 @@
                  (println "event:" (d/name name))
                  (println "data:" (t/encode-str data {:type :json-verbose}))
                  (println))]
-      (.getBytes data "UTF-8"))
+      (.getBytes ^String data "UTF-8"))
     (catch Throwable cause
       (l/err :hint "unexpected error on encoding value on sse stream"
              :cause cause)
@@ -44,7 +44,8 @@
 (def default-headers
   {"Content-Type" "text/event-stream;charset=UTF-8"
    "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
-   "Pragma" "no-cache"})
+   "Pragma" "no-cache"
+   "X-Accel-Buffering" "no"})
 
 (defn response
   [handler & {:keys [buf] :or {buf 32} :as opts}]
@@ -54,7 +55,7 @@
      ::yres/body (yres/stream-body
                   (fn [_ output]
                     (let [channel  (sp/chan :buf buf :xf (keep encode))
-                          listener (events/start-listener
+                          listener (events/spawn-listener
                                     channel
                                     (partial write! output)
                                     (partial pu/close! output))]

backend/src/app/http/websocket.clj

@@ -11,12 +11,12 @@
    [app.common.logging :as l]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.http.session :as session]
    [app.metrics :as mtx]
    [app.msgbus :as mbus]
-   [app.util.time :as dt]
    [app.util.websocket :as ws]
    [integrant.core :as ig]
    [promesa.exec.csp :as sp]
@@ -239,7 +239,7 @@
 
 (defn- on-connect
   [{:keys [::mtx/metrics]} {:keys [::ws/id] :as wsp}]
-  (let [created-at (dt/now)]
+  (let [created-at (ct/now)]
     (l/trace :fn "on-connect" :conn-id id)
     (swap! state assoc id wsp)
     (mtx/run! metrics
@@ -253,7 +253,7 @@
              (mtx/run! metrics :id :websocket-active-connections :dec 1)
              (mtx/run! metrics
                        :id :websocket-session-timing
-                       :val (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0))))))
+                       :val (/ (inst-ms (ct/diff created-at (ct/now))) 1000.0))))))
 
 (defn- on-rcv-message
   [{:keys [::mtx/metrics ::profile-id ::session-id]} message]

backend/src/app/loggers/audit.clj

@@ -11,6 +11,7 @@
    [app.common.data.macros :as dm]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -23,9 +24,9 @@
    [app.setup :as-alias setup]
    [app.util.inet :as inet]
    [app.util.services :as-alias sv]
-   [app.util.time :as dt]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [yetti.request :as yreq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -78,17 +79,32 @@
          (remove #(contains? reserved-props (key %))))
         props))
 
-(defn event-from-rpc-params
+(defn get-external-session-id
-  "Create a base event skeleton with pre-filled some important
+  [request]
-  data that can be extracted from RPC params object"
+  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-  [params]
+    (when-not (or (> (count session-id) 256)
-  (let [context {:external-session-id (::rpc/external-session-id params)
+                  (= session-id "null")
-                 :external-event-origin (::rpc/external-event-origin params)
+                  (str/blank? session-id))
-                 :triggered-by (::rpc/handler-name params)}]
+      session-id)))
-    {::type "action"
+
-     ::profile-id (::rpc/profile-id params)
+(defn- get-client-event-origin
-     ::ip-addr (::rpc/ip-addr params)
+  [request]
-     ::context (d/without-nils context)}))
+  (when-let [origin (yreq/get-header request "x-event-origin")]
+    (when-not (or (= origin "null")
+                  (str/blank? origin))
+      (str/prune origin 200))))
+
+(defn get-client-user-agent
+  [request]
+  (when-let [user-agent (yreq/get-header request "user-agent")]
+    (str/prune user-agent 500)))
+
+(defn- get-client-version
+  [request]
+  (when-let [origin (yreq/get-header request "x-frontend-version")]
+    (when-not (or (= origin "null")
+                  (str/blank? origin))
+      (str/prune origin 100))))
 
 ;; --- SPECS
 
@@ -108,15 +124,42 @@
    [::ip-addr {:optional true} ::sm/text]
    [::props {:optional true} [:map-of :keyword :any]]
    [::context {:optional true} [:map-of :keyword :any]]
-   [::tracked-at {:optional true} ::sm/inst]
+   [::tracked-at {:optional true} ::ct/inst]
    [::webhooks/event? {:optional true} ::sm/boolean]
-   [::webhooks/batch-timeout {:optional true} ::dt/duration]
+   [::webhooks/batch-timeout {:optional true} ::ct/duration]
    [::webhooks/batch-key {:optional true}
     [:or ::sm/fn ::sm/text :keyword]]])
 
 (def ^:private check-event
   (sm/check-fn schema:event))
 
+(defn- prepare-context-from-request
+  [request]
+  (let [client-event-origin (get-client-event-origin request)
+        client-version      (get-client-version request)
+        client-user-agent   (get-client-user-agent request)
+        session-id          (get-external-session-id request)
+        token-id       (::actoken/id request)]
+    (d/without-nils
+     {:external-session-id session-id
+      :access-token-id (some-> token-id str)
+      :client-event-origin client-event-origin
+      :client-user-agent client-user-agent
+      :client-version client-version
+      :version (:full cf/version)})))
+
+(defn event-from-rpc-params
+  "Create a base event skeleton with pre-filled some important
+  data that can be extracted from RPC params object"
+  [params]
+  (let [context (some-> params meta ::http/request prepare-context-from-request)
+        event   {::type "action"
+                 ::profile-id (or (::rpc/profile-id params) uuid/zero)
+                 ::ip-addr (::rpc/ip-addr params)}]
+    (cond-> event
+      (some? context)
+      (assoc ::context context))))
+
 (defn prepare-event
   [cfg mdata params result]
   (let [resultm      (meta result)
@@ -126,23 +169,15 @@
                          (::rpc/profile-id params)
                          uuid/zero)
 
-        session-id   (get params ::rpc/external-session-id)
-        event-origin (get params ::rpc/external-event-origin)
         props        (-> (or (::replace-props resultm)
                              (-> params
                                  (merge (::props resultm))
                                  (dissoc :profile-id)
                                  (dissoc :type)))
-
                          (clean-props))
 
-        token-id     (::actoken/id request)
+        context      (merge (::context resultm)
-        context      (-> (::context resultm)
+                            (prepare-context-from-request request))
-                         (assoc :external-session-id session-id)
-                         (assoc :external-event-origin event-origin)
-                         (assoc :access-token-id (some-> token-id str))
-                         (d/without-nils))
-
         ip-addr      (inet/parse-request request)]
 
     {::type (or (::type resultm)
@@ -199,7 +234,7 @@
 (defn- handle-event!
   [cfg event]
   (let [params (event->params event)
-        tnow   (dt/now)]
+        tnow   (ct/now)]
 
     (when (contains? cf/flags :audit-log)
       ;; NOTE: this operation may cause primary key conflicts on inserts
@@ -273,7 +308,7 @@
     (let [event (-> (d/without-nils event)
                     (check-event))]
       (db/run! cfg (fn [cfg]
-                     (let [tnow   (dt/now)
+                     (let [tnow   (ct/now)
                            params (-> (event->params event)
                                       (assoc :created-at tnow)
                                       (update :tracked-at #(or % tnow)))]

backend/src/app/loggers/audit/archive_task.clj

@@ -16,7 +16,6 @@
    [app.http.client :as http]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [app.util.time :as dt]
    [integrant.core :as ig]
    [lambdaisland.uri :as u]
    [promesa.exec :as px]))
@@ -53,13 +52,12 @@
 
 (defn- send!
   [{:keys [::uri] :as cfg} events]
-  (let [token   (tokens/generate (::setup/props cfg)
+  (let [token   (tokens/generate cfg
                                  {:iss "authentication"
-                                  :iat (dt/now)
                                   :uid uuid/zero})
         body    (t/encode {:events events})
         headers {"content-type" "application/transit+json"
-                 "origin" (cf/get :public-uri)
+                 "origin" (str (cf/get :public-uri))
                  "cookie" (u/map->query-string {:auth-token token})}
         params  {:uri uri
                  :timeout 12000

backend/src/app/loggers/database.clj

@@ -41,7 +41,7 @@
   (if (or (instance? java.util.concurrent.CompletionException cause)
           (instance? java.util.concurrent.ExecutionException cause))
     (-> record
-        (assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
+        (assoc ::trace (ex/format-throwable cause :data? true :explain? false :header? false :summary? false))
         (assoc ::l/cause (ex-cause cause))
         (record->report))
 
@@ -49,7 +49,7 @@
           ctx  (-> context
                    (assoc :tenant (cf/get :tenant))
                    (assoc :host (cf/get :host))
-                   (assoc :public-uri (cf/get :public-uri))
+                   (assoc :public-uri (str (cf/get :public-uri)))
                    (assoc :logger/name logger)
                    (assoc :logger/level level)
                    (dissoc :request/params :value :params :data))]
@@ -64,18 +64,18 @@
                          message))
                      @message)
         :trace   (or (::trace record)
-                     (some-> cause (ex/format-throwable :data? false :explain? false :header? false :summary? false)))}
+                     (some-> cause (ex/format-throwable :data? true :explain? false :header? false :summary? false)))}
 
        (when-let [params (or (:request/params context) (:params context))]
-         {:params (pp/pprint-str params :length 30 :level 13)})
+         {:params (pp/pprint-str params :length 20 :level 20)})
 
        (when-let [value (:value context)]
-         {:value (pp/pprint-str value :length 30 :level 12)})
+         {:value (pp/pprint-str value :length 30 :level 13)})
 
        (when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
-         {:data (pp/pprint-str data :length 30 :level 12)})
+         {:data (pp/pprint-str data :length 30 :level 13)})
 
-       (when-let [explain (ex/explain data :length 30 :level 12)]
+       (when-let [explain (ex/explain data :length 30 :level 13)]
          {:explain explain})))))
 
 (defn error-record?

backend/src/app/loggers/webhooks.clj

@@ -10,13 +10,13 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.logging :as l]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uri :as uri]
    [app.config :as cf]
    [app.db :as db]
    [app.http.client :as http]
    [app.loggers.audit :as audit]
-   [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.data.json :as json]
    [cuerdas.core :as str]
@@ -124,7 +124,7 @@
                               {:id (:id whook)})))
 
               (db/update! pool :webhook
-                          {:updated-at (dt/now)
+                          {:updated-at (ct/now)
                            :error-code nil
                            :error-count 0}
                           {:id (:id whook)})))
@@ -132,7 +132,7 @@
           (report-delivery! [whook req rsp err]
             (db/insert! pool :webhook-delivery
                         {:webhook-id (:id whook)
-                         :created-at (dt/now)
+                         :created-at (ct/now)
                          :error-code err
                          :req-data (db/tjson req)
                          :rsp-data (db/tjson rsp)}))]
@@ -155,7 +155,7 @@
         (let [req {:uri (:uri whook)
                    :headers {"content-type" (:mtype whook)
                              "user-agent" (str/ffmt "penpot/%" (:main cf/version))}
-                   :timeout (dt/duration "4s")
+                   :timeout (ct/duration "4s")
                    :method :post
                    :body body}]
           (try

backend/src/app/main.clj

@@ -11,6 +11,7 @@
    [app.auth.oidc.providers :as-alias oidc.providers]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
+   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
    [app.email :as-alias email]
@@ -19,7 +20,8 @@
    [app.http.awsns :as http.awsns]
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
-   [app.http.session :as-alias session]
+   [app.http.management :as mgmt]
+   [app.http.session :as session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
    [app.loggers.webhooks :as-alias webhooks]
@@ -29,7 +31,6 @@
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [app.srepl :as-alias srepl]
    [app.storage :as-alias sto]
@@ -38,8 +39,9 @@
    [app.storage.gc-touched :as-alias sto.gc-touched]
    [app.storage.s3 :as-alias sto.s3]
    [app.svgo :as-alias svgo]
-   [app.util.time :as dt]
+   [app.util.cron]
    [app.worker :as-alias wrk]
+   [app.worker.executor]
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
@@ -146,23 +148,11 @@
     ::mdef/labels []
     ::mdef/type :histogram}
 
-   :executors-active-threads
+   :http-server-dispatch-timing
-   {::mdef/name "penpot_executors_active_threads"
+   {::mdef/name "penpot_http_server_dispatch_timing"
-    ::mdef/help "Current number of threads available in the executor service."
+    ::mdef/help "Histogram of dispatch handler"
-    ::mdef/labels ["name"]
+    ::mdef/labels []
-    ::mdef/type :gauge}
+    ::mdef/type :histogram}})
-
-   :executors-completed-tasks
-   {::mdef/name "penpot_executors_completed_tasks_total"
-    ::mdef/help "Approximate number of completed tasks by the executor."
-    ::mdef/labels ["name"]
-    ::mdef/type :counter}
-
-   :executors-running-threads
-   {::mdef/name "penpot_executors_running_threads"
-    ::mdef/help "Current number of threads with state RUNNING."
-    ::mdef/labels ["name"]
-    ::mdef/type :gauge}})
 
 (def system-config
   {::db/pool
@@ -174,14 +164,12 @@
     ::db/max-size   (cf/get :database-max-pool-size 60)
     ::mtx/metrics   (ig/ref ::mtx/metrics)}
 
-   ;; Default thread pool for IO operations
+   ;; Default netty IO pool (shared between several services)
-   ::wrk/executor
+   ::wrk/netty-io-executor
-   {}
+   {:threads (cf/get :netty-io-threads)}
 
-   ::wrk/monitor
+   ::wrk/netty-executor
-   {::mtx/metrics  (ig/ref ::mtx/metrics)
+   {:threads (cf/get :executor-threads)}
-    ::wrk/executor (ig/ref ::wrk/executor)
-    ::wrk/name     "default"}
 
    :app.migrations/migrations
    {::db/pool (ig/ref ::db/pool)}
@@ -192,17 +180,27 @@
    ::mtx/routes
    {::mtx/metrics (ig/ref ::mtx/metrics)}
 
-   ::rds/redis
+   ::rds/client
-   {::rds/uri      (cf/get :redis-uri)
+   {::rds/uri
-    ::mtx/metrics  (ig/ref ::mtx/metrics)
+    (cf/get :redis-uri)
-    ::wrk/executor (ig/ref ::wrk/executor)}
+
+    ::wrk/netty-executor
+    (ig/ref ::wrk/netty-executor)
+
+    ::wrk/netty-io-executor
+    (ig/ref ::wrk/netty-io-executor)}
+
+   ::rds/pool
+   {::rds/client  (ig/ref ::rds/client)
+    ::mtx/metrics (ig/ref ::mtx/metrics)}
 
    ::mbus/msgbus
-   {::wrk/executor  (ig/ref ::wrk/executor)
+   {::wrk/executor (ig/ref ::wrk/netty-executor)
-    ::rds/redis     (ig/ref ::rds/redis)}
+    ::rds/client   (ig/ref ::rds/client)
+    ::mtx/metrics  (ig/ref ::mtx/metrics)}
 
    :app.storage.tmp/cleaner
-   {::wrk/executor (ig/ref ::wrk/executor)}
+   {::wrk/executor (ig/ref ::wrk/netty-executor)}
 
    ::sto.gc-deleted/handler
    {::db/pool      (ig/ref ::db/pool)
@@ -230,8 +228,10 @@
     ::http/host                    (cf/get :http-server-host)
     ::http/router                  (ig/ref ::http/router)
     ::http/io-threads              (cf/get :http-server-io-threads)
+    ::http/max-worker-threads      (cf/get :http-server-max-worker-threads)
     ::http/max-body-size           (cf/get :http-server-max-body-size)
-    ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
+    ::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)
+    ::mtx/metrics                  (ig/ref ::mtx/metrics)}
 
    ::ldap/provider
    {:host           (cf/get :ldap-host)
@@ -259,26 +259,33 @@
    ::oidc.providers/generic
    {::http.client/client (ig/ref ::http.client/client)}
 
+   ::oidc/providers
+   [(ig/ref ::oidc.providers/google)
+    (ig/ref ::oidc.providers/github)
+    (ig/ref ::oidc.providers/gitlab)
+    (ig/ref ::oidc.providers/generic)]
+
    ::oidc/routes
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
     ::setup/props        (ig/ref ::setup/props)
-    ::oidc/providers     {:google (ig/ref ::oidc.providers/google)
+    ::oidc/providers     (ig/ref ::oidc/providers)
-                          :github (ig/ref ::oidc.providers/github)
-                          :gitlab (ig/ref ::oidc.providers/gitlab)
-                          :oidc   (ig/ref ::oidc.providers/generic)}
     ::session/manager    (ig/ref ::session/manager)
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
+   ::mgmt/routes
+   {::db/pool            (ig/ref ::db/pool)
+    ::setup/props        (ig/ref ::setup/props)}
+
    :app.http/router
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
     ::rpc/routes         (ig/ref ::rpc/routes)
-    ::rpc.doc/routes     (ig/ref ::rpc.doc/routes)
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
+    ::mgmt/routes        (ig/ref ::mgmt/routes)
     ::http.debug/routes  (ig/ref ::http.debug/routes)
     ::http.assets/routes (ig/ref ::http.assets/routes)
     ::http.ws/routes     (ig/ref ::http.ws/routes)
@@ -294,33 +301,35 @@
    {::db/pool         (ig/ref ::db/pool)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::mbus/msgbus     (ig/ref ::mbus/msgbus)
+    ::setup/props     (ig/ref ::setup/props)
     ::session/manager (ig/ref ::session/manager)}
 
    :app.http.assets/routes
    {::http.assets/path  (cf/get :assets-path)
-    ::http.assets/cache-max-age (dt/duration {:hours 24})
+    ::http.assets/cache-max-age (ct/duration {:hours 24})
-    ::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
+    ::http.assets/cache-max-agesignature-max-age (ct/duration {:hours 24 :minutes 5})
     ::sto/storage  (ig/ref ::sto/storage)}
 
    ::rpc/climit
    {::mtx/metrics        (ig/ref ::mtx/metrics)
-    ::wrk/executor       (ig/ref ::wrk/executor)
+    ::wrk/executor       (ig/ref ::wrk/netty-executor)
     ::climit/config      (cf/get :rpc-climit-config)
     ::climit/enabled     (contains? cf/flags :rpc-climit)}
 
    :app.rpc/rlimit
-   {::wrk/executor (ig/ref ::wrk/executor)}
+   {::wrk/executor (ig/ref ::wrk/netty-executor)}
 
    :app.rpc/methods
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
-    ::wrk/executor       (ig/ref ::wrk/executor)
+    ::rds/pool           (ig/ref ::rds/pool)
+    ::wrk/executor       (ig/ref ::wrk/netty-executor)
     ::session/manager    (ig/ref ::session/manager)
     ::ldap/provider      (ig/ref ::ldap/provider)
     ::sto/storage        (ig/ref ::sto/storage)
     ::mtx/metrics        (ig/ref ::mtx/metrics)
     ::mbus/msgbus        (ig/ref ::mbus/msgbus)
-    ::rds/redis          (ig/ref ::rds/redis)
+    ::rds/client         (ig/ref ::rds/client)
 
     ::rpc/climit         (ig/ref ::rpc/climit)
     ::rpc/rlimit         (ig/ref ::rpc/rlimit)
@@ -330,14 +339,26 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   :app.rpc.doc/routes
+   :app.rpc/management-methods
-   {:app.rpc/methods (ig/ref :app.rpc/methods)}
+   {::http.client/client (ig/ref ::http.client/client)
+    ::db/pool            (ig/ref ::db/pool)
+    ::rds/pool           (ig/ref ::rds/pool)
+    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::session/manager    (ig/ref ::session/manager)
+    ::sto/storage        (ig/ref ::sto/storage)
+    ::mtx/metrics        (ig/ref ::mtx/metrics)
+    ::mbus/msgbus        (ig/ref ::mbus/msgbus)
+    ::rds/client         (ig/ref ::rds/client)
+    ::setup/props        (ig/ref ::setup/props)}
 
-   :app.rpc/routes
+   ::rpc/routes
-   {::rpc/methods     (ig/ref :app.rpc/methods)
+   {::rpc/methods        (ig/ref :app.rpc/methods)
-    ::db/pool         (ig/ref ::db/pool)
+    ::rpc/management-methods (ig/ref :app.rpc/management-methods)
-    ::session/manager (ig/ref ::session/manager)
+
-    ::setup/props     (ig/ref ::setup/props)}
+    ;; FIXME: revisit if db/pool is necessary here
+    ::db/pool                (ig/ref ::db/pool)
+    ::session/manager        (ig/ref ::session/manager)
+    ::setup/props            (ig/ref ::setup/props)}
 
    ::wrk/registry
    {::mtx/metrics (ig/ref ::mtx/metrics)
@@ -425,6 +446,9 @@
     ;; module requires the migrations to run before initialize.
     ::migrations (ig/ref :app.migrations/migrations)}
 
+   ::setup/clock
+   {}
+
    :app.loggers.audit.archive-task/handler
    {::setup/props        (ig/ref ::setup/props)
     ::db/pool            (ig/ref ::db/pool)
@@ -468,49 +492,50 @@
                             (cf/get :objects-storage-s3-bucket))
     ::sto.s3/io-threads (or (cf/get :storage-assets-s3-io-threads)
                             (cf/get :objects-storage-s3-io-threads))
-    ::wrk/executor      (ig/ref ::wrk/executor)}
+
+    ::wrk/netty-io-executor
+    (ig/ref ::wrk/netty-io-executor)}
 
    :app.storage.fs/backend
    {::sto.fs/directory (or (cf/get :storage-assets-fs-directory)
                            (cf/get :objects-storage-fs-directory))}})
 
-
 (def worker-config
   {::wrk/cron
    {::wrk/registry            (ig/ref ::wrk/registry)
     ::db/pool                 (ig/ref ::db/pool)
     ::wrk/entries
-    [{:cron #app/cron "0 0 0 * * ?" ;; daily
+    [{:cron #penpot/cron "0 0 0 * * ?" ;; daily
       :task :session-gc}
 
-     {:cron #app/cron "0 0 0 * * ?" ;; daily
+     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
       :task :objects-gc}
 
-     {:cron #app/cron "0 0 0 * * ?" ;; daily
+     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
       :task :storage-gc-deleted}
 
-     {:cron #app/cron "0 0 0 * * ?" ;; daily
+     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
       :task :storage-gc-touched}
 
-     {:cron #app/cron "0 0 0 * * ?" ;; daily
+     {:cron #penpot/cron "0 0 0 * * ?" ;; daily
       :task :tasks-gc}
 
-     {:cron #app/cron "0 0 2 * * ?" ;; daily
+     {:cron #penpot/cron "0 0 2 * * ?" ;; daily
       :task :file-gc-scheduler}
 
-     {:cron #app/cron "0 30 */3,23 * * ?"
+     {:cron #penpot/cron "0 30 */3,23 * * ?"
       :task :telemetry}
 
      (when (contains? cf/flags :audit-log-archive)
-       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
+       {:cron #penpot/cron "0 */5 * * * ?" ;; every 5m
         :task :audit-log-archive})
 
      (when (contains? cf/flags :audit-log-gc)
-       {:cron #app/cron "30 */5 * * * ?" ;; every 5m
+       {:cron #penpot/cron "30 */5 * * * ?" ;; every 5m
         :task :audit-log-gc})]}
 
    ::wrk/dispatcher
-   {::rds/redis   (ig/ref ::rds/redis)
+   {::rds/client  (ig/ref ::rds/client)
     ::mtx/metrics (ig/ref ::mtx/metrics)
     ::db/pool     (ig/ref ::db/pool)
     ::wrk/tenant  (cf/get :tenant)}
@@ -519,7 +544,7 @@
    {::wrk/parallelism (cf/get ::worker-default-parallelism 1)
     ::wrk/queue       :default
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/redis       (ig/ref ::rds/redis)
+    ::rds/client      (ig/ref ::rds/client)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}
@@ -528,7 +553,7 @@
    {::wrk/parallelism (cf/get ::worker-webhook-parallelism 1)
     ::wrk/queue       :webhooks
     ::wrk/tenant      (cf/get :tenant)
-    ::rds/redis       (ig/ref ::rds/redis)
+    ::rds/client      (ig/ref ::rds/client)
     ::wrk/registry    (ig/ref ::wrk/registry)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::db/pool         (ig/ref ::db/pool)}})

backend/src/app/media.clj

@@ -10,17 +10,20 @@
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.schema.openapi :as-alias oapi]
+   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
+   [app.http.client :as http]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
-   [app.util.time :as dt]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.shell :as sh]
+   [clojure.string]
    [clojure.xml :as xml]
    [cuerdas.core :as str]
    [datoteka.fs :as fs]
@@ -32,19 +35,19 @@
    javax.xml.parsers.SAXParserFactory
    org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
-   org.im4java.core.IMOperation
+   org.im4java.core.IMOperation))
-   org.im4java.core.Info))
+
+(def default-max-file-size
+  (* 1024 1024 10)) ; 10 MiB
 
 (def schema:upload
-  (sm/register!
+  [:map {:title "Upload"}
-   ^{::sm/type ::upload}
+   [:filename :string]
-   [:map {:title "Upload"}
+   [:size ::sm/int]
-    [:filename :string]
+   [:path ::fs/path]
-    [:size ::sm/int]
+   [:mtype {:optional true} :string]
-    [:path ::fs/path]
+   [:headers {:optional true}
-    [:mtype {:optional true} :string]
+    [:map-of :string :string]]])
-    [:headers {:optional true}
-     [:map-of :string :string]]]))
 
 (def ^:private schema:input
   [:map {:title "Input"}
@@ -116,7 +119,7 @@
 (defn- parse-svg
   [text]
   (let [text (strip-doctype text)]
-    (dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
+    (dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
       (xml/parse istream secure-parser-factory))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -215,6 +218,24 @@
                  {:width (int width)
                   :height (int height)})))]))
 
+(defn- get-dimensions-with-orientation [^String path]
+  ;; Image magick doesn't give info about exif rotation so we use the identify command
+  ;; If we are processing an animated gif we use the first frame with -scene 0
+  (let [dim-result (sh/sh "identify" "-format" "%w %h\n" path)
+        orient-result (sh/sh "identify" "-format" "%[EXIF:Orientation]\n" path)]
+    (when (= 0 (:exit dim-result))
+      (let [[w h] (-> (:out dim-result)
+                      str/trim
+                      (clojure.string/split #"\s+")
+                      (->> (mapv #(Integer/parseInt %))))
+            orientation-exit (:exit orient-result)
+            orientation      (-> orient-result :out str/trim)]
+        (if (= 0 orientation-exit)
+          (case orientation
+            ("6" "8") {:width h :height w} ; Rotated 90 or 270 degrees
+            {:width w :height h})          ; Normal or unknown orientation
+          {:width w :height h})))))        ; If orientation can't be read, use dimensions as-is
+
 (defmethod process :info
   [{:keys [input] :as params}]
   (let [{:keys [path mtype] :as input} (check-input input)]
@@ -224,23 +245,39 @@
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (merge input info {:ts (dt/now)}))
+        (merge input info {:ts (ct/now) :size (fs/size path)}))
 
-      (let [instance (Info. (str path))
+      (let [path-str      (str path)
-            mtype'   (.getProperty instance "Mime type")]
+            identify-res  (sh/sh "identify" "-format" "image/%[magick]\n" path-str)
+            ;; identify prints one line per frame (animated GIFs, etc.); we take the first one
+            mtype'        (if (zero? (:exit identify-res))
+                            (-> identify-res
+                                :out
+                                str/trim
+                                (str/split #"\s+" 2)
+                                first
+                                str/lower)
+                            (ex/raise :type :validation
+                                      :code :invalid-image
+                                      :hint "invalid image"))
+            {:keys [width height]}
+            (or (get-dimensions-with-orientation path-str)
+                (do
+                  (l/warn "Failed to read image dimensions with orientation" {:path path})
+                  (ex/raise :type :validation
+                            :code :invalid-image
+                            :hint "invalid image")))]
         (when (and (string? mtype)
-                   (not= mtype mtype'))
+                   (not= (str/lower mtype) mtype'))
           (ex/raise :type :validation
                     :code :media-type-mismatch
                     :hint (str "Seems like you are uploading a file whose content does not match the extension."
                                "Expected: " mtype ". Got: " mtype')))
-        ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
-        ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
-        ;; any frame.
         (assoc input
-               :width  (.getPageWidth instance)
+               :width  width
-               :height (.getPageHeight instance)
+               :height height
-               :ts (dt/now))))))
+               :size (fs/size path)
+               :ts (ct/now))))))
 
 (defmethod process-error org.im4java.core.InfoException
   [error]
@@ -249,6 +286,54 @@
             :hint "invalid image"
             :cause error))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMAGE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn download-image
+  "Download an image from the provided URI and return the media input object"
+  [{:keys [::http/client]} uri]
+  (letfn [(parse-and-validate [{:keys [headers] :as response}]
+            (let [size     (some-> (get headers "content-length") d/parse-integer)
+                  mtype    (get headers "content-type")
+                  format   (cm/mtype->format mtype)
+                  max-size (cf/get :media-max-file-size default-max-file-size)]
+
+              (when-not size
+                (ex/raise :type :validation
+                          :code :unknown-size
+                          :hint "seems like the url points to resource with unknown size"))
+
+              (when (> size max-size)
+                (ex/raise :type :validation
+                          :code :file-too-large
+                          :hint (str/ffmt "the file size % is greater than the maximum %"
+                                          size
+                                          default-max-file-size)))
+
+              (when (nil? format)
+                (ex/raise :type :validation
+                          :code :media-type-not-allowed
+                          :hint "seems like the url points to an invalid media object"))
+
+              {:size size :mtype mtype :format format}))]
+
+    (let [{:keys [body] :as response} (http/req! client
+                                                 {:method :get :uri uri}
+                                                 {:response-type :input-stream})
+          {:keys [size mtype]} (parse-and-validate response)
+          path    (tmp/tempfile :prefix "penpot.media.download.")
+          written (io/write* path body :size size)]
+
+      (when (not= written size)
+        (ex/raise :type :internal
+                  :code :mismatch-write-size
+                  :hint "unexpected state: unable to write to file"))
+
+      {;; :size size
+       :path path
+       :mtype mtype})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FONTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/migrations.clj

@@ -438,7 +438,25 @@
     :fn (mg/resource "app/migrations/sql/0138-mod-file-data-fragment-table.sql")}
 
    {:name "0139-mod-file-change-table.sql"
-    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}
+
+   {:name "0140-mod-file-change-table.sql"
+    :fn (mg/resource "app/migrations/sql/0140-mod-file-change-table.sql")}
+
+   {:name "0140-add-locked-by-column-to-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}
+
+   {:name "0141-add-idx-to-file-library-rel"
+    :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
+
+   {:name "0141-add-file-data-table.sql"
+    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}
+
+   {:name "0142-add-sso-provider-table"
+    :fn (mg/resource "app/migrations/sql/0142-add-sso-provider-table.sql")}
+
+   {:name "0143-http-session-v2-table"
+    :fn (mg/resource "app/migrations/sql/0143-add-http-session-v2-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/media_refs.clj

@@ -10,8 +10,8 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.pprint]
-   [app.srepl.fixes.media-refs :refer [process-file]]
    [app.srepl.main :as srepl]
+   [app.srepl.procs.media-refs]
    [clojure.edn :as edn]))
 
 (def ^:private required-services
@@ -20,7 +20,10 @@
    :app.storage/storage
    :app.metrics/metrics
    :app.db/pool
-   :app.worker/executor])
+   :app.worker/netty-io-executor])
+
+(def default-options
+  {:rollback? false})
 
 (defn -main
   [& [options]]
@@ -28,22 +31,20 @@
     (let [config-var (requiring-resolve 'app.main/system-config)
           start-var  (requiring-resolve 'app.main/start-custom)
           stop-var   (requiring-resolve 'app.main/stop)
-          config     (select-keys @config-var required-services)]
+
+          config     (select-keys @config-var required-services)
+          options    (if (string? options)
+                       (ex/ignoring (edn/read-string options))
+                       {})
+          options    (-> (merge default-options options)
+                         (assoc :proc-fn #'app.srepl.procs.media-refs/fix-media-refs))]
 
       (start-var config)
-
+      (l/inf :hint "executing media-refs migration" :options options)
-      (let [options (if (string? options)
+      (srepl/process! options)
-                      (ex/ignoring (edn/read-string options))
-                      {})]
-
-        (l/inf :hint "executing media-refs migration" :options options)
-        (srepl/process-files! process-file options))
-
       (stop-var)
       (System/exit 0))
     (catch Throwable cause
       (ex/print-throwable cause)
       (flush)
       (System/exit -1))))
-
-

backend/src/app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql

@@ -0,0 +1,11 @@
+-- Add locked_by column to file_change table for version locking feature
+-- This allows users to lock their own saved versions to prevent deletion by others
+
+ALTER TABLE file_change
+  ADD COLUMN locked_by uuid NULL REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;
+
+-- Create index for locked versions queries
+CREATE INDEX file_change__locked_by__idx ON file_change (locked_by) WHERE locked_by IS NOT NULL;
+
+-- Add comment for documentation
+COMMENT ON COLUMN file_change.locked_by IS 'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.';

backend/src/app/migrations/sql/0140-mod-file-change-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE file_change
+ ADD COLUMN migrations text[];

backend/src/app/migrations/sql/0141-add-file-data-table.sql

@@ -0,0 +1,38 @@
+CREATE TABLE file_data (
+  file_id uuid NOT NULL REFERENCES file(id) DEFERRABLE,
+  id uuid NOT NULL,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+  deleted_at timestamptz NULL,
+
+  type text NOT NULL,
+  backend text NULL,
+
+  metadata jsonb NULL,
+  data bytea NULL,
+
+  PRIMARY KEY (file_id, id)
+
+) PARTITION BY HASH (file_id);
+
+CREATE TABLE file_data_00 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 0);
+CREATE TABLE file_data_01 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 1);
+CREATE TABLE file_data_02 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 2);
+CREATE TABLE file_data_03 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 3);
+CREATE TABLE file_data_04 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 4);
+CREATE TABLE file_data_05 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 5);
+CREATE TABLE file_data_06 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 6);
+CREATE TABLE file_data_07 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 7);
+CREATE TABLE file_data_08 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 8);
+CREATE TABLE file_data_09 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 9);
+CREATE TABLE file_data_10 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 10);
+CREATE TABLE file_data_11 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 11);
+CREATE TABLE file_data_12 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 12);
+CREATE TABLE file_data_13 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 13);
+CREATE TABLE file_data_14 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 14);
+CREATE TABLE file_data_15 PARTITION OF file_data FOR VALUES WITH (MODULUS 16, REMAINDER 15);
+
+CREATE INDEX file_data__deleted_at__idx
+    ON file_data (deleted_at, file_id, id)
+ WHERE deleted_at IS NOT NULL;

backend/src/app/migrations/sql/0141-add-idx-to-file-library-rel.sql

@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS file_library_rel__library_file_id__idx
+    ON file_library_rel (library_file_id);

backend/src/app/migrations/sql/0142-add-sso-provider-table.sql

@@ -0,0 +1,33 @@
+CREATE TABLE sso_provider (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  is_enabled boolean NOT NULL DEFAULT true,
+
+  type text NOT NULL CHECK (type IN ('oidc')),
+  domain text NOT NULL,
+
+  client_id text NOT NULL,
+  client_secret text NOT NULL,
+
+  base_uri text NOT NULL,
+  token_uri text NULL,
+  auth_uri text NULL,
+  user_uri text NULL,
+  jwks_uri text NULL,
+  logout_uri text NULL,
+
+  roles_attr text NULL,
+  email_attr text NULL,
+  name_attr text NULL,
+  user_info_source text NOT NULL DEFAULT 'token'
+     CHECK (user_info_source IN ('token', 'userinfo', 'auto')),
+
+  scopes text[] NULL,
+  roles text[] NULL
+);
+
+CREATE UNIQUE INDEX sso_provider__domain__idx
+    ON sso_provider(domain);

backend/src/app/migrations/sql/0143-add-http-session-v2-table.sql

@@ -0,0 +1,23 @@
+CREATE TABLE http_session_v2 (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  profile_id uuid REFERENCES profile(id) ON DELETE CASCADE,
+  user_agent text NULL,
+
+  sso_provider_id uuid NULL REFERENCES sso_provider(id) ON DELETE CASCADE,
+  sso_session_id text NULL
+);
+
+CREATE INDEX http_session_v2__profile_id__idx
+    ON http_session_v2(profile_id);
+
+CREATE INDEX http_session_v2__sso_provider_id__idx
+    ON http_session_v2(sso_provider_id)
+ WHERE sso_provider_id IS NOT NULL;
+
+CREATE INDEX http_session_v2__sso_session_id__idx
+    ON http_session_v2(sso_session_id)
+ WHERE sso_session_id IS NOT NULL;

backend/src/app/msgbus.clj

@@ -10,13 +10,12 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.config :as cfg]
    [app.redis :as rds]
-   [app.util.time :as dt]
    [app.worker :as wrk]
    [integrant.core :as ig]
-   [promesa.core :as p]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]))
 
@@ -56,17 +55,19 @@
   [k v]
   {k (-> (d/without-nils v)
          (assoc ::buffer-size 128)
-         (assoc ::timeout (dt/duration {:seconds 30})))})
+         (assoc ::timeout (ct/duration {:seconds 30})))})
 
 (def ^:private schema:params
-  [:map ::rds/redis ::wrk/executor])
+  [:map
+   ::rds/client
+   ::wrk/executor])
 
 (defmethod ig/assert-key ::msgbus
   [_ params]
   (assert (sm/check schema:params params)))
 
 (defmethod ig/init-key ::msgbus
-  [_ {:keys [::buffer-size ::wrk/executor ::timeout ::rds/redis] :as cfg}]
+  [_ {:keys [::buffer-size ::wrk/executor ::timeout] :as cfg}]
   (l/info :hint "initialize msgbus" :buffer-size buffer-size)
   (let [cmd-ch (sp/chan :buf buffer-size)
         rcv-ch (sp/chan :buf (sp/dropping-buffer buffer-size))
@@ -74,8 +75,9 @@
                         :xf  xform-prefix-topic)
         state  (agent {})
 
-        pconn  (rds/connect redis :type :default :timeout timeout)
+        ;; Open persistent connections to redis
-        sconn  (rds/connect redis :type :pubsub :timeout timeout)
+        pconn  (rds/connect cfg :timeout timeout)
+        sconn  (rds/connect-pubsub cfg :timeout timeout)
 
         _      (set-error-handler! state #(l/error :cause % :hint "unexpected error on agent" ::l/sync? true))
         _      (set-error-mode! state :continue)
@@ -189,14 +191,13 @@
 
 (defn- create-listener
   [rcv-ch]
-  (rds/pubsub-listener
+  {:on-message (fn [_ topic message]
-   :on-message (fn [_ topic message]
                  ;; There are no back pressure, so we use a slidding
                  ;; buffer for cases when the pubsub broker sends
                  ;; more messages that we can process.
-                 (let [val {:topic topic :message (t/decode message)}]
+                 (let [val {:topic topic :message (t/decode-str message)}]
                    (when-not (sp/offer! rcv-ch val)
-                     (l/warn :msg "dropping message on subscription loop"))))))
+                     (l/warn :msg "dropping message on subscription loop"))))})
 
 (defn- process-input
   [{:keys [::state ::wrk/executor] :as cfg} topic message]
@@ -216,8 +217,7 @@
   (rds/add-listener sconn (create-listener rcv-ch))
 
   (px/thread
-    {:name "penpot/msgbus/io-loop"
+    {:name "penpot/msgbus"}
-     :virtual true}
     (try
       (loop []
         (let [timeout-ch (sp/timeout-chan 1000)
@@ -263,7 +263,7 @@
   intended to be used in core.async go blocks."
   [{:keys [::pconn] :as cfg} {:keys [topic message]}]
   (try
-    (p/await! (rds/publish pconn topic (t/encode message)))
+    (rds/publish pconn topic (t/encode-str message))
     (catch InterruptedException cause
       (throw cause))
     (catch Throwable cause

backend/src/app/redis.clj

@@ -6,23 +6,22 @@
 
 (ns app.redis
   "The msgbus abstraction implemented using redis as underlying backend."
-  (:refer-clojure :exclude [eval])
+  (:refer-clojure :exclude [eval get set run!])
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.generic-pool :as gpool]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.metrics :as mtx]
    [app.redis.script :as-alias rscript]
-   [app.util.cache :as cache]
+   [app.worker :as wrk]
-   [app.util.time :as dt]
+   [app.worker.executor]
-   [app.worker :as-alias wrk]
    [clojure.core :as c]
    [clojure.java.io :as io]
    [cuerdas.core :as str]
-   [integrant.core :as ig]
+   [integrant.core :as ig])
-   [promesa.core :as p]
-   [promesa.exec :as px])
   (:import
    clojure.lang.MapEntry
    io.lettuce.core.KeyValue
@@ -32,12 +31,10 @@
    io.lettuce.core.RedisException
    io.lettuce.core.RedisURI
    io.lettuce.core.ScriptOutputType
-   io.lettuce.core.api.StatefulConnection
+   io.lettuce.core.SetArgs
    io.lettuce.core.api.StatefulRedisConnection
-   io.lettuce.core.api.async.RedisAsyncCommands
-   io.lettuce.core.api.async.RedisScriptingAsyncCommands
    io.lettuce.core.api.sync.RedisCommands
-   io.lettuce.core.codec.ByteArrayCodec
+   io.lettuce.core.api.sync.RedisScriptingCommands
    io.lettuce.core.codec.RedisCodec
    io.lettuce.core.codec.StringCodec
    io.lettuce.core.pubsub.RedisPubSubListener
@@ -45,244 +42,238 @@
    io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
    io.lettuce.core.resource.ClientResources
    io.lettuce.core.resource.DefaultClientResources
+   io.netty.channel.nio.NioEventLoopGroup
    io.netty.util.HashedWheelTimer
    io.netty.util.Timer
+   io.netty.util.concurrent.EventExecutorGroup
    java.lang.AutoCloseable
    java.time.Duration))
 
 (set! *warn-on-reflection* true)
 
-(declare ^:private initialize-resources)
+(def ^:const MAX-EVAL-RETRIES 18)
-(declare ^:private shutdown-resources)
-(declare ^:private impl-eval)
 
-(defprotocol IRedis
+(def default-timeout
-  (-connect [_ options])
+  (ct/duration "10s"))
-  (-get-or-connect [_ key options]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMPL & PRIVATE API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol IConnection
-  (publish [_ topic message])
+  (-set-timeout [_ timeout] "set connection timeout")
-  (rpush [_ key payload])
+  (-get-timeout [_] "get current timeout")
-  (blpop [_ timeout keys])
+  (-reset-timeout [_] "reset to default timeout"))
-  (eval [_ script]))
+
+(defprotocol IDefaultConnection
+  "Public API of default redis connection"
+  (-publish [_ topic message])
+  (-rpush [_ key payload])
+  (-blpop [_ timeout keys])
+  (-eval [_ script])
+  (-get [_ key])
+  (-set [_ key val args])
+  (-del [_ key-or-keys])
+  (-ping [_]))
 
 (defprotocol IPubSubConnection
-  (add-listener [_ listener])
+  (-add-listener [_ listener])
-  (subscribe [_ topics])
+  (-subscribe [_ topics])
-  (unsubscribe [_ topics]))
+  (-unsubscribe [_ topics]))
 
-(def default-codec
+(def ^:private default-codec
-  (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE))
-
-(def string-codec
   (RedisCodec/of StringCodec/UTF8 StringCodec/UTF8))
 
-(sm/register!
+(defn- impl-eval
- {:type ::connection
+  [cmd cache metrics script]
-  :pred #(satisfies? IConnection %)
+  (let [keys    (into-array String (map str (::rscript/keys script)))
-  :type-properties
+        vals    (into-array String (map str (::rscript/vals script)))
-  {:title "connection"
+        sname   (::rscript/name script)
-   :description "redis connection instance"}})
 
-(sm/register!
+        read-script
- {:type ::pubsub-connection
+        (fn []
-  :pred #(satisfies? IPubSubConnection %)
+          (-> script ::rscript/path io/resource slurp))
-  :type-properties
-  {:title "connection"
-   :description "redis connection instance"}})
 
-(defn redis?
+        load-script
-  [o]
+        (fn []
-  (satisfies? IRedis o))
+          (let [id (.scriptLoad ^RedisScriptingCommands cmd
+                                ^String (read-script))]
+            (swap! cache assoc sname id)
+            (l/trc :hint "load script" :name sname :id id)
 
-(sm/register!
+            id))
- {:type ::redis
-  :pred redis?})
 
-(def ^:private schema:script
+        eval-script
-  [:map {:title "script"}
+        (fn [id]
-   [::rscript/name qualified-keyword?]
+          (try
-   [::rscript/path ::sm/text]
+            (let [tpoint  (ct/tpoint)
-   [::rscript/keys {:optional true} [:vector :any]]
+                  result  (.evalsha ^RedisScriptingCommands cmd
-   [::rscript/vals {:optional true} [:vector :any]]])
+                                    ^String id
+                                    ^ScriptOutputType ScriptOutputType/MULTI
+                                    ^"[Ljava.lang.String;" keys
+                                    ^"[Ljava.lang.String;" vals)
+                  elapsed (tpoint)]
 
-(def valid-script?
+              (mtx/run! metrics {:id :redis-eval-timing
-  (sm/lazy-validator schema:script))
+                                 :labels [(name sname)]
+                                 :val (inst-ms elapsed)})
 
-(defmethod ig/expand-key ::redis
+              (l/trc :hint "eval script"
-  [k v]
+                     :name (name sname)
-  (let [cpus    (px/get-available-processors)
+                     :id id
-        threads (max 1 (int (* cpus 0.2)))]
+                     :params (str/join "," (::rscript/vals script))
-    {k (-> (d/without-nils v)
+                     :elapsed (ct/format-duration elapsed))
-           (assoc ::timeout (dt/duration "10s"))
-           (assoc ::io-threads (max 3 threads))
-           (assoc ::worker-threads (max 3 threads)))}))
 
-(def ^:private schema:redis-params
+              result)
-  [:map {:title "redis-params"}
-   ::wrk/executor
-   ::mtx/metrics
-   [::uri ::sm/uri]
-   [::worker-threads ::sm/int]
-   [::io-threads ::sm/int]
-   [::timeout ::dt/duration]])
 
-(defmethod ig/assert-key ::redis
+            (catch io.lettuce.core.RedisNoScriptException _cause
-  [_ params]
+              ::load)
-  (assert (sm/check schema:redis-params params)))
 
-(defmethod ig/init-key ::redis
+            (catch Throwable cause
-  [_ params]
+              (when-let [on-error (::rscript/on-error script)]
-  (initialize-resources params))
+                (on-error cause))
+              (throw cause))))
 
-(defmethod ig/halt-key! ::redis
+        eval-script'
-  [_ instance]
+        (fn [id]
-  (d/close! instance))
+          (loop [id      id
+                 retries 0]
+            (if (> retries MAX-EVAL-RETRIES)
+              (ex/raise :type :internal
+                        :code ::max-eval-retries-reached
+                        :hint (str "unable to eval redis script " sname))
+              (let [result (eval-script id)]
+                (if (= result ::load)
+                  (recur (load-script)
+                         (inc retries))
+                  result)))))]
 
-(defn- initialize-resources
+    (if-let [id (c/get @cache sname)]
-  "Initialize redis connection resources"
+      (eval-script' id)
-  [{:keys [::uri ::io-threads ::worker-threads ::wrk/executor ::mtx/metrics] :as params}]
+      (-> (load-script)
+          (eval-script')))))
 
-  (l/inf :hint "initialize redis resources"
+(deftype Connection [^StatefulRedisConnection conn
-         :uri (str uri)
+                     ^RedisCommands cmd
-         :io-threads io-threads
+                     ^Duration timeout
-         :worker-threads worker-threads)
+                     cache metrics]
+  AutoCloseable
+  (close [_]
+    (ex/ignoring (.close conn)))
 
-  (let [timer     (HashedWheelTimer.)
+  IConnection
-        resources (.. (DefaultClientResources/builder)
+  (-set-timeout [_ timeout]
-                      (ioThreadPoolSize ^long io-threads)
+    (.setTimeout conn ^Duration timeout))
-                      (computationThreadPoolSize ^long worker-threads)
-                      (timer ^Timer timer)
-                      (build))
 
-        redis-uri (RedisURI/create ^String (str uri))
+  (-reset-timeout [_]
+    (.setTimeout conn timeout))
 
-        shutdown  (fn [client conn]
+  (-get-timeout [_]
-                    (ex/ignoring (.close ^StatefulConnection conn))
+    (.getTimeout conn))
-                    (ex/ignoring (.close ^RedisClient client))
-                    (l/trc :hint "disconnect" :hid (hash client)))
 
-        on-remove (fn [key val cause]
+  IDefaultConnection
-                    (l/trace :hint "evict connection (cache)" :key key :reason cause)
+  (-publish [_ topic message]
-                    (some-> val d/close!))
+    (.publish cmd ^String topic ^String message))
 
-        cache     (cache/create :executor executor
+  (-rpush [_ key elements]
-                                :on-remove on-remove
+    (try
-                                :keepalive "5m")]
+      (let [vals (make-array String (count elements))]
-    (reify
+        (loop [i 0 xs (seq elements)]
-      java.lang.AutoCloseable
+          (when xs
-      (close [_]
+            (aset ^"[[Ljava.lang.String;" vals i ^String (first xs))
-        (ex/ignoring (cache/invalidate! cache))
+            (recur (inc i) (next xs))))
-        (ex/ignoring (.shutdown ^ClientResources resources))
-        (ex/ignoring (.stop ^Timer timer)))
 
-      IRedis
+        (.rpush cmd
-      (-get-or-connect [this key options]
+                ^String key
-        (let [create (fn [_] (-connect this options))]
+                ^"[[Ljava.lang.String;" vals))
-          (cache/get cache key create)))
 
-      (-connect [_ options]
+      (catch RedisCommandInterruptedException cause
-        (let [timeout (or (:timeout options) (::timeout params))
+        (throw (InterruptedException. (ex-message cause))))))
-              codec   (get options :codec default-codec)
-              type    (get options :type :default)
-              client  (RedisClient/create ^ClientResources resources
-                                          ^RedisURI redis-uri)]
 
-          (l/trc :hint "connect" :hid (hash client))
+  (-blpop [_ keys timeout]
-          (if (= type :pubsub)
+    (try
-            (let [conn (.connectPubSub ^RedisClient client
+      (let [keys (into-array String keys)]
-                                       ^RedisCodec codec)]
+        (when-let [res (.blpop cmd
-              (.setTimeout ^StatefulConnection conn
+                               ^double timeout
-                           ^Duration timeout)
+                               ^"[Ljava.lang.String;" keys)]
-              (reify
+          (MapEntry/create
-                IPubSubConnection
+           (.getKey ^KeyValue res)
-                (add-listener [_ listener]
+           (.getValue ^KeyValue res))))
-                  (assert (instance? RedisPubSubListener listener) "expected listener instance")
+      (catch RedisCommandInterruptedException cause
-                  (.addListener ^StatefulRedisPubSubConnection conn
+        (throw (InterruptedException. (ex-message cause))))))
-                                ^RedisPubSubListener listener))
 
-                (subscribe [_ topics]
+  (-get [_ key]
-                  (try
+    (assert (string? key) "key expected to be string")
-                    (let [topics (into-array String (map str topics))
+    (.get cmd ^String key))
-                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
-                      (.subscribe ^RedisPubSubCommands cmd topics))
-                    (catch RedisCommandInterruptedException cause
-                      (throw (InterruptedException. (ex-message cause))))))
 
-                (unsubscribe [_ topics]
+  (-set [_ key val args]
-                  (try
+    (.set cmd
-                    (let [topics (into-array String (map str topics))
+          ^String key
-                          cmd    (.sync ^StatefulRedisPubSubConnection conn)]
+          ^bytes val
-                      (.unsubscribe ^RedisPubSubCommands cmd topics))
+          ^SetArgs args))
-                    (catch RedisCommandInterruptedException cause
+
-                      (throw (InterruptedException. (ex-message cause))))))
+  (-del [_ keys]
+    (let [keys (into-array String keys)]
+      (.del cmd ^String/1 keys)))
+
+  (-ping [_]
+    (.ping cmd))
+
+  (-eval [_ script]
+    (impl-eval cmd cache metrics script)))
 
 
-                AutoCloseable
+(deftype SubscriptionConnection [^StatefulRedisPubSubConnection conn
-                (close [_] (shutdown client conn))))
+                                 ^RedisPubSubCommands cmd
+                                 ^Duration timeout]
+  AutoCloseable
+  (close [_]
+    (ex/ignoring (.close conn)))
 
-            (let [conn (.connect ^RedisClient client ^RedisCodec codec)]
+  IConnection
-              (.setTimeout ^StatefulConnection conn ^Duration timeout)
+  (-set-timeout [_ timeout]
-              (reify
+    (.setTimeout conn ^Duration timeout))
-                IConnection
-                (publish [_ topic message]
-                  (assert (string? topic) "expected topic to be string")
-                  (assert (bytes? message) "expected message to be a byte array")
 
-                  (let [pcomm (.async ^StatefulRedisConnection conn)]
+  (-reset-timeout [_]
-                    (.publish ^RedisAsyncCommands pcomm ^String topic ^bytes message)))
+    (.setTimeout conn timeout))
 
-                (rpush [_ key payload]
+  (-get-timeout [_]
-                  (assert (or (and (vector? payload)
+    (.getTimeout conn))
-                                   (every? bytes? payload))
-                              (bytes? payload)))
-                  (try
-                    (let [cmd  (.sync ^StatefulRedisConnection conn)
-                          data (if (vector? payload) payload [payload])
-                          vals (make-array (. Class (forName "[B")) (count data))]
 
-                      (loop [i 0 xs (seq data)]
+  IPubSubConnection
-                        (when xs
+  (-add-listener [_ listener]
-                          (aset ^"[[B" vals i ^bytes (first xs))
+    (.addListener conn ^RedisPubSubListener listener))
-                          (recur (inc i) (next xs))))
 
-                      (.rpush ^RedisCommands cmd
+  (-subscribe [_ topics]
-                              ^String key
+    (try
-                              ^"[[B" vals))
+      (let [topics (into-array String topics)]
+        (.subscribe cmd topics))
+      (catch RedisCommandInterruptedException cause
+        (throw (InterruptedException. (ex-message cause))))))
 
-                    (catch RedisCommandInterruptedException cause
+  (-unsubscribe [_ topics]
-                      (throw (InterruptedException. (ex-message cause))))))
+    (try
+      (let [topics (into-array String topics)]
+        (.unsubscribe cmd topics))
+      (catch RedisCommandInterruptedException cause
+        (throw (InterruptedException. (ex-message cause)))))))
 
-                (blpop [_ timeout keys]
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-                  (try
+;; PUBLIC API
-                    (let [keys    (into-array Object (map str keys))
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-                          cmd     (.sync ^StatefulRedisConnection conn)
-                          timeout (/ (double (inst-ms timeout)) 1000.0)]
-                      (when-let [res (.blpop ^RedisCommands cmd
-                                             ^double timeout
-                                             ^"[Ljava.lang.String;" keys)]
-                        (MapEntry/create
-                         (.getKey ^KeyValue res)
-                         (.getValue ^KeyValue res))))
-                    (catch RedisCommandInterruptedException cause
-                      (throw (InterruptedException. (ex-message cause))))))
 
-                (eval [_ script]
+(defn build-set-args
-                  (assert (valid-script? script) "expected valid script")
+  [options]
-                  (impl-eval conn metrics script))
+  (reduce-kv (fn [^SetArgs args k v]
-
+               (case k
-                AutoCloseable
+                 :ex (if (instance? Duration v)
-                (close [_] (shutdown client conn))))))))))
+                       (.ex args ^Duration v)
-
+                       (.ex args (long v)))
-(defn connect
+                 :px (.px args (long v))
-  [instance & {:as opts}]
+                 :nx (if v (.nx args) args)
-  (assert (satisfies? IRedis instance) "expected valid redis instance")
+                 :keep-ttl (if v (.keepttl args) args)))
-  (-connect instance opts))
+             (SetArgs.)
-
+             options))
-(defn get-or-connect
-  [instance key & {:as opts}]
-  (assert (satisfies? IRedis instance) "expected valid redis instance")
-  (-get-or-connect instance key opts))
 
 (defn pubsub-listener
   [& {:keys [on-message on-subscribe on-unsubscribe]}]
@@ -311,61 +302,172 @@
       (when on-unsubscribe
         (on-unsubscribe nil topic count)))))
 
-(def ^:private scripts-cache (atom {}))
+(defn connect
+  [cfg & {:as options}]
+  (assert (contains? cfg ::mtx/metrics) "missing ::mtx/metrics on provided system")
+  (assert (contains? cfg ::client) "missing ::rds/client on provided system")
 
-(defn- impl-eval
+  (let [state   (::client cfg)
-  [^StatefulRedisConnection connection metrics script]
-  (let [cmd   (.async ^StatefulRedisConnection connection)
-        keys  (into-array String (map str (::rscript/keys script)))
-        vals  (into-array String (map str (::rscript/vals script)))
-        sname (::rscript/name script)]
 
-    (letfn [(on-error [cause]
+        cache   (::cache state)
-              (if (instance? io.lettuce.core.RedisNoScriptException cause)
+        client  (::client state)
-                (do
+        timeout (or (some-> (:timeout options) ct/duration)
-                  (l/error :hint "no script found" :name sname :cause cause)
+                    (::timeout state))
-                  (->> (load-script)
-                       (p/mcat eval-script)))
-                (if-let [on-error (::rscript/on-error script)]
-                  (on-error cause)
-                  (p/rejected cause))))
 
-            (eval-script [sha]
+        conn    (.connect ^RedisClient client
-              (let [tpoint (dt/tpoint)]
+                          ^RedisCodec default-codec)
-                (->> (.evalsha ^RedisScriptingAsyncCommands cmd
+        cmd     (.sync ^StatefulRedisConnection conn)]
-                               ^String sha
-                               ^ScriptOutputType ScriptOutputType/MULTI
-                               ^"[Ljava.lang.String;" keys
-                               ^"[Ljava.lang.String;" vals)
-                     (p/fmap (fn [result]
-                               (let [elapsed (tpoint)]
-                                 (mtx/run! metrics {:id :redis-eval-timing
-                                                    :labels [(name sname)]
-                                                    :val (inst-ms elapsed)})
-                                 (l/trace :hint "eval script"
-                                          :name (name sname)
-                                          :sha sha
-                                          :params (str/join "," (::rscript/vals script))
-                                          :elapsed (dt/format-duration elapsed))
-                                 result)))
-                     (p/merr on-error))))
 
-            (read-script []
+    (.setTimeout ^StatefulRedisConnection conn ^Duration timeout)
-              (-> script ::rscript/path io/resource slurp))
+    (->Connection conn cmd timeout cache (::mtx/metrics cfg))))
 
-            (load-script []
+(defn connect-pubsub
-              (l/trace :hint "load script" :name sname)
+  [cfg & {:as options}]
-              (->> (.scriptLoad ^RedisScriptingAsyncCommands cmd
+  (let [state   (::client cfg)
-                                ^String (read-script))
+        client  (::client state)
-                   (p/fmap (fn [sha]
-                             (swap! scripts-cache assoc sname sha)
-                             sha))))]
 
-      (p/await!
+        timeout (or (some-> (:timeout options) ct/duration)
-       (if-let [sha (get @scripts-cache sname)]
+                    (::timeout state))
-         (eval-script sha)
+        conn    (.connectPubSub ^RedisClient client
-         (->> (load-script)
+                                ^RedisCodec default-codec)
-              (p/mapcat eval-script)))))))
+        cmd     (.sync ^StatefulRedisPubSubConnection conn)]
+
+
+    (.setTimeout ^StatefulRedisPubSubConnection conn
+                 ^Duration timeout)
+    (->SubscriptionConnection conn cmd timeout)))
+
+(defn get
+  [conn key]
+  (assert (string? key) "key must be string instance")
+  (try
+    (-get conn key)
+    (catch RedisCommandTimeoutException cause
+      (l/err :hint "timeout on get redis key" :key key :cause cause)
+      nil)))
+
+(defn set
+  ([conn key val]
+   (set conn key val nil))
+  ([conn key val args]
+   (assert (string? key) "key must be string instance")
+   (assert (string? val) "val must be string instance")
+   (let [args (cond
+                (or (instance? SetArgs args)
+                    (nil? args))
+                args
+
+                (map? args)
+                (build-set-args args)
+
+                :else
+                (throw (IllegalArgumentException. "invalid args")))]
+
+     (try
+       (-set conn key val args)
+       (catch RedisCommandTimeoutException cause
+         (l/err :hint "timeout on set redis key" :key key :cause cause)
+         nil)))))
+
+(defn del
+  [conn key-or-keys]
+  (let [keys (if (vector? key-or-keys) key-or-keys [key-or-keys])]
+    (assert (every? string? keys) "only string keys allowed")
+    (try
+      (-del conn keys)
+      (catch RedisCommandTimeoutException cause
+        (l/err :hint "timeout on del redis key" :key key :cause cause)
+        nil))))
+
+(defn ping
+  [conn]
+  (-ping conn))
+
+(defn blpop
+  [conn key-or-keys timeout]
+  (let [keys    (if (vector? key-or-keys) key-or-keys [key-or-keys])
+        timeout (cond
+                  (ct/duration? timeout)
+                  (/ (double (inst-ms timeout)) 1000.0)
+
+                  (double? timeout)
+                  timeout
+
+                  (int? timeout)
+                  (/ (double timeout) 1000.0)
+
+                  :else
+                  0)]
+
+    (assert (every? string? keys) "only string keys allowed")
+    (-blpop conn keys timeout)))
+
+(defn rpush
+  [conn key elements]
+  (assert (string? key) "key must be string instance")
+  (assert (every? string? elements) "elements should be all strings")
+  (let [elements (vec elements)]
+    (-rpush conn key elements)))
+
+(defn publish
+  [conn topic payload]
+  (assert (string? topic) "expected topic to be string")
+  (assert (string? payload) "expected message to be a byte array")
+  (-publish conn topic payload))
+
+(def ^:private schema:script
+  [:map {:title "script"}
+   [::rscript/name qualified-keyword?]
+   [::rscript/path ::sm/text]
+   [::rscript/keys {:optional true} [:vector :any]]
+   [::rscript/vals {:optional true} [:vector :any]]])
+
+(def ^:private valid-script?
+  (sm/lazy-validator schema:script))
+
+(defn eval
+  [conn script]
+  (assert (valid-script? script) "expected valid script")
+  (-eval conn script))
+
+(defn add-listener
+  [conn listener]
+  (let [listener (cond
+                   (map? listener)
+                   (pubsub-listener listener)
+
+                   (instance? RedisPubSubListener listener)
+                   listener
+
+                   :else
+                   (throw (IllegalArgumentException. "invalid listener provided")))]
+
+    (-add-listener conn listener)))
+
+(defn subscribe
+  [conn topic-or-topics]
+  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
+    (assert (every? string? topics))
+    (-subscribe conn topics)))
+
+(defn unsubscribe
+  [conn topic-or-topics]
+  (let [topics (if (vector? topic-or-topics) topic-or-topics [topic-or-topics])]
+    (assert (every? string? topics))
+    (-unsubscribe conn topics)))
+
+(defn set-timeout
+  [conn timeout]
+  (let [timeout (ct/duration timeout)]
+    (-set-timeout conn timeout)))
+
+(defn get-timeout
+  [conn]
+  (-get-timeout conn))
+
+(defn reset-timeout
+  [conn]
+  (-reset-timeout conn))
 
 (defn timeout-exception?
   [cause]
@@ -374,3 +476,121 @@
 (defn exception?
   [cause]
   (instance? RedisException cause))
+
+(defn get-pooled
+  [cfg]
+  (let [pool (::pool cfg)]
+    (gpool/get pool)))
+
+(defn close
+  [o]
+  (.close ^AutoCloseable o))
+
+(defn pool
+  [cfg & {:as options}]
+  (gpool/create :create-fn (partial connect cfg options)
+                :destroy-fn close
+                :dispose-fn -reset-timeout))
+
+(defn run!
+  [cfg f & args]
+  (if (gpool/pool? cfg)
+    (apply f {::pool cfg} f args)
+    (let [pool (::pool cfg)]
+      (with-open [^AutoCloseable conn (gpool/get pool)]
+        (apply f (assoc cfg ::conn @conn) args)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; INITIALIZATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmethod ig/expand-key ::client
+  [k v]
+  {k (-> (d/without-nils v)
+         (assoc ::timeout (ct/duration "10s")))})
+
+(def ^:private schema:client
+  [:map {:title "RedisClient"}
+   [::timer [:fn #(instance? HashedWheelTimer %)]]
+   [::cache ::sm/atom]
+   [::timeout ::ct/duration]
+   [::resources [:fn #(instance? DefaultClientResources %)]]])
+
+(def check-client
+  (sm/check-fn schema:client))
+
+(sm/register! ::client schema:client)
+(sm/register!
+ {:type ::pool
+  :pred gpool/pool?})
+
+(def ^:private schema:client-params
+  [:map {:title "redis-params"}
+   ::wrk/netty-io-executor
+   ::wrk/netty-executor
+   [::uri ::sm/uri]
+   [::timeout ::ct/duration]])
+
+(def ^:private check-client-params
+  (sm/check-fn schema:client-params))
+
+(defmethod ig/assert-key ::client
+  [_ params]
+  (check-client-params params))
+
+(defmethod ig/init-key ::client
+  [_ {:keys [::uri ::wrk/netty-io-executor ::wrk/netty-executor] :as params}]
+
+  (l/inf :hint "initialize redis client" :uri (str uri))
+
+  (let [timer     (HashedWheelTimer.)
+        cache     (atom {})
+
+        resources (.. (DefaultClientResources/builder)
+                      (eventExecutorGroup ^EventExecutorGroup netty-executor)
+
+                      ;; We provide lettuce with a shared event loop
+                      ;; group instance instead of letting lettuce to
+                      ;; create its own
+                      (eventLoopGroupProvider
+                       (reify io.lettuce.core.resource.EventLoopGroupProvider
+                         (allocate [_ _] netty-io-executor)
+                         (threadPoolSize [_]
+                           (.executorCount ^NioEventLoopGroup netty-io-executor))
+                         (release [_ _ _ _ _]
+                           ;; Do nothing
+                           )
+                         (shutdown [_ _ _ _]
+                           ;; Do nothing
+                           )))
+
+                      (timer ^Timer timer)
+                      (build))
+
+        redis-uri (RedisURI/create ^String (str uri))
+        client    (RedisClient/create ^ClientResources resources
+                                      ^RedisURI redis-uri)]
+
+    {::client client
+     ::cache cache
+     ::timer timer
+     ::timeout default-timeout
+     ::resources resources}))
+
+(defmethod ig/halt-key! ::client
+  [_ {:keys [::client ::timer ::resources]}]
+  (ex/ignoring (.shutdown ^RedisClient client))
+  (ex/ignoring (.shutdown ^ClientResources resources))
+  (ex/ignoring (.stop ^Timer timer)))
+
+(defmethod ig/assert-key ::pool
+  [_ {:keys [::client]}]
+  (check-client client))
+
+(defmethod ig/init-key ::pool
+  [_ cfg]
+  (pool cfg {:timeout (ct/duration 2000)}))
+
+(defmethod ig/halt-key! ::pool
+  [_ instance]
+  (.close ^java.lang.AutoCloseable instance))

backend/src/app/rpc.clj

@@ -12,18 +12,25 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.spec :as us]
+   [app.common.time :as ct]
+   [app.common.uri :as u]
+   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
    [app.http.access-token :as actoken]
    [app.http.client :as-alias http.client]
+   [app.http.middleware :as mw]
+   [app.http.security :as sec]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.msgbus :as-alias mbus]
+   [app.redis :as rds]
    [app.rpc.climit :as climit]
    [app.rpc.cond :as cond]
+   [app.rpc.doc :as doc]
    [app.rpc.helpers :as rph]
    [app.rpc.retry :as retry]
    [app.rpc.rlimit :as rlimit]
@@ -31,11 +38,9 @@
    [app.storage :as-alias sto]
    [app.util.inet :as inet]
    [app.util.services :as sv]
-   [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [promesa.core :as p]
    [yetti.request :as yreq]
    [yetti.response :as yres]))
 
@@ -43,7 +48,7 @@
 
 (defn- default-handler
   [_]
-  (p/rejected (ex/error :type :not-found)))
+  (ex/raise :type :not-found))
 
 (defn- handle-response-transformation
   [response request mdata]
@@ -63,74 +68,69 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result (rph/unwrap result)]
+                   (let [result  (rph/unwrap result)
-                     {::yres/status  (::http/status mdata 200)
+                         status  (or (::http/status mdata)
-                      ::yres/headers (::http/headers mdata {})
+                                     (if (nil? result)
+                                       204
+                                       200))
+                         headers (cond-> (::http/headers mdata {})
+                                   (yres/stream-body? result)
+                                   (assoc "content-type" "application/octet-stream"))]
+                     {::yres/status  status
+                      ::yres/headers headers
                       ::yres/body    result}))]
+
     (-> response
         (handle-response-transformation request mdata)
         (handle-before-comple-hook mdata))))
 
-(defn get-external-session-id
+(defn- make-rpc-handler
-  [request]
-  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-    (when-not (or (> (count session-id) 256)
-                  (= session-id "null")
-                  (str/blank? session-id))
-      session-id)))
-
-(defn- get-external-event-origin
-  [request]
-  (when-let [origin (yreq/get-header request "x-event-origin")]
-    (when-not (or (> (count origin) 256)
-                  (= origin "null")
-                  (str/blank? origin))
-      origin)))
-
-(defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
-  [methods {:keys [params path-params method] :as request}]
+  [methods]
-  (let [handler-name (:type path-params)
+  (let [methods (update-vals methods peek)]
-        etag         (yreq/get-header request "if-none-match")
+    (fn [{:keys [params path-params method] :as request}]
-        profile-id   (or (::session/profile-id request)
+      (let [handler-name (:type path-params)
-                         (::actoken/profile-id request))
+            etag         (yreq/get-header request "if-none-match")
+            profile-id   (or (::session/profile-id request)
+                             (::actoken/profile-id request)
+                             (if (::http/auth-with-shared-key request)
+                               uuid/zero
+                               nil))
 
-        ip-addr      (inet/parse-request request)
+            ip-addr      (inet/parse-request request)
-        session-id   (get-external-session-id request)
-        event-origin (get-external-event-origin request)
 
-        data         (-> params
+            data         (-> params
-                         (assoc ::handler-name handler-name)
+                             (assoc ::handler-name handler-name)
-                         (assoc ::ip-addr ip-addr)
+                             (assoc ::ip-addr ip-addr)
-                         (assoc ::request-at (dt/now))
+                             (assoc ::request-at (ct/now))
-                         (assoc ::external-session-id session-id)
+                             (assoc ::cond/key etag)
-                         (assoc ::external-event-origin event-origin)
+                             (cond-> (uuid? profile-id)
-                         (assoc ::session/id (::session/id request))
+                               (assoc ::profile-id profile-id)))
-                         (assoc ::cond/key etag)
-                         (cond-> (uuid? profile-id)
-                           (assoc ::profile-id profile-id)))
 
-        data         (vary-meta data assoc ::http/request request)
+            data         (with-meta data
-        handler-fn   (get methods (keyword handler-name) default-handler)]
+                           {::http/request request})
 
-    (when (and (or (= method :get)
+            handler-fn   (get methods (keyword handler-name) default-handler)]
-                   (= method :head))
-               (not (str/starts-with? handler-name "get-")))
-      (ex/raise :type :restriction
-                :code :method-not-allowed
-                :hint "method not allowed for this request"))
 
-    (binding [cond/*enabled* true]
+        (when (and (or (= method :get)
-      (let [response (handler-fn data)]
+                       (= method :head))
-        (handle-response request response)))))
+                   (not (str/starts-with? handler-name "get-")))
+          (ex/raise :type :restriction
+                    :code :method-not-allowed
+                    :hint "method not allowed for this request"))
+
+        ;; FIXME: why we have this cond enabled here, we need to move it outside this handler
+        (binding [cond/*enabled* true]
+          (let [response (handler-fn data)]
+            (handle-response request response)))))))
 
 (defn- wrap-metrics
   "Wrap service method with metrics measurement."
   [{:keys [::mtx/metrics ::metrics-id]} f mdata]
   (let [labels (into-array String [(::sv/name mdata)])]
     (fn [cfg params]
-      (let [tp (dt/tpoint)]
+      (let [tp (ct/tpoint)]
         (try
           (f cfg params)
           (finally
@@ -200,7 +200,7 @@
                         ::sm/explain (explain params)))))))
     f))
 
-(defn- wrap-all
+(defn- wrap
   [cfg f mdata]
   (as-> f $
     (wrap-db-transaction cfg $ mdata)
@@ -214,17 +214,30 @@
     (wrap-params-validation cfg $ mdata)
     (wrap-authentication cfg $ mdata)))
 
-(defn- wrap
+(defn- wrap-management
   [cfg f mdata]
-  (l/trc :hint "register method" :name (::sv/name mdata))
+  (as-> f $
-  (let [f (wrap-all cfg f mdata)]
+    (wrap-db-transaction cfg $ mdata)
-    (partial f cfg)))
+    (retry/wrap-retry cfg $ mdata)
+    (climit/wrap cfg $ mdata)
+    (wrap-metrics cfg $ mdata)
+    (wrap-audit cfg $ mdata)
+    (wrap-spec-conform cfg $ mdata)
+    (wrap-params-validation cfg $ mdata)
+    (wrap-authentication cfg $ mdata)))
 
 (defn- process-method
-  [cfg [vfn mdata]]
+  [cfg module wrap-fn [f mdata]]
-  [(keyword (::sv/name mdata)) [mdata (wrap cfg vfn mdata)]])
+  (l/trc :hint "add method" :module module :name (::sv/name mdata))
+  (let [f (wrap-fn cfg f mdata)
+        k (keyword (::sv/name mdata))]
+    [k [mdata (partial f cfg)]]))
 
-(defn- resolve-command-methods
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; API METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-methods
   [cfg]
   (let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
     (->> (sv/scan-ns
@@ -239,7 +252,6 @@
           'app.rpc.commands.files
           'app.rpc.commands.files-create
           'app.rpc.commands.files-share
-          'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
           'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails
@@ -254,7 +266,7 @@
           'app.rpc.commands.verify-token
           'app.rpc.commands.viewer
           'app.rpc.commands.webhooks)
-         (map (partial process-method cfg))
+         (map (partial process-method cfg "rpc" wrap))
          (into {}))))
 
 (def ^:private schema:methods-params
@@ -262,6 +274,7 @@
    ::session/manager
    ::http.client/client
    ::db/pool
+   ::rds/pool
    ::mbus/msgbus
    ::sto/storage
    ::mtx/metrics
@@ -277,7 +290,50 @@
 (defmethod ig/init-key ::methods
   [_ cfg]
   (let [cfg (d/without-nils cfg)]
-    (resolve-command-methods cfg)))
+    (resolve-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MANAGEMENT METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-management-methods
+  [cfg]
+  (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
+    (->> (sv/scan-ns
+          'app.rpc.management.subscription
+          'app.rpc.management.exporter)
+         (map (partial process-method cfg "management" wrap-management))
+         (into {}))))
+
+(def ^:private schema:management-methods-params
+  [:map {:title "management-methods-params"}
+   ::session/manager
+   ::http.client/client
+   ::db/pool
+   ::rds/pool
+   ::mbus/msgbus
+   ::sto/storage
+   ::mtx/metrics
+   ::setup/props])
+
+(defmethod ig/assert-key ::management-methods
+  [_ params]
+  (assert (sm/check schema:management-methods-params params)))
+
+(defmethod ig/init-key ::management-methods
+  [_ cfg]
+  (let [cfg (d/without-nils cfg)]
+    (resolve-management-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ROUTES
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- redirect
+  [href]
+  (fn [_]
+    {::yres/status 308
+     ::yres/headers {"location" (str href)}}))
 
 (def ^:private schema:methods
   [:map-of :keyword [:tuple :map ::sm/fn]])
@@ -292,11 +348,50 @@
   (assert (db/pool? (::db/pool params)) "expect valid database pool")
   (assert (some? (::setup/props params)))
   (assert (session/manager? (::session/manager params)) "expect valid session manager")
-  (assert (valid-methods? (::methods params)) "expect valid methods map"))
+  (assert (valid-methods? (::methods params)) "expect valid methods map")
+  (assert (valid-methods? (::management-methods params)) "expect valid methods map"))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [::methods] :as cfg}]
+  [_ {:keys [::methods ::management-methods ::setup/props] :as cfg}]
-  (let [methods (update-vals methods peek)]
+
-    [["/rpc" {:middleware [[session/authz cfg]
+  (let [public-uri     (cf/get :public-uri)
-                           [actoken/authz cfg]]}
+        management-key (or (cf/get :management-api-key)
-      ["/command/:type" {:handler (partial rpc-handler methods)}]]]))
+                           (get props :management-key))]
+
+    ["/api"
+     ["/management"
+      ["/methods/:type"
+       {:middleware [[mw/shared-key-auth management-key]
+                     [session/authz cfg]]
+        :handler (make-rpc-handler management-methods)}]
+
+      (doc/routes :methods management-methods
+                  :label "management"
+                  :base-uri (u/join public-uri "/api/management")
+                  :description "MANAGEMENT API")]
+
+     ["/main"
+      ["/methods/:type"
+       {:middleware [[mw/cors]
+                     [sec/client-header-check]
+                     [session/authz cfg]
+                     [actoken/authz cfg]]
+        :handler (make-rpc-handler methods)}]
+
+      (doc/routes :methods methods
+                  :label "main"
+                  :base-uri (u/join public-uri "/api/main")
+                  :description "MAIN API")]
+
+     ;; BACKWARD COMPATIBILITY
+     ["/_doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/openapi" {:handler (redirect (u/join public-uri "/api/main/doc/openapi"))}]
+     ["/openapi.join" {:handler (redirect (u/join public-uri "/api/main/doc/openapi.json"))}]
+
+     ["/rpc/command/:type"
+      {:middleware [[mw/cors]
+                    [sec/client-header-check]
+                    [session/authz cfg]
+                    [actoken/authz cfg]]
+       :handler (make-rpc-handler methods)}]]))

backend/src/app/rpc/climit.clj

@@ -11,17 +11,16 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
    [app.util.cache :as cache]
    [app.util.services :as-alias sv]
-   [app.util.time :as dt]
    [app.worker :as-alias wrk]
    [clojure.edn :as edn]
    [clojure.set :as set]
    [datoteka.fs :as fs]
    [integrant.core :as ig]
-   [promesa.exec :as px]
    [promesa.exec.bulkhead :as pbh])
   (:import
    clojure.lang.ExceptionInfo
@@ -154,7 +153,7 @@
            :id limit-id
            :label limit-label
            :queue queue
-           :elapsed (some-> elapsed dt/format-duration)
+           :elapsed (some-> elapsed ct/format-duration)
            :params @limit-params)))
 
 (def ^:private idseq (AtomicLong. 0))
@@ -171,19 +170,19 @@
         mlabels     (into-array String [(id->str limit-id)])
         limit-id    (id->str limit-id limit-key)
         limiter     (cache/get cache limit-id (partial create-limiter config))
-        tpoint      (dt/tpoint)
+        tpoint      (ct/tpoint)
         req-id      (.incrementAndGet ^AtomicLong idseq)]
     (try
       (let [stats (pbh/get-stats limiter)]
         (measure metrics mlabels stats nil)
         (log "enqueued" req-id stats limit-id limit-label limit-params nil))
 
-      (px/invoke! limiter (fn []
+      (pbh/invoke! limiter (fn []
-                            (let [elapsed (tpoint)
+                             (let [elapsed (tpoint)
-                                  stats   (pbh/get-stats limiter)]
+                                   stats   (pbh/get-stats limiter)]
-                              (measure metrics mlabels stats elapsed)
+                               (measure metrics mlabels stats elapsed)
-                              (log "acquired" req-id stats limit-id limit-label limit-params elapsed)
+                               (log "acquired" req-id stats limit-id limit-label limit-params elapsed)
-                              (handler))))
+                               (handler))))
 
       (catch ExceptionInfo cause
         (let [{:keys [type code]} (ex-data cause)]
@@ -289,13 +288,9 @@
           (get-limits cfg)))
 
 (defn invoke!
-  "Run a function in context of climit.
+  "Run a function in context of climit."
-  Intended to be used in virtual threads."
+  [{:keys [::rpc/climit] :as cfg} f params]
-  [{:keys [::executor ::rpc/climit] :as cfg} f params]
   (let [f (if climit
-            (let [f (if (some? executor)
+            (build-exec-chain cfg f)
-                      (fn [cfg params] (px/await! (px/submit! executor (fn [] (f cfg params)))))
-                      f)]
-              (build-exec-chain cfg f))
             f)]
     (f cfg params)))

backend/src/app/rpc/commands/access_token.clj

@@ -7,6 +7,7 @@
 (ns app.rpc.commands.access-token
   (:require
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.main :as-alias main]
@@ -15,22 +16,22 @@
    [app.rpc.quotes :as quotes]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [app.util.services :as sv]
+   [app.util.services :as sv]))
-   [app.util.time :as dt]))
 
 (defn- decode-row
   [row]
   (dissoc row :perms))
 
 (defn create-access-token
-  [{:keys [::db/conn ::setup/props]} profile-id name expiration]
+  [{:keys [::db/conn] :as cfg} profile-id name expiration]
-  (let [created-at (dt/now)
+  (let [token-id   (uuid/next)
-        token-id   (uuid/next)
+        expires-at (some-> expiration (ct/in-future))
-        token      (tokens/generate props {:iss "access-token"
+        created-at (ct/now)
-                                           :tid token-id
+        token      (tokens/generate cfg {:iss "access-token"
-                                           :iat created-at})
+                                         :uid profile-id
+                                         :iat created-at
+                                         :tid token-id})
 
-        expires-at (some-> expiration dt/in-future)
         token      (db/insert! conn :access-token
                                {:id token-id
                                 :name name
@@ -49,7 +50,7 @@
 (def ^:private schema:create-access-token
   [:map {:title "create-access-token"}
    [:name [:string {:max 250 :min 1}]]
-   [:expiration {:optional true} ::dt/duration]])
+   [:expiration {:optional true} ::ct/duration]])
 
 (sv/defmethod ::create-access-token
   {::doc/added "1.18"

backend/src/app/rpc/commands/audit.clj

@@ -10,6 +10,7 @@
    [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -20,8 +21,7 @@
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
    [app.util.inet :as inet]
-   [app.util.services :as sv]
+   [app.util.services :as sv]))
-   [app.util.time :as dt]))
 
 (def ^:private event-columns
   [:id
@@ -49,7 +49,7 @@
 
 (defn- adjust-timestamp
   [{:keys [timestamp created-at] :as event}]
-  (let [margin (inst-ms (dt/diff timestamp created-at))]
+  (let [margin (inst-ms (ct/diff timestamp created-at))]
     (if (or (neg? margin)
             (> margin 3600000))
       ;; If event is in future or lags more than 1 hour, we reasign
@@ -63,7 +63,7 @@
   [{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}]
   (let [request (-> params meta ::http/request)
         ip-addr (inet/parse-request request)
-        tnow    (dt/now)
+        tnow    (ct/now)
         xform   (comp
                  (map (fn [event]
                         (-> event

backend/src/app/rpc/commands/auth.clj

@@ -6,20 +6,25 @@
 
 (ns app.rpc.commands.auth
   (:require
+   [app.auth :as auth]
+   [app.auth.oidc :as oidc]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.email :as eml]
    [app.email.blacklist :as email.blacklist]
    [app.email.whitelist :as email.whitelist]
+   [app.http :as-alias http]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
+   [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
    [app.rpc.commands.profile :as profile]
@@ -28,9 +33,9 @@
    [app.rpc.helpers :as rph]
    [app.setup :as-alias setup]
    [app.setup.welcome-file :refer [create-welcome-file]]
+   [app.storage :as sto]
    [app.tokens :as tokens]
    [app.util.services :as sv]
-   [app.util.time :as dt]
    [app.worker :as wrk]
    [cuerdas.core :as str]))
 
@@ -42,7 +47,7 @@
 
 (defn- elapsed-verify-threshold?
   [profile]
-  (let [elapsed (dt/diff (:modified-at profile) (dt/now))
+  (let [elapsed (ct/diff (:modified-at profile) (ct/now))
         verify-threshold (cf/get :email-verify-threshold)]
     (pos? (compare elapsed verify-threshold))))
 
@@ -62,7 +67,7 @@
               (ex/raise :type :validation
                         :code :account-without-password
                         :hint "the current account does not have password")
-              (let [result (profile/verify-password cfg password (:password profile))]
+              (let [result (auth/verify-password password (:password profile))]
                 (when (:update result)
                   (l/trc :hint "updating profile password"
                          :id (str (:id profile))
@@ -85,7 +90,7 @@
               (ex/raise :type :validation
                         :code :wrong-credentials))
             (when-let [deleted-at (:deleted-at profile)]
-              (when (dt/is-after? (dt/now) deleted-at)
+              (when (ct/is-after? (ct/now) deleted-at)
                 (ex/raise :type :validation
                           :code :wrong-credentials)))
 
@@ -98,7 +103,7 @@
                                   (profile/strip-private-attrs))
 
                   invitation (when-let [token (:invitation-token params)]
-                               (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
+                               (tokens/verify cfg {:token token :iss :team-invitation}))
 
                   ;; If invitation member-id does not matches the profile-id, we just proceed to ignore the
                   ;; invitation because invitations matches exactly; and user can't login with other email and
@@ -108,7 +113,7 @@
                                (assoc profile :is-admin (let [admins (cf/get :admins)]
                                                           (contains? admins (:email profile)))))]
               (-> response
-                  (rph/with-transform (session/create-fn cfg (:id profile)))
+                  (rph/with-transform (session/create-fn cfg profile))
                   (rph/with-meta {::audit/props (audit/profile->props profile)
                                   ::audit/profile-id (:id profile)}))))]
 
@@ -144,7 +149,24 @@
   [cfg params]
   (if (= (:profile-id params)
          (::rpc/profile-id params))
-    (rph/with-transform {} (session/delete-fn cfg))
+    (let [{:keys [claims]}
+          (rph/get-auth-data params)
+
+          provider
+          (some->> (get claims :sso-provider-id)
+                   (oidc/get-provider cfg))
+
+          response
+          (if (and provider (:logout-uri provider))
+            (let [params {"logout_hint" (get claims :sso-session-id)
+                          "client_id" (get provider :client-id)
+                          "post_logout_redirect_uri" (str (cf/get :public-uri))}
+                  uri    (-> (u/uri (:logout-uri provider))
+                             (assoc :query (u/map->query-string params)))]
+              {:redirect-uri uri})
+            {})]
+
+      (rph/with-transform response (session/delete-fn cfg)))
     {}))
 
 ;; ---- COMMAND: Recover Profile
@@ -152,11 +174,11 @@
 (defn recover-profile
   [{:keys [::db/conn] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
-            (let [tdata (tokens/verify (::setup/props cfg) {:token token :iss :password-recovery})]
+            (let [tdata (tokens/verify cfg {:token token :iss :password-recovery})]
               (:profile-id tdata)))
 
           (update-password [conn profile-id]
-            (let [pwd (profile/derive-password cfg password)]
+            (let [pwd (auth/derive-password password)]
               (db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
               nil))]
 
@@ -191,7 +213,7 @@
               :hint "registration disabled"))
 
   (when (contains? params :invitation-token)
-    (let [invitation (tokens/verify (::setup/props cfg)
+    (let [invitation (tokens/verify cfg
                                     {:token (:invitation-token params)
                                      :iss :team-invitation})]
       (when-not (= (:email params) (:member-email invitation))
@@ -244,11 +266,11 @@
                  :backend "penpot"
                  :iss :prepared-register
                  :profile-id (:id profile)
-                 :exp (dt/in-future {:days 7})
+                 :exp (ct/in-future {:days 7})
                  :props {:newsletter-updates (or accept-newsletter-updates false)}}
 
         params (d/without-nils params)
-        token  (tokens/generate (::setup/props cfg) params)]
+        token  (tokens/generate cfg params)]
 
     (with-meta {:token token}
       {::audit/profile-id uuid/zero})))
@@ -270,11 +292,30 @@
 
 ;; ---- COMMAND: Register Profile
 
-(defn create-profile!
+(defn import-profile-picture
+  [cfg uri]
+  (try
+    (let [storage (sto/resolve cfg)
+          input   (media/download-image cfg uri)
+          input   (media/run {:cmd :info :input input})
+          hash    (sto/calculate-hash (:path input))
+          content (-> (sto/content (:path input) (:size input))
+                      (sto/wrap-with-hash hash))
+          sobject (sto/put-object! storage {::sto/content content
+                                            ::sto/deduplicate? true
+                                            :bucket "profile"
+                                            :content-type (:mtype input)})]
+      (:id sobject))
+    (catch Throwable cause
+      (l/wrn :hint "unable to import profile picture"
+             :uri uri
+             :cause cause)
+      nil)))
+
+(defn create-profile
   "Create the profile entry on the database with limited set of input
   attrs (all the other attrs are filled with default values)."
-  [conn {:keys [email] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
-  (dm/assert! ::sm/email email)
   (let [id        (or (:id params) (uuid/next))
         props     (-> (audit/extract-utm-params params)
                       (merge (:props params))
@@ -282,8 +323,7 @@
                               :viewed-walkthrough? false
                               :nudge {:big 10 :small 1}
                               :v2-info-shown true
-                              :release-notes-viewed (:main cf/version)})
+                              :release-notes-viewed (:main cf/version)}))
-                      (db/tjson))
 
         password  (or (:password params) "!")
 
@@ -298,6 +338,12 @@
         theme     (:theme params nil)
         email     (str/lower email)
 
+        photo-id  (some->> (or (:oidc/picture props)
+                               (:google/picture props)
+                               (:github/picture props)
+                               (:gitlab/picture props))
+                           (import-profile-picture cfg))
+
         params    {:id id
                    :fullname (:fullname params)
                    :email email
@@ -305,27 +351,26 @@
                    :lang locale
                    :password password
                    :deleted-at (:deleted-at params)
-                   :props props
+                   :props (db/tjson props)
                    :theme theme
+                   :photo-id photo-id
                    :is-active is-active
                    :is-muted is-muted
                    :is-demo is-demo}]
+
     (try
       (-> (db/insert! conn :profile params)
           (profile/decode-row))
       (catch org.postgresql.util.PSQLException cause
-        (let [state (.getSQLState cause)]
+        (if (db/duplicate-key-error? cause)
-          (if (not= state "23505")
+          (ex/raise :type :validation
-            (throw cause)
+                    :code :email-already-exists
+                    :hint "email already exists"
+                    :cause cause)
+          (throw cause))))))
 
-            (do
-              (l/error :hint "not an error" :cause cause)
-              (ex/raise :type :validation
-                        :code :email-already-exists
-                        :hint "email already exists"
-                        :cause cause))))))))
 
-(defn create-profile-rels!
+(defn create-profile-rels
   [conn {:keys [id] :as profile}]
   (let [features (cfeat/get-enabled-features cf/flags)
         team     (teams/create-team conn
@@ -342,17 +387,17 @@
 
 (defn send-email-verification!
   [{:keys [::db/conn] :as cfg} profile]
-  (let [vtoken (tokens/generate (::setup/props cfg)
+  (let [vtoken (tokens/generate cfg
                                 {:iss :verify-email
-                                 :exp (dt/in-future "72h")
+                                 :exp (ct/in-future "72h")
                                  :profile-id (:id profile)
                                  :email (:email profile)})
         ;; NOTE: this token is mainly used for possible complains
         ;; identification on the sns webhook
-        ptoken (tokens/generate (::setup/props cfg)
+        ptoken (tokens/generate cfg
                                 {:iss :profile-identity
                                  :profile-id (:id profile)
-                                 :exp (dt/in-future {:days 30})})]
+                                 :exp (ct/in-future {:days 30})})]
     (eml/send! {::eml/conn conn
                 ::eml/factory eml/register
                 :public-uri (cf/get :public-uri)
@@ -363,7 +408,7 @@
 
 (defn register-profile
   [{:keys [::db/conn ::wrk/executor] :as cfg} {:keys [token] :as params}]
-  (let [claims     (tokens/verify (::setup/props cfg) {:token token :iss :prepared-register})
+  (let [claims     (tokens/verify cfg {:token token :iss :prepared-register})
         params     (into claims params)
 
         profile    (if-let [profile-id (:profile-id claims)]
@@ -375,18 +420,19 @@
                      ;; to detect if the profile is already registered
                      (or (profile/get-profile-by-email conn (:email claims))
                          (let [is-active (or (boolean (:is-active claims))
+                                             (boolean (:email-verified claims))
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
-                                             (update :password #(profile/derive-password cfg %)))
+                                             (update :password auth/derive-password))
-                               profile   (->> (create-profile! conn params)
+                               profile   (->> (create-profile cfg params)
-                                              (create-profile-rels! conn))]
+                                              (create-profile-rels conn))]
                            (vary-meta profile assoc :created true))))
 
         created?   (-> profile meta :created true?)
 
         invitation (when-let [token (:invitation-token params)]
-                     (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
+                     (tokens/verify cfg {:token token :iss :team-invitation}))
 
         props      (-> (audit/profile->props profile)
                        (assoc :from-invitation (some? invitation)))
@@ -418,10 +464,10 @@
       (and (some? invitation)
            (= (:email profile)
               (:member-email invitation)))
-      (let [claims (assoc invitation :member-id  (:id profile))
+      (let [invitation (assoc invitation :member-id  (:id profile))
-            token  (tokens/generate (::setup/props cfg) claims)]
+            token      (tokens/generate cfg invitation)]
         (-> {:invitation-token token}
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-meta {::audit/replace-props props
                             ::audit/context {:action "accept-invitation"}
                             ::audit/profile-id (:id profile)})))
@@ -432,7 +478,7 @@
       created?
       (if (:is-active profile)
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-defer create-welcome-file-when-needed)
             (rph/with-meta
               {::audit/replace-props props
@@ -466,7 +512,7 @@
 
         (when (= action "resend-email-verification")
           (db/update! conn :profile
-                      {:modified-at (dt/now)}
+                      {:modified-at (ct/now)}
                       {:id (:id profile)})
           (send-email-verification! cfg profile))
 
@@ -493,17 +539,17 @@
 (defn- request-profile-recovery
   [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
   (letfn [(create-recovery-token [{:keys [id] :as profile}]
-            (let [token (tokens/generate (::setup/props cfg)
+            (let [token (tokens/generate cfg
                                          {:iss :password-recovery
-                                          :exp (dt/in-future "15m")
+                                          :exp (ct/in-future "15m")
                                           :profile-id id})]
               (assoc profile :token token)))
 
           (send-email-notification [conn profile]
-            (let [ptoken (tokens/generate (::setup/props cfg)
+            (let [ptoken (tokens/generate cfg
                                           {:iss :profile-identity
                                            :profile-id (:id profile)
-                                           :exp (dt/in-future {:days 30})})]
+                                           :exp (ct/in-future {:days 30})})]
               (eml/send! {::eml/conn conn
                           ::eml/factory eml/password-recovery
                           :public-uri (cf/get :public-uri)
@@ -544,7 +590,7 @@
         :else
         (do
           (db/update! conn :profile
-                      {:modified-at (dt/now)}
+                      {:modified-at (ct/now)}
                       {:id (:id profile)})
           (->> profile
                (create-recovery-token)
@@ -561,4 +607,32 @@
   [cfg params]
   (db/tx-run! cfg request-profile-recovery params))
 
+;; --- COMMAND: get-sso-config
 
+(defn- extract-domain
+  "Extract the domain part from email"
+  [email]
+  (let [at (str/last-index-of email "@")]
+    (when (and (>= at 0)
+               (< at (dec (count email))))
+      (-> (subs email (inc at))
+          (str/trim)
+          (str/lower)))))
+
+(def ^:private schema:get-sso-provider
+  [:map {:title "get-sso-config"}
+   [:email ::sm/email]])
+
+(def ^:private schema:get-sso-provider-result
+  [:map {:title "SSOProvider"}
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::get-sso-provider
+  {::rpc/auth false
+   ::doc/added "2.12"
+   ::sm/params schema:get-sso-provider
+   ::sm/result schema:get-sso-provider-result}
+  [cfg {:keys [email]}]
+  (when-let [domain (extract-domain email)]
+    (when-let [config (db/get* cfg :sso-provider {:domain domain})]
+      (select-keys config [:id]))))

backend/src/app/rpc/commands/binfile.clj

@@ -11,8 +11,9 @@
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
    [app.common.features :as cfeat]
-   [app.common.logging :as l]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
@@ -24,12 +25,12 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
+   [app.storage :as sto]
+   [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.services :as sv]
-   [app.util.time :as dt]
    [app.worker :as-alias wrk]
-   [promesa.exec :as px]
+   [datoteka.fs :as fs]))
-   [yetti.response :as yres]))
 
 (set! *warn-on-reflection* true)
 
@@ -39,62 +40,47 @@
   schema:export-binfile
   [:map {:title "export-binfile"}
    [:file-id ::sm/uuid]
-   [:version {:optional true} ::sm/int]
    [:include-libraries ::sm/boolean]
    [:embed-assets ::sm/boolean]])
 
-(defn stream-export-v1
+(defn- export-binfile
-  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+  [{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
-  (yres/stream-body
+  (let [output  (tmp/tempfile*)]
-   (fn [_ output-stream]
+    (try
-     (try
+      (-> cfg
-       (-> cfg
+          (assoc ::bfc/ids #{file-id})
-           (assoc ::bfc/ids #{file-id})
+          (assoc ::bfc/embed-assets embed-assets)
-           (assoc ::bfc/embed-assets embed-assets)
+          (assoc ::bfc/include-libraries include-libraries)
-           (assoc ::bfc/include-libraries include-libraries)
+          (bf.v3/export-files! output))
-           (bf.v1/export-files! output-stream))
-       (catch Throwable cause
-         (l/err :hint "exception on exporting file"
-                :file-id (str file-id)
-                :cause cause))))))
 
-(defn stream-export-v3
+      (let [data   (sto/content output)
-  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+            object (sto/put-object! storage
-  (yres/stream-body
+                                    {::sto/content data
-   (fn [_ output-stream]
+                                     ::sto/touched-at (ct/in-future {:minutes 60})
-     (try
+                                     :content-type "application/zip"
-       (-> cfg
+                                     :bucket "tempfile"})]
-           (assoc ::bfc/ids #{file-id})
+
-           (assoc ::bfc/embed-assets embed-assets)
+        (-> (cf/get :public-uri)
-           (assoc ::bfc/include-libraries include-libraries)
+            (u/join "/assets/by-id/")
-           (bf.v3/export-files! output-stream))
+            (u/join (str (:id object)))))
-       (catch Throwable cause
+
-         (l/err :hint "exception on exporting file"
+      (finally
-                :file-id (str file-id)
+        (fs/delete output)))))
-                :cause cause))))))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
+   ::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
    ::webhooks/event? true
    ::sm/params schema:export-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (fn [_]
+  (sse/response (partial export-binfile cfg params)))
-    (let [version (or version 1)
-          body    (case (int version)
-                    1 (stream-export-v1 cfg params)
-                    2 (throw (ex-info "not-implemented" {}))
-                    3 (stream-export-v3 cfg params))]
-
-      {::yres/status 200
-       ::yres/headers {"content-type" "application/octet-stream"}
-       ::yres/body body})))
 
 ;; --- Command: import-binfile
 
 (defn- import-binfile
-  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id project-id version name file]}]
+  [{:keys [::db/pool] :as cfg} {:keys [profile-id project-id version name file]}]
   (let [team   (teams/get-team pool
                                :profile-id profile-id
                                :project-id project-id)
@@ -105,16 +91,12 @@
                    (assoc ::bfc/name name)
                    (assoc ::bfc/input (:path file)))
 
-        ;; NOTE: the importation process performs some operations that are
-        ;; not very friendly with virtual threads, and for avoid
-        ;; unexpected blocking of other concurrent operations we dispatch
-        ;; that operation to a dedicated executor.
         result (case (int version)
-                 1 (px/invoke! executor (partial bf.v1/import-files! cfg))
+                 1 (bf.v1/import-files! cfg)
-                 3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
+                 3 (bf.v3/import-files! cfg))]
 
     (db/update! pool :project
-                {:modified-at (dt/now)}
+                {:modified-at (ct/now)}
                 {:id project-id}
                 {::db/return-keys false})
 
@@ -125,20 +107,42 @@
    [:name [:or [:string {:max 250}]
            [:map-of ::sm/uuid [:string {:max 250}]]]]
    [:project-id ::sm/uuid]
+   [:file-id {:optional true} ::sm/uuid]
    [:version {:optional true} ::sm/int]
-   [:file ::media/upload]])
+   [:file media/schema:upload]])
 
 (sv/defmethod ::import-binfile
-  "Import a penpot file in a binary format."
+  "Import a penpot file in a binary format. If `file-id` is provided,
+  an in-place import will be performed instead of creating a new file.
+
+  The in-place imports are only supported for binfile-v3 and when a
+  .penpot file only contains one penpot file.
+  "
   {::doc/added "1.15"
+   ::doc/changes ["1.20" "Add file-id param for in-place import"
+                  "1.20" "Set default version to 3"]
+
    ::webhooks/event? true
    ::sse/stream? true
    ::sm/params schema:import-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}]
   (projects/check-edition-permissions! pool profile-id project-id)
-  (let [params (-> params
+  (let [version  (or version 3)
-                   (assoc :profile-id profile-id)
+        params   (-> params
-                   (assoc :version (or version 1)))]
+                     (assoc :profile-id profile-id)
+                     (assoc :version version))
+
+        cfg      (cond-> cfg
+                   (uuid? file-id)
+                   (assoc ::bfc/file-id file-id))
+
+        manifest (case (int version)
+                   1 nil
+                   3 (bf.v3/get-manifest (:path file)))]
+
     (with-meta
       (sse/response (partial import-binfile cfg params))
-      {::audit/props {:file nil}})))
+      {::audit/props {:file nil
+                      :file-id file-id
+                      :generated-by (:generated-by manifest)
+                      :referer (:referer manifest)}})))

backend/src/app/rpc/commands/comments.clj

@@ -6,11 +6,13 @@
 
 (ns app.rpc.commands.comments
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uri :as uri]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -29,7 +31,6 @@
    [app.rpc.retry :as rtry]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
-   [app.util.time :as dt]
    [clojure.set :as set]
    [cuerdas.core :as str]))
 
@@ -163,34 +164,16 @@
 (def xf-decode-row
   (map decode-row))
 
-(def ^:private
-  sql:get-file
-  "SELECT f.id, f.modified_at, f.revn, f.features, f.name,
-          f.project_id, p.team_id, f.data,
-          f.data_ref_id, f.data_backend
-     FROM file as f
-    INNER JOIN project as p on (p.id = f.project_id)
-    WHERE f.id = ?
-      AND (f.deleted_at IS NULL OR f.deleted_at > now())")
-
 (defn- get-file
   "A specialized version of get-file for comments module."
   [cfg file-id page-id]
-  (let [file (db/exec-one! cfg [sql:get-file file-id])]
+  (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
-    (when-not file
+    (let [file (bfc/get-file cfg file-id)
-      (ex/raise :type :not-found
+          data (get file :data)]
-                :code :object-not-found
+      (-> file
-                :hint "file not found"))
+          (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
-
+          (assoc :page-id page-id)
-    (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
+          (dissoc :data)))))
-      (let [file (->> file
-                      (files/decode-row)
-                      (feat.fdata/resolve-file-data cfg))
-            data (get file :data)]
-        (-> file
-            (assoc :page-name (dm/get-in data [:pages-index page-id :name]))
-            (assoc :page-id page-id)
-            (dissoc :data))))))
 
 ;; FIXME: rename
 (defn- get-comment-thread
@@ -222,7 +205,7 @@
 
 (defn upsert-comment-thread-status!
   ([conn profile-id thread-id]
-   (upsert-comment-thread-status! conn profile-id thread-id (dt/in-future "1s")))
+   (upsert-comment-thread-status! conn profile-id thread-id (ct/in-future "1s")))
   ([conn profile-id thread-id mod-at]
    (db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id mod-at mod-at])))
 
@@ -251,34 +234,39 @@
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
-(def ^:private sql:comment-threads
+(defn- get-comment-threads-sql
-  "SELECT DISTINCT ON (ct.id)
+  [where]
-          ct.*,
+  (str/ffmt
-          pf.fullname AS owner_fullname,
+   "SELECT DISTINCT ON (ct.id)
-          pf.email AS owner_email,
+           ct.*,
-          pf.photo_id AS owner_photo_id,
+           pf.fullname AS owner_fullname,
-          p.team_id AS team_id,
+           pf.email AS owner_email,
-          f.name AS file_name,
+           pf.photo_id AS owner_photo_id,
-          f.project_id AS project_id,
+           p.team_id AS team_id,
-          first_value(c.content) OVER w AS content,
+           f.name AS file_name,
-          (SELECT count(1)
+           f.project_id AS project_id,
-             FROM comment AS c
+           first_value(c.content) OVER w AS content,
-            WHERE c.thread_id = ct.id) AS count_comments,
+           (SELECT count(1)
-          (SELECT count(1)
+              FROM comment AS c
-             FROM comment AS c
+             WHERE c.thread_id = ct.id) AS count_comments,
-            WHERE c.thread_id = ct.id
+           (SELECT count(1)
-              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+              FROM comment AS c
-     FROM comment_thread AS ct
+             WHERE c.thread_id = ct.id
-    INNER JOIN comment AS c ON (c.thread_id = ct.id)
+               AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
-    INNER JOIN file AS f ON (f.id = ct.file_id)
+      FROM comment_thread AS ct
-    INNER JOIN project AS p ON (p.id = f.project_id)
+     INNER JOIN comment AS c ON (c.thread_id = ct.id)
-     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+     INNER JOIN file AS f ON (f.id = ct.file_id)
-     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+     INNER JOIN project AS p ON (p.id = f.project_id)
-   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
+      LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+      LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+     WHERE f.deleted_at IS NULL
+       AND p.deleted_at IS NULL
+       %1
+    WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)"
+   where))
 
 (def ^:private sql:comment-threads-by-file-id
-  (str "WITH threads AS (" sql:comment-threads ")"
+  (get-comment-threads-sql "AND ct.file_id = ?"))
-       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
@@ -287,7 +275,30 @@
 
 ;; --- COMMAND: Get Unread Comment Threads
 
-(declare ^:private get-unread-comment-threads)
+(def ^:private sql:unread-all-comment-threads-by-team
+  (str "WITH threads AS ("
+       (get-comment-threads-sql "AND p.team_id = ?")
+       ")"
+       "SELECT t.* FROM threads AS t
+         WHERE t.count_unread_comments > 0"))
+
+(def ^:private sql:unread-partial-comment-threads-by-team
+  (str "WITH threads AS ("
+       (get-comment-threads-sql "AND p.team_id = ? AND (ct.owner_id = ? OR ? = ANY(ct.mentions))")
+       ")"
+       "SELECT t.* FROM threads AS t
+         WHERE t.count_unread_comments > 0"))
+
+(defn- get-unread-comment-threads
+  [cfg profile-id team-id]
+  (let [profile (-> (db/get cfg :profile {:id profile-id} ::db/remove-deleted false)
+                    (profile/decode-row))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)
+        result  (case notify
+                  :all     (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
+                  :partial (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+                  [])]
+    (into [] xf-decode-row result)))
 
 (def ^:private
   schema:get-unread-comment-threads
@@ -298,41 +309,8 @@
   {::doc/added "1.15"
    ::sm/params schema:get-unread-comment-threads}
   [cfg {:keys [::rpc/profile-id team-id] :as params}]
-  (db/run!
+  (teams/check-read-permissions! cfg profile-id team-id)
-   cfg
+  (get-unread-comment-threads cfg profile-id team-id))
-   (fn [{:keys [::db/conn]}]
-     (teams/check-read-permissions! conn profile-id team-id)
-     (get-unread-comment-threads conn profile-id team-id))))
-
-(def sql:unread-all-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
-       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
-
-;; The partial configuration will retrieve only comments created by the user and
-;; threads that have a mention to the user.
-(def sql:unread-partial-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
-       "SELECT * FROM threads
-         WHERE count_unread_comments > 0
-           AND team_id = ?
-           AND (owner_id = ? OR ? = ANY(mentions))"))
-
-(defn- get-unread-comment-threads
-  [conn profile-id team-id]
-  (let [profile (-> (db/get conn :profile {:id profile-id})
-                    (profile/decode-row))
-        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
-
-    (case notify
-      :all
-      (->> (db/exec! conn [sql:unread-all-comment-threads-by-team profile-id team-id])
-           (into [] xf-decode-row))
-
-      :partial
-      (->> (db/exec! conn [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-           (into [] xf-decode-row))
-
-      [])))
 
 ;; --- COMMAND: Get Single Comment Thread
 
@@ -343,16 +321,17 @@
    [:id ::sm/uuid]
    [:share-id {:optional true} [:maybe ::sm/uuid]]])
 
+(def ^:private sql:get-comment-thread
+  (get-comment-threads-sql "AND ct.file_id = ? AND ct.id = ?"))
+
 (sv/defmethod ::get-comment-thread
   {::doc/added "1.15"
    ::sm/params schema:get-comment-thread}
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
+                 (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
-                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                         (decode-row)))))
-                   (-> (db/exec-one! conn [sql profile-id id file-id])
-                       (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
 

backend/src/app/rpc/commands/demo.clj

@@ -7,16 +7,16 @@
 (ns app.rpc.commands.demo
   "A demo specific mutations."
   (:require
+   [app.auth :refer [derive-password]]
    [app.common.exceptions :as ex]
+   [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.auth :as auth]
-   [app.rpc.commands.profile :as profile]
    [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
-   [app.util.time :as dt]
    [buddy.core.codecs :as bc]
    [buddy.core.nonce :as bn]))
 
@@ -39,21 +39,20 @@
         fullname (str "Demo User " sem)
 
         password (-> (bn/random-bytes 16)
-                     (bc/bytes->b64u)
+                     (bc/bytes->b64 true)
                      (bc/bytes->str))
 
         params   {:email email
                   :fullname fullname
                   :is-active true
-                  :deleted-at (dt/in-future (cf/get-deletion-delay))
+                  :is-demo true
-                  :password (profile/derive-password cfg password)
+                  :deleted-at (ct/in-future (cf/get-deletion-delay))
-                  :props {}}]
+                  :password (derive-password password)
-
+                  :props {}}
-
+        profile  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-    (let [profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+                                   (->> (auth/create-profile cfg params)
-                                    (->> (auth/create-profile! conn params)
+                                        (auth/create-profile-rels conn))))]
-                                         (auth/create-profile-rels! conn))))]
+    (with-meta {:email email
-      (with-meta {:email email
+                :password password}
-                  :password password}
+      {::audit/profile-id (:id profile)})))
-        {::audit/profile-id (:id profile)}))))